Java Code Examples for org.apache.hadoop.hbase.client.Get#addColumn()

/**
 * Ensure that optimize does not cause the Get to do more seeking than required. Optimize
 * (see HBASE-15392) was causing us to seek all Cells in a block when a Get Scan if the next block
 * index/start key was a different row to the current one. A bug. We'd call next too often
 * because we had to exhaust all Cells in the current row making us load the next block just to
 * discard what we read there. This test is a little cryptic. Takes a bit of staring to figure
 * what it up to.
 */
@Test
public void testOptimizeAndGetWithFakedNextBlockIndexStart() throws IOException {
  // First test a Get of second column in the row R2. Every Get is a Scan. Second column has a
  // qualifier of R2.
  Get get = new Get(THREE);
  get.addColumn(CF, TWO);
  Scan scan = new Scan(get);
  try (CellGridStoreScanner scanner = new CellGridStoreScanner(scan, this.scanInfo)) {
    List<Cell> results = new ArrayList<>();
    // For a Get there should be no more next's after the first call.
    assertEquals(false, scanner.next(results));
    // Should be one result only.
    assertEquals(1, results.size());
    // And we should have gone through optimize twice only.
    assertEquals("First qcode is SEEK_NEXT_COL and second INCLUDE_AND_SEEK_NEXT_ROW", 2,
      scanner.count.get());
  }
}
 

/**
 * Creates the get request.
 * 
 * @param key
 *          the key
 * @param startTime
 *          the start time
 * @param endTime
 *          the end time
 * @return the gets the
 * @throws IOException
 *           Signals that an I/O exception has occurred.
 */
@VisibleForTesting
Get createGetRequest(String key, long startTime, long endTime)
    throws IOException {
  Get get = new Get(Bytes.toBytes(key));
  // set family name
  get.addFamily(ConfigurationUtil.getColumnFamily());

  // set column family, qualifier
  get.addColumn(ConfigurationUtil.getColumnFamily(),
      ConfigurationUtil.getColumnQualifier());

  // set max versions
  get.setMaxVersions(ConfigurationUtil.getMaxVersions());

  // set time range
  setTimeRangeOnGet(get, startTime, endTime);
  return get;
}
 

/**
 * Returns true if the particular cell passed exists in the datastore.
 * @param row row
 * @param family column family
 * @param qualifier columnn name
 * @param version version
 * @param cellGetter an instance of CellGetter
 * @return true if the cell specified exists. false otherwise
 * @throws IOException
 */
public static boolean hasCell(byte[] row,
                              byte[] family,
                              byte[] qualifier,
                              long version,
                              CellGetter cellGetter)
        throws IOException {
    Get get = new Get(row);
    get.addColumn(family, qualifier);
    get.setTimeStamp(version);

    Result result = cellGetter.get(get);

    return result.containsColumn(family, qualifier);
}
 

/**
 * Returns an instance of Get that retrieves the matches records from the HBase table.
 *
 * @return The appropriate instance of Get for this use case.
 */
public Get createGet(Object rowKey) {
	byte[] rowkey = HBaseTypeUtils.serializeFromObject(
		rowKey,
		rowKeyType,
		charset);
	Get get = new Get(rowkey);
	for (int f = 0; f < families.length; f++) {
		byte[] family = families[f];
		for (byte[] qualifier : qualifiers[f]) {
			get.addColumn(family, qualifier);
		}
	}
	return get;
}
 

@Override
public void open() throws IOException {
    if (isOpen) {
        return; // silently return
    }
    try {
        tbl = EagleConfigFactory.load().getHTable(ed.getTable());
    } catch (RuntimeException ex) {
        throw new IOException(ex);
    }
    final byte[] family = ed.getColumnFamily().getBytes();
    List<Get> gets = new ArrayList<>(this.rowkeys.size());

    for (byte[] rowkey : rowkeys) {
        Get get = new Get(rowkey);
        get.addFamily(family);

        if (qualifiers != null) {
            for (byte[] qualifier : qualifiers) {
                get.addColumn(family, qualifier);
            }
        }

        gets.add(get);
    }

    entityResult = tbl.get(gets);
    isOpen = true;
}
 

/**
 * Ensures that family delete markers are present in the columns requested for any get operation.
 * @param get The original get request
 * @return The modified get request with the family delete qualifiers represented
 */
private Get projectFamilyDeletes(Get get) {
  for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
    NavigableSet<byte[]> columns = entry.getValue();
    // wildcard scans will automatically include the delete marker, so only need to add it when we have
    // explicit columns listed
    if (columns != null && !columns.isEmpty()) {
      get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
    }
  }
  return get;
}
 

@VisibleForTesting
int getRegionCountForTime(Table stateTable, long time) throws IOException {
  Get get = new Get(makeTimeRegionCountKey(Bytes.toBytes(getInvertedTime(time))));
  get.addColumn(FAMILY, REGION_TIME_COL);
  Result result = stateTable.get(get);
  byte[] value = result.getValue(FAMILY, REGION_TIME_COL);
  return value == null ? -1 : Bytes.toInt(value);
}
 

@Override
boolean testRow(final int i) throws IOException, InterruptedException {
  if (opts.randomSleep > 0) {
    Thread.sleep(rd.nextInt(opts.randomSleep));
  }
  Get get = new Get(getRandomRow(this.rand, opts.totalRows));
  for (int family = 0; family < opts.families; family++) {
    byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
    if (opts.addColumns) {
      for (int column = 0; column < opts.columns; column++) {
        byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
        get.addColumn(familyName, qualifier);
      }
    } else {
      get.addFamily(familyName);
    }
  }
  if (opts.filterAll) {
    get.setFilter(new FilterAllFilter());
  }
  get.setConsistency(consistency);
  if (LOG.isTraceEnabled()) LOG.trace(get.toString());
  if (opts.multiGet > 0) {
    this.gets.add(get);
    if (this.gets.size() == opts.multiGet) {
      Result [] rs = this.table.get(this.gets);
      updateValueSize(rs);
      this.gets.clear();
    } else {
      return false;
    }
  } else {
    updateValueSize(this.table.get(get));
  }
  return true;
}
 

/**
 * Ensures that family delete markers are present in the columns requested for any get operation.
 * @param get The original get request
 * @return The modified get request with the family delete qualifiers represented
 */
private Get projectFamilyDeletes(Get get) {
  for (Map.Entry<byte[], NavigableSet<byte[]>> entry : get.getFamilyMap().entrySet()) {
    NavigableSet<byte[]> columns = entry.getValue();
    // wildcard scans will automatically include the delete marker, so only need to add it when we have
    // explicit columns listed
    if (columns != null && !columns.isEmpty()) {
      get.addColumn(entry.getKey(), TxConstants.FAMILY_DELETE_QUALIFIER);
    }
  }
  return get;
}
 

private static Quotas getQuotas(final Connection connection, final byte[] rowKey,
    final byte[] qualifier) throws IOException {
  Get get = new Get(rowKey);
  get.addColumn(QUOTA_FAMILY_INFO, qualifier);
  Result result = doGet(connection, get);
  if (result.isEmpty()) {
    return null;
  }
  return quotasFromData(result.getValue(QUOTA_FAMILY_INFO, qualifier));
}
 

/** Use get to retrieve the HRegionInfo and validate it */
private void getRegionInfo(Table table) throws IOException {
  Get get = new Get(ROW_KEY);
  get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  Result result = table.get(get);
  byte [] bytes = result.value();
  validateRegionInfo(bytes);
}
 

/**
 * creates a Get to be fetch daily aggregation status from the RAW table
 * @param row key
 * @return {@link Get}
 * @throws IOException
 */
public boolean getStatusAgg(byte[] row, byte[] col) throws IOException {
  Get g = new Get(row);
  g.addColumn(Constants.INFO_FAM_BYTES, col);
  Table rawTable = null;
  Cell cell = null;
  try {
    rawTable = hbaseConnection
        .getTable(TableName.valueOf(Constants.HISTORY_RAW_TABLE));
    Result r = rawTable.get(g);
    cell = r.getColumnLatestCell(Constants.INFO_FAM_BYTES, col);
  } finally {
    if (rawTable != null) {
      rawTable.close();
    }
  }
  boolean status = false;
  try {
    if (cell != null) {
      status = Bytes.toBoolean(CellUtil.cloneValue(cell));
    }
  } catch (IllegalArgumentException iae) {
    LOG.error("Caught " + iae);
  }
  LOG.info("Returning from Raw, " + Bytes.toString(col) + " for this job="
      + status);
  return status;
}
 

public static Result exist(Connection conn, TableName tableName, byte[] key) throws IOException {
    Table htable = conn.getTable(tableName);
    Get get = new Get(key);
    get.addColumn(DATA_COLUMN_FAMILY_BYTES, SYS_COLUMN_VERSION_BYTES);
    Result r = htable.get(get);
    return r;
}
 

private RegionState.State getCurrentRegionState(RegionInfo regionInfo) throws IOException{
  Table metaTable = TEST_UTIL.getConnection().getTable(TableName.valueOf("hbase:meta"));
  Get get = new Get(regionInfo.getRegionName());
  get.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER);
  Result result = metaTable.get(get);
  byte[] currentStateValue = result.getValue(HConstants.CATALOG_FAMILY,
    HConstants.STATE_QUALIFIER);
  return currentStateValue != null ?
    RegionState.State.valueOf(Bytes.toString(currentStateValue))
    : null;
}
 

@Test(timeOut = 60_000)
public void testScenario1() throws Exception {
    try (TTable txTable = new TTable(connection, TEST_TABLE)) {

        // Write initial values for the test
        HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
        long initialEpoch = tx0.getEpoch();
        LOG.info("Starting Tx {} writing initial values for cells ({}) ", tx0, Bytes.toString(initialData));
        Put putInitialDataRow1 = new Put(row1);
        putInitialDataRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1, initialData);
        txTable.put(tx0, putInitialDataRow1);
        Put putInitialDataRow2 = new Put(row2);
        putInitialDataRow2.addColumn(TEST_FAMILY.getBytes(), qualifier2, initialData);
        txTable.put(tx0, putInitialDataRow2);
        tm.commit(tx0);

        // Initial checks
        checkRowValues(txTable, initialData, initialData);

        HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
        LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx1, Bytes.toString(data1_q1),
                 Bytes.toString(data1_q2));
        Put putData1R1Q1 = new Put(row1);
        putData1R1Q1.addColumn(TEST_FAMILY.getBytes(), qualifier1, data1_q1);
        txTable.put(tx1, putData1R1Q1);
        Put putData1R2Q2 = new Put(row2);
        putData1R2Q2.addColumn(TEST_FAMILY.getBytes(), qualifier2, data1_q2);
        txTable.put(tx1, putData1R2Q2);

        Transaction interleavedReadTx = tm.begin();

        LOG.info("Starting Interleaving Read Tx {} for checking cell values", interleavedReadTx.getTransactionId());

        // Simulate a GC pause to change mastership (should throw a ServiceUnavailable exception)
        LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
        LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
        LOG.info("++++++++++++++++++++ PAUSING TSO 1 +++++++++++++++++++");
        LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
        LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
        leaseManager1.pausedInStillInLeasePeriod();

        // Read interleaved and check the values writen by tx 1
        Get getRow1 = new Get(row1).setMaxVersions(1);
        getRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1);
        Result r = txTable.get(interleavedReadTx, getRow1);
        assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), initialData,
                     "Unexpected value for SI read R1Q1" + interleavedReadTx + ": "
                             + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));

        // Try to commit, but it should abort due to the change in mastership
        try {
            tm.commit(tx1);
            fail();
        } catch (RollbackException e) {
            // Expected
            LOG.info("Rollback cause for Tx {}: ", tx1, e.getCause());
            assertEquals(tx1.getStatus(), Transaction.Status.ROLLEDBACK);
            assertEquals(tx1.getEpoch(), initialEpoch);
        }

        // Read interleaved and check the values written by tx 1
        Get getRow2 = new Get(row2).setMaxVersions(1);
        r = txTable.get(interleavedReadTx, getRow2);
        assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), initialData,
                     "Unexpected value for SI read R2Q2" + interleavedReadTx + ": "
                             + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));

        // Should commit because its a read only tx does not have to contact the TSO
        tm.commit(interleavedReadTx);
        assertEquals(interleavedReadTx.getEpoch(), initialEpoch);
        assertEquals(interleavedReadTx.getStatus(), Transaction.Status.COMMITTED_RO);

        LOG.info("Wait till the client is informed about the connection parameters of the new TSO");
        TestUtils.waitForSocketListening("localhost", TSO2_PORT, 100);

        checkRowValues(txTable, initialData, initialData);

        // Need to resume to let other test progress
        leaseManager1.resume();

    }

}
 

/**
 * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift).
 *
 * This ignores any timestamps set on {@link TColumn} objects.
 *
 * @param in the <code>TGet</code> to convert
 *
 * @return <code>Get</code> object
 *
 * @throws IOException if an invalid time range or max version parameter is given
 */
public static Get getFromThrift(TGet in) throws IOException {
  Get out = new Get(in.getRow());

  // Timestamp overwrites time range if both are set
  if (in.isSetTimestamp()) {
    out.setTimestamp(in.getTimestamp());
  } else if (in.isSetTimeRange()) {
    out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp());
  }

  if (in.isSetMaxVersions()) {
    out.readVersions(in.getMaxVersions());
  }

  if (in.isSetFilterString()) {
    ParseFilter parseFilter = new ParseFilter();
    out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
  }

  if (in.isSetAttributes()) {
    addAttributes(out,in.getAttributes());
  }

  if (in.isSetAuthorizations()) {
    out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
  }

  if (in.isSetConsistency()) {
    out.setConsistency(consistencyFromThrift(in.getConsistency()));
  }

  if (in.isSetTargetReplicaId()) {
    out.setReplicaId(in.getTargetReplicaId());
  }

  if (in.isSetCacheBlocks()) {
    out.setCacheBlocks(in.isCacheBlocks());
  }
  if (in.isSetStoreLimit()) {
    out.setMaxResultsPerColumnFamily(in.getStoreLimit());
  }
  if (in.isSetStoreOffset()) {
    out.setRowOffsetPerColumnFamily(in.getStoreOffset());
  }
  if (in.isSetExistence_only()) {
    out.setCheckExistenceOnly(in.isExistence_only());
  }

  if (in.isSetColumns()) {
    for (TColumn column : in.getColumns()) {
      if (column.isSetQualifier()) {
        out.addColumn(column.getFamily(), column.getQualifier());
      } else {
        out.addFamily(column.getFamily());
      }
    }
  }

  if (in.isSetFilterBytes()) {
    out.setFilter(filterFromThrift(in.getFilterBytes()));
  }
  return out;
}
 

@Test(timeOut = 60_000)
public void testShadowCellIsHealedAfterCommitCrash(ITestContext context) throws Exception {

    CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());

    HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
    hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
    hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
    PostCommitActions syncPostCommitter = spy(
            new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient, connection));
    AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
            .postCommitter(syncPostCommitter)
            .commitTableWriter(getCommitTable(context).getWriter())
            .commitTableClient(commitTableClient)
            .build());

    // The following line emulates a crash after commit that is observed in (*) below
    doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));

    TTable table = new TTable(connection, TEST_TABLE);

    HBaseTransaction t1 = (HBaseTransaction) tm.begin();

    // Test shadow cell are created properly
    Put put = new Put(row);
    put.addColumn(family, qualifier, data1);
    table.put(t1, put);
    try {
        tm.commit(t1);
    } catch (Exception e) { // (*) Crash
        // Do nothing
    }

    assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
            "Cell should be there");
    assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
            "Shadow cell should not be there");

    Transaction t2 = tm.begin();
    Get get = new Get(row);
    get.addColumn(family, qualifier);

    // This get should heal the shadow cell
    Result getResult = table.get(t2, get);
    assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
    verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());

    assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
            "Cell should be there");
    assertTrue(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
            "Shadow cell should be there after being healed");

    // As the shadow cell is healed, this get shouldn't have to hit the storage,
    // so the number of invocations to commitTableClient.getCommitTimestamp()
    // should remain the same
    getResult = table.get(t2, get);
    assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
    verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
}
 

public RowResultGenerator(final String tableName, final RowSpec rowspec,
    final Filter filter, final boolean cacheBlocks)
    throws IllegalArgumentException, IOException {
  try (Table table = RESTServlet.getInstance().getTable(tableName)) {
    Get get = new Get(rowspec.getRow());
    if (rowspec.hasColumns()) {
      for (byte[] col : rowspec.getColumns()) {
        byte[][] split = CellUtil.parseColumn(col);
        if (split.length == 1) {
          get.addFamily(split[0]);
        } else if (split.length == 2) {
          get.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid column specifier.");
        }
      }
    }
    get.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    get.readVersions(rowspec.getMaxVersions());
    if (filter != null) {
      get.setFilter(filter);
    }
    get.setCacheBlocks(cacheBlocks);
    Result result = table.get(get);
    if (result != null && !result.isEmpty()) {
      valuesI = result.listCells().iterator();
    }
  } catch (DoNotRetryIOException e) {
    // Warn here because Stargate will return 404 in the case if multiple
    // column families were specified but one did not exist -- currently
    // HBase will fail the whole Get.
    // Specifying multiple columns in a URI should be uncommon usage but
    // help to avoid confusion by leaving a record of what happened here in
    // the log.
    LOG.warn(StringUtils.stringifyException(e));
    // Lets get the exception rethrown to get a more meaningful error message than 404
    if (e instanceof AccessDeniedException) {
      throw e;
    }
  }
}
 

@Test
// HBase-3583
public void testHBase3583() throws IOException {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  util.createTable(tableName, new byte[][] { A, B, C });
  util.waitUntilAllRegionsAssigned(tableName);

  verifyMethodResult(SimpleRegionObserver.class,
    new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" },
    tableName, new Boolean[] { false, false, false, false });

  Table table = util.getConnection().getTable(tableName);
  Put put = new Put(ROW);
  put.addColumn(A, A, A);
  table.put(put);

  Get get = new Get(ROW);
  get.addColumn(A, A);
  table.get(get);

  // verify that scannerNext and scannerClose upcalls won't be invoked
  // when we perform get().
  verifyMethodResult(SimpleRegionObserver.class,
    new String[] { "hadPreGet", "hadPostGet", "wasScannerNextCalled", "wasScannerCloseCalled" },
    tableName, new Boolean[] { true, true, false, false });

  Scan s = new Scan();
  ResultScanner scanner = table.getScanner(s);
  try {
    for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    }
  } finally {
    scanner.close();
  }

  // now scanner hooks should be invoked.
  verifyMethodResult(SimpleRegionObserver.class,
    new String[] { "wasScannerNextCalled", "wasScannerCloseCalled" }, tableName,
    new Boolean[] { true, true });
  util.deleteTable(tableName);
  table.close();
}
 

/**
 * Parses a combined family and qualifier and adds either both or just the
 * family in case there is not qualifier. This assumes the older colon
 * divided notation, e.g. "data:contents" or "meta:".
 * <p>
 * Note: It will through an error when the colon is missing.
 * 
 * @param familyAndQualifier
 *            family and qualifier
 * @return A reference to this instance.
 * @throws IllegalArgumentException
 *             When the colon is missing.
 */
public static void addColumn(Get get, byte[] familyAndQualifier) {
	byte[][] fq = KeyValue.parseColumn(familyAndQualifier);
	if (fq.length > 1 && fq[1] != null && fq[1].length > 0) {
		get.addColumn(fq[0], fq[1]);
	} else {
		get.addFamily(fq[0]);
	}
}