Java Code Examples for org.apache.hadoop.hbase.client.Table#put()

public void run(Connection connection, List<String> args) throws InvalidArgsException, IOException {
    if (args.size() != 5) {
        throw new InvalidArgsException(args);
    }

    // Get the arguments passed by the user.
    String tableName = args.get(0);
    String rowId = args.get(1);
    String columnFamily = args.get(2);
    String column = args.get(3);
    String value = args.get(4);

    Table table = connection.getTable(TableName.valueOf(tableName));

    // Create a new Put request.
    Put put = new Put(Bytes.toBytes(rowId));

    // Here we add only one column value to the row but
    // multiple column values can be added to the row at
    // once by calling this method multiple times.
    put.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(column), Bytes.toBytes(value));

    // Execute the put on the table.
    table.put(put);
}
 

@Test
public void testReadWrite() throws IOException {
  byte[] row = Bytes.toBytes("row");
  byte[] cf = Bytes.toBytes("cf");
  byte[] cq = Bytes.toBytes("cq");
  byte[] value = Bytes.toBytes("value");

  TableName name = TableName.valueOf(getClass().getSimpleName());

  Table t = UTIL.createTable(name, cf);
  t.put(new Put(row).addColumn(cf, cq, value));

  UTIL.getAdmin().flush(name);

  assertArrayEquals(value, t.get(new Get(row)).getValue(cf, cq));
}
 

private static void loadData(final Table ht, final byte[][] families,
    final int rows, final int flushes) throws IOException {
  List<Put> puts = new ArrayList<>(rows);
  byte[] qualifier = Bytes.toBytes("val");
  for (int i = 0; i < flushes; i++) {
    for (int k = 0; k < rows; k++) {
      byte[] row = Bytes.toBytes(random.nextLong());
      Put p = new Put(row);
      for (int j = 0; j < families.length; ++j) {
        p.addColumn(families[j], qualifier, row);
      }
      puts.add(p);
    }
    ht.put(puts);
    TEST_UTIL.flush();
    puts.clear();
  }
}
 

@Test
public void testAsyncSnapshotWillNotBlockSnapshotHFileCleaner() throws Exception {
  // Write some data
  Table table = UTIL.getConnection().getTable(TABLE_NAME);
  for (int i = 0; i < 10; i++) {
    Put put = new Put(Bytes.toBytes(i)).addColumn(TEST_FAM, Bytes.toBytes("q"), Bytes.toBytes(i));
    table.put(put);
  }
  String snapshotName = "testAsyncSnapshotWillNotBlockSnapshotHFileCleaner01";
  Future<Void> future =
    UTIL.getAdmin().snapshotAsync(new org.apache.hadoop.hbase.client.SnapshotDescription(
      snapshotName, TABLE_NAME, SnapshotType.FLUSH));
  Waiter.waitFor(UTIL.getConfiguration(), 10 * 1000L, 200L,
    () -> UTIL.getAdmin().listSnapshots(Pattern.compile(snapshotName)).size() == 1);
  UTIL.waitFor(30000, () -> !master.getSnapshotManager().isTakingAnySnapshot());
}
 

@Override
public void save(String tableName, String id, Map<String, Object> map) {
    if (isDisabled() || validator.isEmpty(id) || validator.isEmpty(map))
        return;

    try {
        Table table = getTable(tableName);
        delete(table, id);
        Put put = new Put(Bytes.toBytes(id));
        map.forEach((key, value) -> put.addColumn(Bytes.toBytes(key), null, Bytes.toBytes(converter.toString(value))));
        table.put(put);
        table.close();
    } catch (IOException e) {
        logger.warn(e, "保存数据[{}:{}]到HBase[{}]时发生异常！", id, converter.toString(map), tableName);
    }
}
 

private HStore prepareData() throws IOException {
  Admin admin = TEST_UTIL.getAdmin();
  TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
      .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
        FIFOCompactionPolicy.class.getName())
      .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
        DisabledRegionSplitPolicy.class.getName())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
      .build();
  admin.createTable(desc);
  Table table = TEST_UTIL.getConnection().getTable(tableName);
  TimeOffsetEnvironmentEdge edge =
      (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate();
  for (int i = 0; i < 10; i++) {
    for (int j = 0; j < 10; j++) {
      byte[] value = new byte[128 * 1024];
      ThreadLocalRandom.current().nextBytes(value);
      table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value));
    }
    admin.flush(tableName);
    edge.increment(1001);
  }
  return getStoreWithName(tableName);
}
 

private static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
    throws Exception {
  Table table = null;
  try {
    table = TEST_UTIL.createTable(tableName, fam);
    int i = 1;
    List<Put> puts = new ArrayList<>(labelExps.length);
    for (String labelExp : labelExps) {
      Put put = new Put(Bytes.toBytes("row" + i));
      put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
      put.setCellVisibility(new CellVisibility(labelExp));
      puts.add(put);
      i++;
    }
    table.put(puts);
  } finally {
    if (table != null) {
      table.close();
    }
  }
  return table;
}
 

@Test
public void testReverseMultiRowRangeFilterIncludingMaxRow() throws IOException {
  tableName = TableName.valueOf(name.getMethodName());
  Table ht = TEST_UTIL.createTable(tableName, family);
  for (String rowkey : Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h")) {
    byte[] row = Bytes.toBytes(rowkey);
    Put p = new Put(row);
    p.addColumn(family, qf, value);
    ht.put(p);
  }
  TEST_UTIL.flush();

  Scan scan = new Scan();
  scan.setReversed(true);
  List<RowRange> ranges = Arrays.asList(
      new RowRange(Bytes.toBytes("b"), true, Bytes.toBytes("c"), true),
      new RowRange(Bytes.toBytes("f"), true, Bytes.toBytes("h"), true)
  );
  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);

  List<String> expected = Arrays.asList("h", "g", "f", "c", "b");
  List<String> actual = new ArrayList<>();
  for (Cell cell : getResults(ht, scan)) {
    actual.add(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
  }

  assertEquals(expected, actual);
}
 

private void populateTable(Table tbl) throws IOException {
  byte[] values = { 'A', 'B', 'C', 'D' };
  List<Put> puts = new ArrayList<>();
  for (int i = 0; i < values.length; i++) {
    for (int j = 0; j < values.length; j++) {
      Put put = new Put(new byte[] { values[i], values[j] });
      put.addColumn(Bytes.toBytes("fam"), new byte[]{}, new byte[]{values[i],
              values[j]});
      puts.add(put);
    }
  }
  tbl.put(puts);
}
 

/**
 * Moves a flow_queue record from one row key to another. All Cells in the
 * existing row will be written to the new row. This would primarily be used
 * for transitioning a flow's data from one status to another.
 *
 * @param oldKey the existing row key to move
 * @param newKey the new row key to move to
 * @throws IOException
 */
public void moveFlow(FlowQueueKey oldKey, FlowQueueKey newKey)
    throws DataException, IOException {
  byte[] oldRowKey = queueKeyConverter.toBytes(oldKey);
  Get get = new Get(oldRowKey);
  Table flowQueueTable = null;
  try {
    flowQueueTable = hbaseConnection
        .getTable(TableName.valueOf(Constants.FLOW_QUEUE_TABLE));
    Result result = flowQueueTable.get(get);
    if (result == null || result.isEmpty()) {
      // no existing row
      throw new DataException(
          "No row for key " + Bytes.toStringBinary(oldRowKey));
    }
    // copy the existing row to the new key
    Put p = new Put(queueKeyConverter.toBytes(newKey));
    for (Cell c : result.rawCells()) {
      p.addColumn(CellUtil.cloneFamily(c), CellUtil.cloneQualifier(c),
          CellUtil.cloneValue(c));
    }
    flowQueueTable.put(p);
    // delete the old row
    Delete d = new Delete(oldRowKey);
    flowQueueTable.delete(d);
  } finally {
    if (flowQueueTable != null) {
      flowQueueTable.close();
    }
  }
}
 

@Test public void testInterruptingRunningCompactions() throws Exception {
  // setup a compact/split thread on a mock server
  conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY,
      WaitThroughPutController.class.getName());
  HRegionServer mockServer = Mockito.mock(HRegionServer.class);
  Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
  CompactSplit thread = new CompactSplit(mockServer);

  Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);

  // setup a region/store with some files
  HStore store = r.getStore(COLUMN_FAMILY);
  int jmax = (int) Math.ceil(15.0 / compactionThreshold);
  byte[] pad = new byte[1000]; // 1 KB chunk
  for (int i = 0; i < compactionThreshold; i++) {
    Table loader = new RegionAsTable(r);
    Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
    p.setDurability(Durability.SKIP_WAL);
    for (int j = 0; j < jmax; j++) {
      p.addColumn(COLUMN_FAMILY, Bytes.toBytes(j), pad);
    }
    HTestConst.addContent(loader, Bytes.toString(COLUMN_FAMILY));
    loader.put(p);
    r.flush(true);
  }
  HStore s = r.getStore(COLUMN_FAMILY);
  int initialFiles = s.getStorefilesCount();

  thread.requestCompaction(r, store, "test custom comapction", PRIORITY_USER,
      CompactionLifeCycleTracker.DUMMY, null);

  Thread.sleep(3000);
  thread.switchCompaction(false);
  assertEquals(initialFiles, s.getStorefilesCount());
  //don't mess up future tests
  thread.switchCompaction(true);
}
 

/**
 * Test that we run a passing constraint
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testConstraintPasses() throws Exception {
  // create the table
  // it would be nice if this was also a method on the util
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  for (byte[] family : new byte[][]{dummy, test}) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  // add a constraint
  Constraints.add(tableDescriptor, CheckWasRunConstraint.class);

  util.getAdmin().createTable(tableDescriptor);
  Table table = util.getConnection().getTable(tableName);
  try {
    // test that we don't fail on a valid put
    Put put = new Put(row1);
    byte[] value = Bytes.toBytes(Integer.toString(10));
    byte[] qualifier = new byte[0];
    put.addColumn(dummy, qualifier, value);
    table.put(put);
  } finally {
    table.close();
  }
  assertTrue(CheckWasRunConstraint.wasRun);
}
 

@Test
public void testFiltersWithOR() throws Exception {
  TableName tn = TableName.valueOf(name.getMethodName());
  Table table = TEST_UTIL.createTable(tn, new String[] { "cf1", "cf2" });
  byte[] CF1 = Bytes.toBytes("cf1");
  byte[] CF2 = Bytes.toBytes("cf2");
  Put put1 = new Put(Bytes.toBytes("0"));
  put1.addColumn(CF1, Bytes.toBytes("col_a"), Bytes.toBytes(0));
  table.put(put1);
  Put put2 = new Put(Bytes.toBytes("0"));
  put2.addColumn(CF2, Bytes.toBytes("col_b"), Bytes.toBytes(0));
  table.put(put2);
  FamilyFilter filterCF1 =
      new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF1));
  FamilyFilter filterCF2 =
      new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF2));
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  filterList.addFilter(filterCF1);
  filterList.addFilter(filterCF2);
  Scan scan = new Scan();
  scan.setFilter(filterList);
  ResultScanner scanner = table.getScanner(scan);
  LOG.info("Filter list: " + filterList);
  for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    Assert.assertEquals(2, rr.size());
  }
}
 

private void testPreWALAppendHook(Table table, TableName tableName) throws IOException {
  int expectedCalls = 0;
  String [] methodArray = new String[1];
  methodArray[0] = "getCtPreWALAppend";
  Object[] resultArray = new Object[1];

  Put p = new Put(ROW);
  p.addColumn(A, A, A);
  table.put(p);
  resultArray[0] = ++expectedCalls;
  verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);

  Append a = new Append(ROW);
  a.addColumn(B, B, B);
  table.append(a);
  resultArray[0] = ++expectedCalls;
  verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);

  Increment i = new Increment(ROW);
  i.addColumn(C, C, 1);
  table.increment(i);
  resultArray[0] = ++expectedCalls;
  verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);

  Delete d = new Delete(ROW);
  table.delete(d);
  resultArray[0] = ++expectedCalls;
  verifyMethodResult(SimpleRegionObserver.class, methodArray, tableName, resultArray);
}
 

public static void writeSimple(String projectId, String instanceId, String tableId) {
  // String projectId = "my-project-id";
  // String instanceId = "my-instance-id";
  // String tableId = "mobile-time-series";

  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    final Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(tableId)));
    long timestamp = System.currentTimeMillis();
    byte[] one = new byte[]{0, 0, 0, 0, 0, 0, 0, 1};

    String rowKey = "phone#4c410523#20190501";
    Put put = new Put(Bytes.toBytes(rowKey));

    put.addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_cell"), timestamp, one);
    put.addColumn(COLUMN_FAMILY_NAME, Bytes.toBytes("connected_wifi"), timestamp, one);
    put.addColumn(
        COLUMN_FAMILY_NAME,
        Bytes.toBytes("os_build"),
        timestamp,
        Bytes.toBytes("PQ2A.190405.003"));
    table.put(put);

    System.out.printf("Successfully wrote row %s", rowKey);

  } catch (Exception e) {
    System.out.println("Error during WriteSimple: \n" + e.toString());
  }
}
 

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
 

@Test
public void Test() throws Exception {
  String cf = "f";
  Table ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
          Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i1 = 0; i1 < 2; i1++) {
    for (int i2 = 0; i2 < 5; i2++) {
      byte[] rk = new byte[10];

      ByteBuffer buf = ByteBuffer.wrap(rk);
      buf.clear();
      buf.putShort((short) 2);
      buf.putInt(i1);
      buf.putInt(i2);

      for (int c = 0; c < 5; c++) {
        byte[] cq = new byte[4];
        Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

        Put p = new Put(rk);
        p.setDurability(Durability.SKIP_WAL);
        p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
        ht.put(p);
        LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
                + Bytes.toStringBinary(cq));
      }
    }
  }

  TEST_UTIL.flush();

  // test passes
  runTest(ht, 0, 10);

  // test fails
  runTest(ht, 1, 8);
}
 

@Test
public void testBulkLoad() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  long l = System.currentTimeMillis();
  Admin admin = TEST_UTIL.getAdmin();
  createTable(admin, tableName);
  Scan scan = createScan();
  final Table table = init(admin, l, scan, tableName);
  // use bulkload
  final Path hfilePath = writeToHFile(l, "/temp/testBulkLoad/", "/temp/testBulkLoad/col/file",
    false);
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.mapreduce.bulkload.assign.sequenceNumbers", true);
  BulkLoadHFiles.create(conf).bulkLoad(tableName, hfilePath);
  ResultScanner scanner = table.getScanner(scan);
  Result result = scanner.next();
  result = scanAfterBulkLoad(scanner, result, "version2");
  Put put0 = new Put(Bytes.toBytes("row1"));
  put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes
      .toBytes("version3")));
  table.put(put0);
  admin.flush(tableName);
  scanner = table.getScanner(scan);
  result = scanner.next();
  while (result != null) {
    List<Cell> cells = result.getColumnCells(Bytes.toBytes("col"), Bytes.toBytes("q"));
    for (Cell _c : cells) {
      if (Bytes.toString(_c.getRowArray(), _c.getRowOffset(), _c.getRowLength())
          .equals("row1")) {
        System.out
            .println(Bytes.toString(_c.getRowArray(), _c.getRowOffset(), _c.getRowLength()));
        System.out.println(Bytes.toString(_c.getQualifierArray(), _c.getQualifierOffset(),
          _c.getQualifierLength()));
        System.out.println(
          Bytes.toString(_c.getValueArray(), _c.getValueOffset(), _c.getValueLength()));
        Assert.assertEquals("version3",
          Bytes.toString(_c.getValueArray(), _c.getValueOffset(), _c.getValueLength()));
      }
    }
    result = scanner.next();
  }
  scanner.close();
  table.close();
}
 

/**
 * Test a case when the timerange is specified with --starttime and --endtime options
 *
 * @throws Exception in case of any unexpected error.
 */
@Test
public void testCreateSubmittableJobWithArgsTimeRange() throws Exception {
  final byte[] family = Bytes.toBytes(COL_FAM);
  final byte[] col1 = Bytes.toBytes(COL1);
  Put put1 = new Put(Bytes.toBytes("row_timerange_" + 1));
  Put put2 = new Put(Bytes.toBytes("row_timerange_" + 2));
  Put put3 = new Put(Bytes.toBytes("row_timerange_" + 3));

  long ts;

  String tableName = TABLE_NAME_TS_RANGE+"CreateSubmittableJobWithArgs";
  // clean up content of TABLE_NAME
  Table table = TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(COL_FAM));

  ts = System.currentTimeMillis();
  put1.addColumn(family, col1, ts, Bytes.toBytes("val1"));
  table.put(put1);
  Thread.sleep(100);

  ts = System.currentTimeMillis();
  put2.addColumn(family, col1, ts, Bytes.toBytes("val2"));
  put3.addColumn(family, col1, ts, Bytes.toBytes("val3"));
  table.put(put2);
  table.put(put3);
  table.close();

  String[] args = new String[] {
    tableName, COL_FAM + ":" + COL1,
    "--starttime=" + 0,
    "--endtime=" + ts
  };
  runCreateSubmittableJobWithArgs(args, 1);

  args = new String[] {
    tableName, COL_FAM + ":" + COL1,
    "--starttime=" + 0,
    "--endtime=" + (ts - 10)
  };
  runCreateSubmittableJobWithArgs(args, 1);

  args = new String[] {
    tableName, COL_FAM + ":" + COL1,
    "--starttime=" + ts,
    "--endtime=" + (ts + 1000)
  };
  runCreateSubmittableJobWithArgs(args, 2);

  args = new String[] {
    tableName, COL_FAM + ":" + COL1,
    "--starttime=" + (ts - 30 * 1000),
    "--endtime=" + (ts + 30 * 1000),
  };
  runCreateSubmittableJobWithArgs(args, 3);
}
 

/**
 * See HBASE-17611
 *
 * Latency metrics were capped at ~ 2 seconds due to the use of an int variable to capture the
 * duration.
 */
@Test
public void testMetricsPrecision() throws Exception {
  byte[] rowkey = Bytes.toBytes("row1");
  byte[] family = Bytes.toBytes("f");
  byte[] col = Bytes.toBytes("c");
  // create a table which will throw exceptions for requests
  TableName tableName = TableName.valueOf("testMetricsPrecision");
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setCoprocessor(DelayingRegionObserver.class.getName());
  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));

  Table table = null;
  try {
    table = UTIL.createTable(tableDescriptor, null);

    table.put(new Put(rowkey).addColumn(family, col, Bytes.toBytes("val1")));

    ThriftHBaseServiceHandler hbaseHandler = createHandler();
    ThriftMetrics metrics = getMetrics(UTIL.getConfiguration());
    THBaseService.Iface handler =
        HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, null);
    ByteBuffer tTableName = wrap(tableName.getName());

    // check metrics latency with a successful get
    TGet tGet = new TGet(wrap(rowkey));
    TResult tResult = handler.get(tTableName, tGet);

    List<TColumnValue> expectedColumnValues = Lists.newArrayList(
        new TColumnValue(wrap(family), wrap(col), wrap(Bytes.toBytes("val1")))
    );
    assertArrayEquals(rowkey, tResult.getRow());
    List<TColumnValue> returnedColumnValues = tResult.getColumnValues();
    assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);

    metricsHelper.assertGaugeGt("get_max", 3000L, metrics.getSource());
  } finally {
    if (table != null) {
      try {
        table.close();
      } catch (IOException ignored) {
      }
      UTIL.deleteTable(tableName);
    }
  }
}