Java Code Examples for org.apache.hadoop.hbase.util.Bytes#toBytes()

public void addNodes(ArrayList<NodePojo> nodes) throws IOException {
  Table nodeTable = connection.getTable(HBaseConst.NODE_LIST_TABLE);

  ArrayList<Put> putList = new ArrayList<Put>();

  for (NodePojo node: nodes) {
    Put put = new Put(Bytes.toBytes(node.getIpAddress()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Nm"), Bytes.toBytes(node.getName()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Zip"), Bytes.toBytes(node.getZipCode()));
    put.addColumn(HBaseConst.BASE_COLUMN_FAMILY, Bytes.toBytes("Grp"), Bytes.toBytes(node.getGroup()));
    putList.add(put);
  }
  nodeTable.put(putList);

  nodeTable.close();
}
 

public static KeyValue create(
    String row,
    String family,
    String qualifier,
    long timestamp,
    KeyValue.Type type,
    String value)
{
    return new KeyValue(
        Bytes.toBytes(row),
        Bytes.toBytes(family),
        Bytes.toBytes(qualifier),
        timestamp,
        type,
        Bytes.toBytes(value)
    );
}
 

/**
 * Run MR job to test autobalance for setting number of mappers for TIF This does not run real MR
 * job
 */
protected void testAutobalanceNumOfSplit() throws IOException {
  // set up splits for testing
  List<InputSplit> splits = new ArrayList<>(5);
  int[] regionLen = { 10, 20, 20, 40, 60 };
  for (int i = 0; i < 5; i++) {
    InputSplit split = new TableSplit(TABLE_NAME, new Scan(), Bytes.toBytes(i),
      Bytes.toBytes(i + 1), "", "", regionLen[i] * 1048576);
    splits.add(split);
  }
  TableInputFormat tif = new TableInputFormat();
  List<InputSplit> res = tif.calculateAutoBalancedSplits(splits, 1073741824);

  assertEquals("Saw the wrong number of splits", 5, res.size());
  TableSplit ts1 = (TableSplit) res.get(0);
  assertEquals("The first split end key should be", 2, Bytes.toInt(ts1.getEndRow()));
  TableSplit ts2 = (TableSplit) res.get(1);
  assertEquals("The second split regionsize should be", 20 * 1048576, ts2.getLength());
  TableSplit ts3 = (TableSplit) res.get(2);
  assertEquals("The third split start key should be", 3, Bytes.toInt(ts3.getStartRow()));
  TableSplit ts4 = (TableSplit) res.get(4);
  assertNotEquals("The seventh split start key should not be", 4, Bytes.toInt(ts4.getStartRow()));
}
 

/**
 * 数据表trace, 保存一个跟踪链条的所有span信息
 * rowkey: traceId
 * columnFamily: span
 * qualifier: [spanId+c, spanId+s ...](有N个span就有N*2个, c/s表示是client还是server采集到的)
 * value: span json value
 * @param span
 * @param spanJson
 * @param annotationMap
 * @return
 */
@Override
public Put storeSpan(Span span, String spanJson, Map<String, Annotation> annotationMap) {
    String traceId = span.getTraceId();
    String spanId = span.getId();
    if (annotationMap.containsKey(AnnotationType.CS.symbol())) {
        // 如果是client
        spanId += NodeProperty.C.symbol();
    } else {
        // 如果是server
        spanId += NodeProperty.S.symbol();
    }
    Put put = new Put(Bytes.toBytes(traceId));
    put.addColumn(Bytes.toBytes(Constants.TABLE_TRACE_COLUMN_FAMILY), Bytes.toBytes(spanId), Bytes.toBytes(spanJson));

    return put;
}
 

CrossRowCellIncrementer(final int i, final int count, final HRegion region, final int range) {
  super("" + i);
  setDaemon(true);
  this.count = count;
  this.region = region;
  this.increments = new Increment[range];
  for (int ii = 0; ii < range; ii++) {
    this.increments[ii] = new Increment(Bytes.toBytes(i));
    this.increments[ii].addColumn(INCREMENT_BYTES, INCREMENT_BYTES, 1);
  }
}
 

@Test(timeOut = 60_000)
public void testScanSecondResult() throws Throwable {

    byte[] rowName1 = Bytes.toBytes("row1");
    byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
    byte[] colName1 = Bytes.toBytes("col1");
    byte[] dataValue1 = Bytes.toBytes("testWrite-1");

    String TEST_TABLE = "testScanSecondResult";
    createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
    TTable tt = new TTable(connection, TEST_TABLE);

    Transaction tx1 = tm.begin();

    Put put1 = new Put(rowName1);
    put1.addColumn(famName1, colName1, dataValue1);
    tt.put(tx1, put1);

    tm.commit(tx1);

    Transaction tx2 = tm.begin();

    Put put2 = new Put(rowName1);
    put2.addColumn(famName1, colName1, dataValue1);
    tt.put(tx2, put2);

    Transaction tx3 = tm.begin();

    ResultScanner iterableRS = tt.getScanner(tx3, new Scan().setStartRow(rowName1).setStopRow(rowName1));
    Result result = iterableRS.next();
    long tsRow = result.rawCells()[0].getTimestamp();
    assertEquals(tsRow, tx1.getTransactionId(), "Reading differnt version");

    assertFalse(iterableRS.next() != null);

    tm.commit(tx3);

    tt.close();
}
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    String jdbcUrl = getJdbcUrl(conf);

    // pass client configuration into driver
    Properties clientInfos = new Properties();
    Iterator<Entry<String, String>> iterator = conf.iterator();
    while(iterator.hasNext()) {
        Entry<String,String> entry = iterator.next();
        clientInfos.setProperty(entry.getKey(), entry.getValue());
    }
    
    // This statement also ensures that the driver class is loaded
    LOG.info("Connection with driver {} with url {}", PhoenixDriver.class.getName(), jdbcUrl);

    try {
        conn = (PhoenixConnection) DriverManager.getConnection(jdbcUrl, clientInfos);
    } catch (SQLException e) {
        throw new RuntimeException(e);
    }

    upsertListener = new MapperUpsertListener(
            context, conf.getBoolean(IGNORE_INVALID_ROW_CONFKEY, true));
    csvUpsertExecutor = buildUpsertExecutor(conf);
    csvLineParser = new CsvLineParser(conf.get(FIELD_DELIMITER_CONFKEY).charAt(0), conf.get(QUOTE_CHAR_CONFKEY).charAt(0),
            conf.get(ESCAPE_CHAR_CONFKEY).charAt(0));

    preUpdateProcessor = loadPreUpsertProcessor(conf);
    if(!conf.get(CsvToKeyValueMapper.INDEX_TABLE_NAME_CONFKEY, "").isEmpty()){
    	tableName = Bytes.toBytes(conf.get(CsvToKeyValueMapper.INDEX_TABLE_NAME_CONFKEY));
    } else {
    	tableName = Bytes.toBytes(conf.get(CsvToKeyValueMapper.TABLE_NAME_CONFKEY, ""));
    }
}
 

public static byte[] randomRowOrQualifier(Random rand) {
  StringBuilder field = new StringBuilder();
  int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
      + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
      - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
  for (int i = 0; i < fieldLen; ++i)
    field.append(randomReadableChar(rand));
  return Bytes.toBytes(field.toString());
}
 

@Override
public void deleteApplicationName(String applicationName) {
    Objects.requireNonNull(applicationName, "applicationName");

    byte[] rowKey = Bytes.toBytes(applicationName);
    Delete delete = new Delete(rowKey);

    TableName applicationIndexTableName = descriptor.getTableName();
    hbaseOperations2.delete(applicationIndexTableName, delete);
}
 

@Test
public void testTruncateWithPreserveAfterSplit() throws Exception {
  String[] families = new String[] { "f1", "f2" };
  byte[][] splitKeys =
    new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") };
  TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(getMasterProcedureExecutor(),
    tableName, splitKeys, families);
  splitAndTruncate(tableName, regions, 1);
}
 

@SuppressWarnings("SelfComparison")
@Test
public void testComparator() {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo older = RegionInfoBuilder.newBuilder(tableName).setRegionId(0).build();
  RegionInfo newer = RegionInfoBuilder.newBuilder(tableName).setRegionId(1).build();
  assertTrue(older.compareTo(newer) < 0);
  assertTrue(newer.compareTo(older) > 0);
  assertEquals(0, older.compareTo(older));
  assertEquals(0, newer.compareTo(newer));

  RegionInfo a = RegionInfoBuilder.newBuilder(TableName.valueOf("a")).build();
  RegionInfo b = RegionInfoBuilder.newBuilder(TableName.valueOf("b")).build();
  assertNotEquals(0, a.compareTo(b));
  TableName t = TableName.valueOf("t");
  byte [] midway = Bytes.toBytes("midway");
  a = RegionInfoBuilder.newBuilder(t).setEndKey(midway).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(midway).build();
  assertTrue(a.compareTo(b) < 0);
  assertTrue(b.compareTo(a) > 0);
  assertEquals(a, a);
  assertEquals(0, a.compareTo(a));
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("a"))
    .setEndKey(Bytes.toBytes("d")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("e"))
    .setEndKey(Bytes.toBytes("g")).build();
  assertTrue(a.compareTo(b) < 0);
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("dddd")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("e"))
    .setEndKey(Bytes.toBytes("g")).build();
  assertTrue(a.compareTo(b) < 0);
  a = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("dddd")).build();
  b = RegionInfoBuilder.newBuilder(t).setStartKey(Bytes.toBytes("aaaa"))
    .setEndKey(Bytes.toBytes("eeee")).build();
  assertTrue(a.compareTo(b) < 0);

}
 

@Override
	protected void map(LongWritable key, Text value, Context context)
			throws IOException, InterruptedException {

		try {
			String lineString = value.toString();
			String[] arr = lineString.split("\t", -1);
			if (arr.length == 2) {
				String midTs = arr[0];
				String cfq = arr[1];
				String[] keys = midTs.split(Const.FIELD_COMMON_SEPARATOR, -1);
				if (keys.length == 2) {
					rowkey = Bytes.toBytes(keys[0]);
					ts = Long.parseLong(keys[1]);
				}

				String[] vals = cfq.split(Const.FIELD_COMMON_SEPARATOR, -1);
				if (vals.length == 3) {
					family = Bytes.toBytes(vals[0]);
					qualifier = Bytes.toBytes(vals[1]);
					cellValue = Bytes.toBytes(vals[2]);

				}

				Put put = new Put(rowkey, ts);
				put.add(family, qualifier, cellValue);

				context.write(new ImmutableBytesWritable(rowkey), put);
//				table.incrementColumnValue(rowkey, family,
//						Const.COLUMN_RK_COUNTER_BYTE, 1L);
				long cur = table.incrementColumnValue(rowkey, family,
						Const.COLUMN_RK_COUNTER_BYTE, 1L);
				System.out.println(cur);
			
			}
		} catch (Exception e) {
			e.printStackTrace();
		}

	}
 

@Test
public void testSameTypeOfMutationWithSameValueButDifferentTimeStamp() throws Exception {
    SetupInfo info = setup(TABLE_NAME,
            INDEX_NAME,
            "ROW_KEY VARCHAR, C1 VARCHAR, C2 VARCHAR",
            "C1",
            "ROW_KEY",
            "");

    Put dataPut = new Put(Bytes.toBytes(ROW_KEY));
    addCellToPutMutation(dataPut,
            info.indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
            Bytes.toBytes("C2"),
            1,
            Bytes.toBytes("v2"));
    addEmptyColumnToDataPutMutation(dataPut, info.pDataTable, 1);
    addCellToPutMutation(dataPut,
            info.indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(),
            Bytes.toBytes("C2"),
            1,
            Bytes.toBytes("v3"));
    addEmptyColumnToDataPutMutation(dataPut, info.pDataTable, 2);

    List<Mutation> actualIndexMutations = IndexRebuildRegionScanner.prepareIndexMutationsForRebuild(info.indexMaintainer,
            dataPut,
            null);

    byte[] idxKeyBytes = generateIndexRowKey(null);

    // idxPut1 and idxPut2 have same value but different time stamp
    Put idxPut1 = new Put(idxKeyBytes);
    addEmptyColumnToIndexPutMutation(idxPut1, info.indexMaintainer, 1);

    Put idxPut2 = new Put(idxKeyBytes);
    addEmptyColumnToIndexPutMutation(idxPut2, info.indexMaintainer, 2);

    assertEqualMutationList(Arrays.asList((Mutation)idxPut1, (Mutation)idxPut2), actualIndexMutations);
}
 

NavigableSet<byte[]> getCols(String... strCols) {
  NavigableSet<byte[]> cols = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  for (String col : strCols) {
    byte[] bytes = Bytes.toBytes(col);
    cols.add(bytes);
  }
  return cols;
}
 

@Test
public void testSplitFailedCompactionAndSplit() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  // Create table then get the single region for our new table.
  byte[] cf = Bytes.toBytes("cf");
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf)).build();
  admin.createTable(htd);

  for (int i = 0; cluster.getRegions(tableName).isEmpty() && i < 100; i++) {
    Thread.sleep(100);
  }
  assertEquals(1, cluster.getRegions(tableName).size());

  HRegion region = cluster.getRegions(tableName).get(0);
  HStore store = region.getStore(cf);
  int regionServerIndex = cluster.getServerWith(region.getRegionInfo().getRegionName());
  HRegionServer regionServer = cluster.getRegionServer(regionServerIndex);

  Table t = TESTING_UTIL.getConnection().getTable(tableName);
  // insert data
  insertData(tableName, admin, t);
  insertData(tableName, admin, t);

  int fileNum = store.getStorefiles().size();
  // 0, Compaction Request
  store.triggerMajorCompaction();
  Optional<CompactionContext> cc = store.requestCompaction();
  assertTrue(cc.isPresent());
  // 1, A timeout split
  // 1.1 close region
  assertEquals(2, region.close(false).get(cf).size());
  // 1.2 rollback and Region initialize again
  region.initialize();

  // 2, Run Compaction cc
  assertFalse(region.compact(cc.get(), store, NoLimitThroughputController.INSTANCE));
  assertTrue(fileNum > store.getStorefiles().size());

  // 3, Split
  requestSplitRegion(regionServer, region, Bytes.toBytes("row3"));
  assertEquals(2, cluster.getRegions(tableName).size());
}
 

public JoniRegexEngine(String regex, int flags) {
  this.regex = regex;
  byte[] b = Bytes.toBytes(regex);
  this.pattern = new Regex(b, 0, b.length, patternToJoniFlags(flags), encoding, Syntax.Java);
}
 

/**
 *
 * @throws Exception
 */
@Test
public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
  final TableName tableName =
      TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
  byte[] family1 = Bytes.toBytes("cf1");
  byte[] family2 = Bytes.toBytes("cf2");
  byte[] qualifier = Bytes.toBytes("q");
  byte[] value = Bytes.toBytes("testV");
  byte[][] familys = { family1, family2 };
  TEST_UTIL.createTable(tableName, familys);
  Table htable = TEST_UTIL.getConnection().getTable(tableName);
  Put put = new Put(Bytes.toBytes("r1"));
  put.addColumn(family1, qualifier, value);
  htable.put(put);
  ResultScanner resultScanner = htable.getScanner(new Scan());
  int count = 0;
  while (resultScanner.next() != null) {
    count++;
  }
  resultScanner.close();
  assertEquals(1, count);

  MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
  List<HRegion> regions = hbaseCluster.getRegions(tableName);
  assertEquals(1, regions.size());

  // move region to another regionserver
  Region destRegion = regions.get(0);
  int originServerNum = hbaseCluster.getServerWith(destRegion.getRegionInfo().getRegionName());
  assertTrue("Please start more than 1 regionserver",
      hbaseCluster.getRegionServerThreads().size() > 1);
  int destServerNum = 0;
  while (destServerNum == originServerNum) {
    destServerNum++;
  }
  HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
  HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
  // move region to destination regionserver
  TEST_UTIL.moveRegionAndWait(destRegion.getRegionInfo(), destServer.getServerName());

  // delete the row
  Delete del = new Delete(Bytes.toBytes("r1"));
  htable.delete(del);
  resultScanner = htable.getScanner(new Scan());
  count = 0;
  while (resultScanner.next() != null) {
    count++;
  }
  resultScanner.close();
  assertEquals(0, count);

  // flush region and make major compaction
  HRegion region =
      (HRegion) destServer.getOnlineRegion(destRegion.getRegionInfo().getRegionName());
  region.flush(true);
  // wait to complete major compaction
  for (HStore store : region.getStores()) {
    store.triggerMajorCompaction();
  }
  region.compact(true);

  // move region to origin regionserver
  TEST_UTIL.moveRegionAndWait(destRegion.getRegionInfo(), originServer.getServerName());
  // abort the origin regionserver
  originServer.abort("testing");

  // see what we get
  Result result = htable.get(new Get(Bytes.toBytes("r1")));
  if (result != null) {
    assertTrue("Row is deleted, but we get" + result.toString(),
        (result == null) || result.isEmpty());
  }
  resultScanner.close();
}
 

@Test
public void testNoInsertsWithIncrement() throws Exception {
  Increment i = new Increment(Bytes.toBytes("to_reject"));
  i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
  helper.writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
}
 

@Test
public void testLabelsWithAppend() throws Throwable {
  PrivilegedExceptionAction<VisibilityLabelsResponse> action =
      new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
    @Override
    public VisibilityLabelsResponse run() throws Exception {
      try (Connection conn = ConnectionFactory.createConnection(conf)) {
        return VisibilityClient.setAuths(conn, new String[] { TOPSECRET },
            USER.getShortName());
      } catch (Throwable e) {
      }
      return null;
    }
  };
  SUPERUSER.runAs(action);
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = TEST_UTIL.createTable(tableName, fam)) {
    final byte[] row1 = Bytes.toBytes("row1");
    final byte[] val = Bytes.toBytes("a");
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Put put = new Put(row1);
          put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, val);
          put.setCellVisibility(new CellVisibility(TOPSECRET));
          table.put(put);
        }
        return null;
      }
    };
    USER.runAs(actiona);
    actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Append append = new Append(row1);
          append.addColumn(fam, qual, Bytes.toBytes("b"));
          table.append(append);
        }
        return null;
      }
    };
    USER.runAs(actiona);
    actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Append append = new Append(row1);
          append.addColumn(fam, qual, Bytes.toBytes("c"));
          append.setCellVisibility(new CellVisibility(PUBLIC));
          table.append(append);
          Assert.fail("Testcase should fail with AccesDeniedException");
        } catch (Throwable t) {
          assertTrue(t.getMessage().contains("AccessDeniedException"));
        }
        return null;
      }
    };
    USER.runAs(actiona);
  }
}
 

/**
 * Builds the 'time' portion of the row key
 * @param period the period
 */
private static byte[] timeKey(long period) {
  return Bytes.toBytes(period);
}