Java Code Examples for org.apache.hadoop.hbase.util.Bytes#putBytes()

private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
 

/**
 * Send the response for connection header
 */
private void responseConnectionHeader(RPCProtos.ConnectionHeaderResponse.Builder chrBuilder)
    throws FatalConnectionException {
  // Response the connection header if Crypto AES is enabled
  if (!chrBuilder.hasCryptoCipherMeta()) return;
  try {
    byte[] connectionHeaderResBytes = chrBuilder.build().toByteArray();
    // encrypt the Crypto AES cipher meta data with sasl server, and send to client
    byte[] unwrapped = new byte[connectionHeaderResBytes.length + 4];
    Bytes.putBytes(unwrapped, 0, Bytes.toBytes(connectionHeaderResBytes.length), 0, 4);
    Bytes.putBytes(unwrapped, 4, connectionHeaderResBytes, 0, connectionHeaderResBytes.length);
    byte[] wrapped = saslServer.wrap(unwrapped, 0, unwrapped.length);
    BufferChain bc;
    try (ByteBufferOutputStream response = new ByteBufferOutputStream(wrapped.length + 4);
        DataOutputStream out = new DataOutputStream(response)) {
      out.writeInt(wrapped.length);
      out.write(wrapped);
      bc = new BufferChain(response.getByteBuffer());
    }
    doRespond(() -> bc);
  } catch (IOException ex) {
    throw new UnsupportedCryptoException(ex.getMessage(), ex);
  }
}
 

public static ProjectedValueTuple mergeProjectedValue(ProjectedValueTuple dest, KeyValueSchema destSchema, ValueBitSet destBitSet,
		Tuple src, KeyValueSchema srcSchema, ValueBitSet srcBitSet, int offset) throws IOException {
	ImmutableBytesWritable destValue = new ImmutableBytesWritable(dest.getProjectedValue());
	destBitSet.clear();
	destBitSet.or(destValue);
	int origDestBitSetLen = dest.getBitSetLength();
	ImmutableBytesWritable srcValue = new ImmutableBytesWritable();
	decodeProjectedValue(src, srcValue);
	srcBitSet.clear();
	srcBitSet.or(srcValue);
	int origSrcBitSetLen = srcBitSet.getEstimatedLength();
	for (int i = 0; i < srcBitSet.getMaxSetBit(); i++) {
		if (srcBitSet.get(i)) {
			destBitSet.set(offset + i);
		}
	}
	int destBitSetLen = destBitSet.getEstimatedLength();
	byte[] merged = new byte[destValue.getLength() - origDestBitSetLen + srcValue.getLength() - origSrcBitSetLen + destBitSetLen];
	int o = Bytes.putBytes(merged, 0, destValue.get(), destValue.getOffset(), destValue.getLength() - origDestBitSetLen);
	o = Bytes.putBytes(merged, o, srcValue.get(), srcValue.getOffset(), srcValue.getLength() - origSrcBitSetLen);
	destBitSet.toBytes(merged, o);
	ImmutableBytesWritable keyPtr = dest.getKeyPtr();
    return new ProjectedValueTuple(keyPtr.get(), keyPtr.getOffset(), keyPtr.getLength(), dest.getTimestamp(), merged, destBitSetLen);
}
 

public static byte[] generateOutputTableRowKey(long ts, byte[] indexTableName, byte[] dataRowKey ) {
    byte[] keyPrefix = Bytes.toBytes(Long.toString(ts));
    byte[] rowKey;
    int targetOffset = 0;
    // The row key for the output table : timestamp | index table name | data row key
    rowKey = new byte[keyPrefix.length + ROW_KEY_SEPARATOR_BYTE.length + indexTableName.length +
        ROW_KEY_SEPARATOR_BYTE.length + dataRowKey.length];
    Bytes.putBytes(rowKey, targetOffset, keyPrefix, 0, keyPrefix.length);
    targetOffset += keyPrefix.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, indexTableName, 0, indexTableName.length);
    targetOffset += indexTableName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, dataRowKey, 0, dataRowKey.length);
    return rowKey;
}
 

/**
* Extracts a simple filter expression from the filter string given by the user
* <p>
* A simpleFilterExpression is of the form: FilterName('arg', 'arg', 'arg')
* The user given filter string can have many simpleFilterExpressions combined
* using operators.
* <p>
* This function extracts a simpleFilterExpression from the
* larger filterString given the start offset of the simpler expression
* <p>
* @param filterStringAsByteArray filter string given by the user
* @param filterExpressionStartOffset start index of the simple filter expression
* @return byte array containing the simple filter expression
*/
 public byte [] extractFilterSimpleExpression (byte [] filterStringAsByteArray,
                                               int filterExpressionStartOffset)
   throws CharacterCodingException {
   int quoteCount = 0;
   for (int i=filterExpressionStartOffset; i<filterStringAsByteArray.length; i++) {
     if (filterStringAsByteArray[i] == ParseConstants.SINGLE_QUOTE) {
       if (isQuoteUnescaped(filterStringAsByteArray, i)) {
         quoteCount ++;
       } else {
         // To skip the next quote that has been escaped
         i++;
       }
     }
     if (filterStringAsByteArray[i] == ParseConstants.RPAREN && (quoteCount %2 ) == 0) {
       byte [] filterSimpleExpression = new byte [i - filterExpressionStartOffset + 1];
       Bytes.putBytes(filterSimpleExpression, 0, filterStringAsByteArray,
                      filterExpressionStartOffset, i-filterExpressionStartOffset + 1);
       return filterSimpleExpression;
     }
   }
   throw new IllegalArgumentException("Incorrect Filter String");
 }
 

/**
* Returns the filter name given a simple filter expression
* <p>
* @param filterStringAsByteArray a simple filter expression
* @return name of filter in the simple filter expression
*/
 public static byte [] getFilterName (byte [] filterStringAsByteArray) {
   int filterNameStartIndex = 0;
   int filterNameEndIndex = 0;

   for (int i=filterNameStartIndex; i<filterStringAsByteArray.length; i++) {
     if (filterStringAsByteArray[i] == ParseConstants.LPAREN ||
         filterStringAsByteArray[i] == ParseConstants.WHITESPACE) {
       filterNameEndIndex = i;
       break;
     }
   }

   if (filterNameEndIndex == 0) {
     throw new IllegalArgumentException("Incorrect Filter Name");
   }

   byte [] filterName = new byte[filterNameEndIndex - filterNameStartIndex];
   Bytes.putBytes(filterName, 0, filterStringAsByteArray, 0,
                  filterNameEndIndex - filterNameStartIndex);
   return filterName;
 }
 

public static byte[] concatTags(byte[] tags, Cell cell) {
  int cellTagsLen = cell.getTagsLength();
  if (cellTagsLen == 0) {
    // If no Tags, return early.
    return tags;
  }
  byte[] b = new byte[tags.length + cellTagsLen];
  int pos = Bytes.putBytes(b, 0, tags, 0, tags.length);
  if (cell instanceof ByteBufferExtendedCell) {
    ByteBufferUtils.copyFromBufferToArray(b, ((ByteBufferExtendedCell) cell).getTagsByteBuffer(),
        ((ByteBufferExtendedCell) cell).getTagsPosition(), pos, cellTagsLen);
  } else {
    Bytes.putBytes(b, pos, cell.getTagsArray(), cell.getTagsOffset(), cellTagsLen);
  }
  return b;
}
 

private short put(byte[] array, int offset, int length, boolean copy) {
  if (copy) {
    // We copy the bytes we want, otherwise we might be holding references to
    // massive arrays in our dictionary (or those arrays might change)
    byte[] stored = new byte[length];
    Bytes.putBytes(stored, 0, array, offset, length);
    return putInternal(stored);
  } else {
    return putInternal(array);
  }
}
 

/**
 * Extract byte array for column specified by colIdx.
 * 
 * @param lineBytes
 * @param parsed
 * @param colIdx
 * @return
 */
protected byte[] getInputColBytes(byte[] lineBytes,
		ImportTsv.TsvParser.ParsedLine parsed, int colIdx) {
	if (colIdx >= columnTypes.length)
		return null;
	int colOffset = parsed.getColumnOffset(colIdx);
	int colLen = parsed.getColumnLength(colIdx);
	byte[] colBytes = new byte[colLen];
	Bytes.putBytes(colBytes, 0, lineBytes, colOffset, colLen);
	return colBytes;
}
 

/**
 * Format for a tag :
 * {@code <length of tag - 2 bytes><type code - 1 byte><tag>} tag length is serialized
 * using 2 bytes only but as this will be unsigned, we can have max tag length of
 * (Short.MAX_SIZE * 2) +1. It includes 1 byte type length and actual tag bytes length.
 */
public ArrayBackedTag(byte tagType, byte[] tag) {
  int tagLength = tag.length + TYPE_LENGTH_SIZE;
  if (tagLength > MAX_TAG_LENGTH) {
    throw new IllegalArgumentException(
        "Invalid tag data being passed. Its length can not exceed " + MAX_TAG_LENGTH);
  }
  length = TAG_LENGTH_SIZE + tagLength;
  bytes = new byte[length];
  int pos = Bytes.putAsShort(bytes, 0, tagLength);
  pos = Bytes.putByte(bytes, pos, tagType);
  Bytes.putBytes(bytes, pos, tag, 0, tag.length);
  this.type = tagType;
}
 

/**
 * Copies the tag's value bytes to the given byte array
 * @param tag The Tag
 * @param out The byte array where to copy the Tag value.
 * @param offset The offset within 'out' array where to copy the Tag value.
 */
public static void copyValueTo(Tag tag, byte[] out, int offset) {
  if (tag.hasArray()) {
    Bytes.putBytes(out, offset, tag.getValueArray(), tag.getValueOffset(), tag.getValueLength());
  } else {
    ByteBufferUtils.copyFromBufferToArray(out, tag.getValueByteBuffer(), tag.getValueOffset(),
      offset, tag.getValueLength());
  }
}
 

private static byte[] generateResultTableRowKey(long ts, byte[] indexTableName,  byte [] regionName,
                                                byte[] startRow, byte[] stopRow) {
    byte[] keyPrefix = Bytes.toBytes(Long.toString(ts));
    int targetOffset = 0;
    // The row key for the result table : timestamp | index table name | datable table region name |
    //                                    scan start row | scan stop row
    byte[] rowKey = new byte[keyPrefix.length + ROW_KEY_SEPARATOR_BYTE.length + indexTableName.length +
        ROW_KEY_SEPARATOR_BYTE.length + regionName.length + ROW_KEY_SEPARATOR_BYTE.length +
        startRow.length + ROW_KEY_SEPARATOR_BYTE.length + stopRow.length];
    Bytes.putBytes(rowKey, targetOffset, keyPrefix, 0, keyPrefix.length);
    targetOffset += keyPrefix.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, indexTableName, 0, indexTableName.length);
    targetOffset += indexTableName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, regionName, 0, regionName.length);
    targetOffset += regionName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, startRow, 0, startRow.length);
    targetOffset += startRow.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, stopRow, 0, stopRow.length);
    return rowKey;
}
 

/**
 * Write KeyValue format into a byte array.
 * @param row row key
 * @param roffset row offset
 * @param rlength row length
 * @param family family name
 * @param foffset family offset
 * @param flength family length
 * @param qualifier column qualifier
 * @param qoffset qualifier offset
 * @param qlength qualifier length
 * @param timestamp version timestamp
 * @param type key type
 * @param value column value
 * @param voffset value offset
 * @param vlength value length
 * @return The newly created byte array.
 */
private static byte [] createByteArray(final byte [] row, final int roffset,
    final int rlength, final byte [] family, final int foffset, int flength,
    final byte [] qualifier, final int qoffset, int qlength,
    final long timestamp, final Type type,
    final byte [] value, final int voffset,
    int vlength, byte[] tags, int tagsOffset, int tagsLength) {

  checkParameters(row, rlength, family, flength, qlength, vlength);
  RawCell.checkForTagsLength(tagsLength);
  // Allocate right-sized byte array.
  int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
  byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
    tagsLength)];
  // Write key, value and key row length.
  int pos = 0;
  pos = Bytes.putInt(bytes, pos, keyLength);
  pos = Bytes.putInt(bytes, pos, vlength);
  pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
  pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
  pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
  if(flength != 0) {
    pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
  }
  if(qlength != 0) {
    pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
  }
  pos = Bytes.putLong(bytes, pos, timestamp);
  pos = Bytes.putByte(bytes, pos, type.getCode());
  if (value != null && value.length > 0) {
    pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
  }
  // Add the tags after the value part
  if (tagsLength > 0) {
    pos = Bytes.putAsShort(bytes, pos, tagsLength);
    pos = Bytes.putBytes(bytes, pos, tags, tagsOffset, tagsLength);
  }
  return bytes;
}
 

@Test
public void testBasic() {
  Random rand = new Random();
  byte[] testBytes = new byte[10];
  rand.nextBytes(testBytes);

  // Verify that our randomly generated array doesn't exist in the dictionary
  assertEquals(-1, testee.findEntry(testBytes, 0, testBytes.length));

  // now since we looked up an entry, we should have added it to the
  // dictionary, so it isn't empty

  assertFalse(isDictionaryEmpty(testee));

  // Check if we can find it using findEntry
  short t = testee.findEntry(testBytes, 0, testBytes.length);

  // Making sure we do find what we're looking for
  assertTrue(t != -1);

  byte[] testBytesCopy = new byte[20];

  Bytes.putBytes(testBytesCopy, 10, testBytes, 0, testBytes.length);

  // copy byte arrays, make sure that we check that equal byte arrays are
  // equal without just checking the reference
  assertEquals(testee.findEntry(testBytesCopy, 10, testBytes.length), t);

  // make sure the entry retrieved is the same as the one put in
  assertTrue(Arrays.equals(testBytes, testee.getEntry(t)));

  testee.clear();

  // making sure clear clears the dictionary
  assertTrue(isDictionaryEmpty(testee));
}
 

public static byte[] appendMetaData(byte[] id, byte[] data) {
  if (data == null || data.length == 0) {
    return data;
  }
  byte[] salt = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
  int idLength = id.length + salt.length;
  byte[] newData = new byte[MAGIC_SIZE + ID_LENGTH_SIZE + idLength + data.length];
  int pos = 0;
  pos = Bytes.putByte(newData, pos, MAGIC);
  pos = Bytes.putInt(newData, pos, idLength);
  pos = Bytes.putBytes(newData, pos, id, 0, id.length);
  pos = Bytes.putBytes(newData, pos, salt, 0, salt.length);
  pos = Bytes.putBytes(newData, pos, data, 0, data.length);
  return newData;
}
 

@Test
public void testSerialization() {
  TaskKeyConverter conv = new TaskKeyConverter();

  TaskKey key1 = new TaskKey(
      new JobKey("test@local", "testuser", "app", 1234L, "job_20120101000000_1111"), "m_001");
  assertEquals("test@local", key1.getCluster());
  assertEquals("testuser", key1.getUserName());
  assertEquals("app", key1.getAppId());
  assertEquals(1234L, key1.getRunId());
  assertEquals("job_20120101000000_1111", key1.getJobId().getJobIdString());
  assertEquals("m_001", key1.getTaskId());

  byte[] key1Bytes = conv.toBytes(key1);
  TaskKey key2 = conv.fromBytes(key1Bytes);
  assertKey(key1, key2);

  TaskKey key3 = conv.fromBytes( conv.toBytes(key2) );
  assertKey(key1, key3);

  // test with a run ID containing the separator
  long now = System.currentTimeMillis();
  byte[] encoded = Bytes.toBytes(Long.MAX_VALUE - now);
  // replace last byte with separator and reconvert to long
  Bytes.putBytes(encoded, encoded.length-Constants.SEP_BYTES.length,
      Constants.SEP_BYTES, 0, Constants.SEP_BYTES.length);
  long badId = Long.MAX_VALUE - Bytes.toLong(encoded);
  LOG.info("Bad run ID is " + badId);

  TaskKey badKey1 = new TaskKey(
      new JobKey(key1.getQualifiedJobId(), key1.getUserName(), key1.getAppId(), badId),
      key1.getTaskId());
  byte[] badKeyBytes = conv.toBytes(badKey1);
  TaskKey badKey2 = conv.fromBytes(badKeyBytes);
  assertKey(badKey1, badKey2);
}
 

@Override
public void setTimestamp(long ts) {
  Bytes.putBytes(this.bytes, this.getTimestampOffset(), Bytes.toBytes(ts), 0, Bytes.SIZEOF_LONG);
}
 

@Override
public void setTimestamp(byte[] ts) {
  Bytes.putBytes(this.bytes, this.getTimestampOffset(), ts, 0, Bytes.SIZEOF_LONG);
}
 

@SuppressWarnings("deprecation")
@Test
public void testFilterList() throws Exception {
  String cf = "f";
  Table ht =
      TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i1 = 0; i1 < 5; i1++) {
    for (int i2 = 0; i2 < 5; i2++) {
      byte[] rk = new byte[10];

      ByteBuffer buf = ByteBuffer.wrap(rk);
      buf.clear();
      buf.putShort((short) 2);
      buf.putInt(i1);
      buf.putInt(i2);

      // Each row contains 5 columns
      for (int c = 0; c < 5; c++) {
        byte[] cq = new byte[4];
        Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

        Put p = new Put(rk);
        p.setDurability(Durability.SKIP_WAL);
        p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
        ht.put(p);
        LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
            + Bytes.toStringBinary(cq));
      }
    }
  }

  TEST_UTIL.flush();

  // test passes if we get back 5 KV's (1 row)
  runTest(ht, 5);

}
 

@Test
public void testEndToEnd() throws Exception {
  String cf = "f";

  Table ht =
      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i0 = 0; i0 < firstPartCardinality; i0++) {

    for (int i1 = 0; i1 < secondPartCardinality; i1++) {

      for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
        byte[] rk = new byte[10];

        ByteBuffer buf = ByteBuffer.wrap(rk);
        buf.clear();
        buf.putShort((short) i0);
        buf.putInt(i1);
        buf.putInt(i2);
        for (int c = 0; c < colQualifiersTotal; c++) {
          byte[] cq = new byte[4];
          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

          Put p = new Put(rk);
          p.setDurability(Durability.SKIP_WAL);
          p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
          ht.put(p);
        }
      }
    }
  }

  TEST_UTIL.flush();

  // test passes
  runTest1(ht);
  runTest2(ht);

}