Java Code Examples for org.apache.hadoop.io.BytesWritable#set()

/**
*
* Read a next block. 
*
* @param key is a 32 byte array (parentHash)
* @param value is a deserialized Java object of class EthereumBlock
*
* @return true if next block is available, false if not
*/
@Override
public boolean next(BytesWritable key, EthereumBlock value) throws IOException {
	// read all the blocks, if necessary a block overlapping a split
	while(getFilePosition()<=getEnd()) { // did we already went beyond the split (remote) or do we have no further data left?
		EthereumBlock dataBlock=null;
		try {
			dataBlock=getEbr().readBlock();
		} catch (EthereumBlockReadException e) {
			LOG.error(e);
			throw new RuntimeException(e.toString());
		}
		if (dataBlock==null) { 
			return false;
		}
		byte[] newKey=dataBlock.getEthereumBlockHeader().getParentHash();
		key.set(newKey,0,newKey.length);
		value.set(dataBlock);
		return true;
	}
	return false;
}
 

/**
 * Main logic called by hive if sketchSize is also passed in. Computes the
 * hash in first sketch excluding the hash in second sketch of two sketches of
 * same or different column.
 *
 * @param firstSketchBytes
 *          first sketch to be included.
 * @param secondSketchBytes
 *          second sketch to be excluded.
 * @param hashSeed
 *          Only required if input sketches were constructed using an update seed that was not the default.
 * @return resulting sketch of exclusion.
 */
public BytesWritable evaluate(final BytesWritable firstSketchBytes,
    final BytesWritable secondSketchBytes, final long hashSeed) {

  Sketch firstSketch = null;
  if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) {
    firstSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(firstSketchBytes), hashSeed);
  }

  Sketch secondSketch = null;
  if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) {
    secondSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(secondSketchBytes), hashSeed);
  }

  final AnotB anotb = SetOperation.builder().setSeed(hashSeed).buildANotB();
  anotb.update(firstSketch, secondSketch);
  final byte[] excludeSketchBytes = anotb.getResult().toByteArray();
  final BytesWritable result = new BytesWritable();
  result.set(excludeSketchBytes, 0, excludeSketchBytes.length);
  return result;
}
 

@Test
public void testReuse()
{
    BytesWritable value = new BytesWritable();

    byte[] first = "hello world".getBytes(UTF_8);
    value.set(first, 0, first.length);

    byte[] second = "bye".getBytes(UTF_8);
    value.set(second, 0, second.length);

    Type type = new TypeToken<Map<BytesWritable, Long>>() {}.getType();
    ObjectInspector inspector = getInspector(type);

    Block actual = getBlockObject(mapType(createUnboundedVarcharType(), BIGINT), ImmutableMap.of(value, 0L), inspector);
    Block expected = mapBlockOf(createUnboundedVarcharType(), BIGINT, "bye", 0L);

    assertBlockEquals(actual, expected);
}
 

public void next(BytesWritable key, BytesWritable value, boolean dupKey) {
  if (dupKey) {
    key.set(lastKey);
  }
  else {
    fillKey(key);
  }
  fillValue(value);
}
 

private void timeWrite(Path path, KVAppendable appendable, int baseKlen,
    int baseVlen, long fileSize) throws IOException {
  int maxKlen = baseKlen * 2;
  int maxVlen = baseVlen * 2;
  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  byte[] keyBuffer = new byte[maxKlen];
  byte[] valueBuffer = new byte[maxVlen];
  Random rng = new Random(options.seed);
  long totalBytes = 0;
  printlnWithTimestamp("Start writing: " + path.getName() + "...");
  startTime();

  for (long i = 0; true; ++i) {
    if (i % 1000 == 0) { // test the size for every 1000 rows.
      if (fs.getFileStatus(path).getLen() >= fileSize) {
        break;
      }
    }
    int klen = rng.nextInt(baseKlen) + baseKlen;
    int vlen = rng.nextInt(baseVlen) + baseVlen;
    fillBuffer(rng, key, keyBuffer, klen);
    fillBuffer(rng, value, valueBuffer, vlen);
    key.set(keyBuffer, 0, klen);
    value.set(valueBuffer, 0, vlen);
    appendable.append(key, value);
    totalBytes += klen;
    totalBytes += vlen;
  }
  stopTime();
  appendable.close();
  reportStats(path, totalBytes);
}
 

private void fillBuffer(Random rng, BytesWritable bw, byte[] tmp, int len) {
  int n = 0;
  while (n < len) {
    byte[] word = dictionary[rng.nextInt(dictionary.length)];
    int l = Math.min(word.length, len - n);
    System.arraycopy(word, 0, tmp, n, l);
    n += l;
  }
  bw.set(tmp, 0, len);
}
 

@Override
public BytesWritable convert( ValueMetaInterface meta, Object obj ) throws TypeConversionException {
  try {
    BytesWritable result = new BytesWritable();
    byte[] binary = meta.getBinary( obj );
    result.set( binary, 0, binary.length );
    return result;
  } catch ( Exception ex ) {
    throw new TypeConversionException( BaseMessages
      .getString( TypeConverterFactory.class, "ErrorConverting", BytesWritable.class.getSimpleName(), obj ), ex );
  }
}
 

/**
*
* Read a next block. 
*
* @param key is a 68 byte array (hashMerkleRoot, prevHashBlock, transActionCounter)
* @param value is a deserialized Java object of class BitcoinBlock
*
* @return true if next block is available, false if not
*/
@Override
public boolean next(BytesWritable key, BitcoinTransaction value) throws IOException {
	// read all the blocks, if necessary a block overlapping a split
	while(getFilePosition()<=getEnd()) { // did we already went beyond the split (remote) or do we have no further data left?
		if ((currentBitcoinBlock==null) || (currentBitcoinBlock.getTransactions().size()==currentTransactionCounterInBlock)){
			try {
				currentBitcoinBlock=getBbr().readBlock();
				currentTransactionCounterInBlock=0;
			} catch (BitcoinBlockReadException e) {
				// log
				LOG.error(e);
			}
		}

		if (currentBitcoinBlock==null) {
			return false;
		}
		BitcoinTransaction currentTransaction=currentBitcoinBlock.getTransactions().get(currentTransactionCounterInBlock);
		// the unique identifier that is linked in other transaction is usually its hash
		byte[] newKey = BitcoinUtil.getTransactionHash(currentTransaction);
		key.set(newKey,0,newKey.length);
		value.set(currentTransaction);
		currentTransactionCounterInBlock++;
		return true;
	}
	return false;
}
 

@Override
public boolean next(BytesWritable key, MapWritable value) throws IOException {

  if (!nextKeyValue()) {
    return false;
  }

  key.set(getCurrentKey());

  value.clear();
  value.putAll(getCurrentValue());

  return true;
}
 

private void fillBuffer(Random rng, BytesWritable bw, byte[] tmp, int len) {
  int n = 0;
  while (n < len) {
    byte[] word = dictionary[rng.nextInt(dictionary.length)];
    int l = Math.min(word.length, len - n);
    System.arraycopy(word, 0, tmp, n, l);
    n += l;
  }
  bw.set(tmp, 0, len);
}
 

private void timeWrite(Path path, KVAppendable appendable, int baseKlen,
    int baseVlen, long fileSize) throws IOException {
  int maxKlen = baseKlen * 2;
  int maxVlen = baseVlen * 2;
  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  byte[] keyBuffer = new byte[maxKlen];
  byte[] valueBuffer = new byte[maxVlen];
  Random rng = new Random(options.seed);
  long totalBytes = 0;
  printlnWithTimestamp("Start writing: " + path.getName() + "...");
  startTime();

  for (long i = 0; true; ++i) {
    if (i % 1000 == 0) { // test the size for every 1000 rows.
      if (fs.getFileStatus(path).getLen() >= fileSize) {
        break;
      }
    }
    int klen = rng.nextInt(baseKlen) + baseKlen;
    int vlen = rng.nextInt(baseVlen) + baseVlen;
    fillBuffer(rng, key, keyBuffer, klen);
    fillBuffer(rng, value, valueBuffer, vlen);
    key.set(keyBuffer, 0, klen);
    value.set(valueBuffer, 0, vlen);
    appendable.append(key, value);
    totalBytes += klen;
    totalBytes += vlen;
  }
  stopTime();
  appendable.close();
  reportStats(path, totalBytes);
}
 

public void next(BytesWritable key, BytesWritable value, boolean dupKey) {
  if (dupKey) {
    key.set(lastKey);
  }
  else {
    fillKey(key);
  }
  fillValue(value);
}
 

/**
*
* Read a next block. 
*
* @param key is a 64 byte array (hashMerkleRoot and prevHashBlock)
* @param value is a deserialized Java object of class BitcoinBlock
*
* @return true if next block is available, false if not
*/
@Override
public boolean next(BytesWritable key, BitcoinBlock value) throws IOException {
	// read all the blocks, if necessary a block overlapping a split
	while(getFilePosition()<=getEnd()) { // did we already went beyond the split (remote) or do we have no further data left?
		BitcoinBlock dataBlock=null;
		try {
			dataBlock=getBbr().readBlock();
			
		} catch (BitcoinBlockReadException e) {
			// log
			LOG.error(e);
		}	
		if (dataBlock==null) { 
			return false;
		}
		byte[] hashMerkleRoot=dataBlock.getHashMerkleRoot();
		byte[] hashPrevBlock=dataBlock.getHashPrevBlock();
		byte[] newKey=new byte[hashMerkleRoot.length+hashPrevBlock.length];
		for (int i=0;i<hashMerkleRoot.length;i++) {
			newKey[i]=hashMerkleRoot[i];
		}
		for (int j=0;j<hashPrevBlock.length;j++) {
			newKey[j+hashMerkleRoot.length]=hashPrevBlock[j];
		}
		key.set(newKey,0,newKey.length);
		value.set(dataBlock);
		return true;
	}
	return false;
}
 

@Override
public synchronized boolean next(LongWritable key, BytesWritable value)
    throws IOException {
  boolean dataRead = reader.nextKeyValue();
  if (dataRead) {
    LongWritable newKey = reader.getCurrentKey();
    BytesWritable newValue = reader.getCurrentValue();
    key.set(newKey.get());
    value.set(newValue);
  }
  return dataRead;
}
 

private Object convertBinaryTypes(Object val, String javaColType) {
  byte[] bb = (byte[]) val;
  if (javaColType.equals(BYTESWRITABLE)) {
    BytesWritable bw = new BytesWritable();
    bw.set(bb, 0, bb.length);
    return bw;
  }
  return null;
}
 

public void next(BytesWritable key, BytesWritable value, boolean dupKey) {
  if (dupKey) {
    key.set(lastKey);
  }
  else {
    fillKey(key);
  }
  fillValue(value);
}
 

private void fillBuffer(Random rng, BytesWritable bw, byte[] tmp, int len) {
  int n = 0;
  while (n < len) {
    byte[] word = dictionary[rng.nextInt(dictionary.length)];
    int l = Math.min(word.length, len - n);
    System.arraycopy(word, 0, tmp, n, l);
    n += l;
  }
  bw.set(tmp, 0, len);
}
 

public void dump(String path) throws IOException {
    
    long nano = System.nanoTime();
    int gts = 0;
    long bytes = 0L;
    
    Configuration conf = new Configuration();
        
    conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
    conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

    BytesWritable key = new BytesWritable();
    BytesWritable value = new BytesWritable();
    
    CompressionCodec Codec = new DefaultCodec();
    SequenceFile.Writer writer = null;
    SequenceFile.Writer.Option optPath = SequenceFile.Writer.file(new Path(path));
    SequenceFile.Writer.Option optKey = SequenceFile.Writer.keyClass(key.getClass());
    SequenceFile.Writer.Option optVal = SequenceFile.Writer.valueClass(value.getClass());
    SequenceFile.Writer.Option optCom = SequenceFile.Writer.compression(CompressionType.RECORD,  Codec);
    
    writer = SequenceFile.createWriter(conf, optPath, optKey, optVal, optCom);

    TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
    
    try {
      for (Entry<BigInteger,GTSEncoder> entry: this.series.entrySet()) {
        gts++;
        Metadata metadata = this.directoryClient.getMetadataById(entry.getKey());

        GTSWrapper wrapper = new GTSWrapper();
        wrapper.setMetadata(metadata);        
        
        GTSEncoder encoder = entry.getValue();

        wrapper.setBase(encoder.getBaseTimestamp());
        wrapper.setCount(encoder.getCount());
        
        byte[] data = serializer.serialize(wrapper);
        key.set(data, 0, data.length);
        
        data = encoder.getBytes();
        value.set(data, 0, data.length);

        bytes += key.getLength() + value.getLength();
        
        writer.append(key, value);
      }
/*      
      for (Entry<BigInteger,Metadata> entry: this.metadatas.entrySet()) {
        gts++;
        byte[] data = serializer.serialize(entry.getValue());
        key.set(data, 0, data.length);
        
        GTSEncoder encoder = this.series.get(entry.getKey());
        data = encoder.getBytes();
        value.set(data, 0, data.length);

        bytes += key.getLength() + value.getLength();
        
        writer.append(key, value);
      }
*/      
    } catch (IOException ioe) {
      ioe.printStackTrace();
      throw ioe;
    } catch (Exception e) {
      e.printStackTrace();
      throw new IOException(e);
    }
    
    writer.close();

    nano = System.nanoTime() - nano;
    
    System.out.println("Dumped " + gts + " GTS (" + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
  }
 

public void dump(String path) throws IOException {
  
  if (null == this.directoryClient) {
    return;
  }
  
  long nano = System.nanoTime();
  int gts = 0;
  long chunks = 0;
  long bytes = 0L;
  long datapoints = 0;
  
  Configuration conf = new Configuration();
      
  conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
  conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());

  BytesWritable key = new BytesWritable();
  BytesWritable value = new BytesWritable();
  
  CompressionCodec Codec = new DefaultCodec();
  SequenceFile.Writer writer = null;
  SequenceFile.Writer.Option optPath = SequenceFile.Writer.file(new Path(path));
  SequenceFile.Writer.Option optKey = SequenceFile.Writer.keyClass(key.getClass());
  SequenceFile.Writer.Option optVal = SequenceFile.Writer.valueClass(value.getClass());
  SequenceFile.Writer.Option optCom = SequenceFile.Writer.compression(CompressionType.RECORD,  Codec);
  
  writer = SequenceFile.createWriter(conf, optPath, optKey, optVal, optCom);

  TSerializer serializer = new TSerializer(new TCompactProtocol.Factory());
  
  System.out.println("Dumping memory to '" + path + "'.");
  
  try {
    for (Entry<BigInteger,InMemoryChunkSet> entry: this.series.entrySet()) {
      gts++;        
      Metadata metadata = this.directoryClient.getMetadataById(entry.getKey());

      List<GTSDecoder> decoders = entry.getValue().getDecoders();

      //GTSEncoder encoder = entry.getValue().fetchEncoder(now, this.chunkcount * this.chunkspan);

      for (GTSDecoder decoder: decoders) {
        chunks++;
        GTSWrapper wrapper = new GTSWrapper();
        wrapper.setMetadata(metadata);        

        wrapper.setBase(decoder.getBaseTimestamp());
        wrapper.setCount(decoder.getCount());
        datapoints += wrapper.getCount();
        
        byte[] data = serializer.serialize(wrapper);
        key.set(data, 0, data.length);
        
        ByteBuffer bb = decoder.getBuffer();
        
        ByteBuffer rwbb = ByteBuffer.allocate(bb.remaining());
        rwbb.put(bb);
        rwbb.rewind();
        value.set(rwbb.array(), rwbb.arrayOffset(), rwbb.remaining());

        bytes += key.getLength() + value.getLength();
        
        writer.append(key, value);
      }        
    }
  } catch (IOException ioe) {
    ioe.printStackTrace();
    throw ioe;
  } catch (Exception e) {
    e.printStackTrace();
    throw new IOException(e);
  }
  
  writer.close();

  nano = System.nanoTime() - nano;
  
  System.out.println("Dumped " + gts + " GTS (" + chunks + " chunks, " + datapoints + " datapoints, " + bytes + " bytes) in " + (nano / 1000000.0D) + " ms.");
}
 

private static String textifyBytes(Text t) {
  BytesWritable b = new BytesWritable();
  b.set(t.getBytes(), 0, t.getLength());
  return b.toString();
}