Java Code Examples for org.apache.hadoop.io.WritableUtils#readVInt()

/**
 * {@inheritDoc}
 * @throws IOException If the child InputSplit cannot be read, typically
 *                     for failing access checks.
 */
@SuppressWarnings("unchecked")  // Generic array assignment
public void readFields(DataInput in) throws IOException {
  int card = WritableUtils.readVInt(in);
  if (splits == null || splits.length != card) {
    splits = new InputSplit[card];
  }
  Class<? extends InputSplit>[] cls = new Class[card];
  try {
    for (int i = 0; i < card; ++i) {
      cls[i] =
        Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
    }
    for (int i = 0; i < card; ++i) {
      splits[i] = ReflectionUtils.newInstance(cls[i], null);
      SerializationFactory factory = new SerializationFactory(conf);
      Deserializer deserializer = factory.getDeserializer(cls[i]);
      deserializer.open((DataInputStream)in);
      splits[i] = (InputSplit)deserializer.deserialize(splits[i]);
    }
  } catch (ClassNotFoundException e) {
    throw new IOException("Failed split init", e);
  }
}
 

public static List<IndexMaintainer> deserialize(byte[] buf, int offset, int length) {
    ByteArrayInputStream stream = new ByteArrayInputStream(buf, offset, length);
    DataInput input = new DataInputStream(stream);
    List<IndexMaintainer> maintainers = Collections.emptyList();
    try {
        int size = WritableUtils.readVInt(input);
        boolean isDataTableSalted = size < 0;
        size = Math.abs(size);
        RowKeySchema rowKeySchema = new RowKeySchema();
        rowKeySchema.readFields(input);
        maintainers = Lists.newArrayListWithExpectedSize(size);
        for (int i = 0; i < size; i++) {
            IndexMaintainer maintainer = new IndexMaintainer(rowKeySchema, isDataTableSalted);
            maintainer.readFields(input);
            maintainers.add(maintainer);
        }
    } catch (IOException e) {
        throw new RuntimeException(e); // Impossible
    }
    return maintainers;
}
 

@Override
public void readFields(DataInput input) throws IOException {
    super.readFields(input);
    this.decodedColumnQualifier = WritableUtils.readVInt(input);
    int serializedEncodingScheme = WritableUtils.readVInt(input);
    // prior to PHOENIX-4432 we weren't writing out the immutableStorageScheme in write(),
    // so we use the decodedColumnQualifier sign to determine whether it's there
    if (Integer.signum(serializedEncodingScheme) == -1) {
        this.immutableStorageScheme =
                ImmutableStorageScheme
                        .fromSerializedValue((byte) WritableUtils.readVInt(input));
        serializedEncodingScheme = -serializedEncodingScheme;
    } else {
        this.immutableStorageScheme = ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
    }
    this.encodingScheme = QualifierEncodingScheme.values()[serializedEncodingScheme];
    setKeyValueExpression();
}
 

/** Read a compressed buffer */
private synchronized void readBuffer(DataInputBuffer buffer, 
                                     CompressionInputStream filter) throws IOException {
  // Read data into a temporary buffer
  DataOutputBuffer dataBuffer = new DataOutputBuffer();

  try {
    int dataBufferLength = WritableUtils.readVInt(in);
    dataBuffer.write(in, dataBufferLength);
  
    // Set up 'buffer' connected to the input-stream
    buffer.reset(dataBuffer.getData(), 0, dataBuffer.getLength());
  } finally {
    dataBuffer.close();
  }

  // Reset the codec
  filter.resetState();
}
 

private List<Expression> deserializeGroupByExpressions(byte[] expressionBytes, int offset)
        throws IOException {
    List<Expression> expressions = new ArrayList<Expression>(3);
    ByteArrayInputStream stream = new ByteArrayInputStream(expressionBytes);
    try {
        DataInputStream input = new DataInputStream(stream);
        while (true) {
            try {
                int expressionOrdinal = WritableUtils.readVInt(input);
                Expression expression =
                        ExpressionType.values()[expressionOrdinal].newInstance();
                expression.readFields(input);
                if (offset != 0) {
                    IndexUtil.setRowKeyExpressionOffset(expression, offset);
                }
                expressions.add(expression);
            } catch (EOFException e) {
                break;
            }
        }
    } finally {
        stream.close();
    }
    return expressions;
}
 

@Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
  try {
    di.reset(b1, s1, l1);
    final int x1 = WritableUtils.readVInt(di);
    di.reset(b2, s2, l2);
    final int x2 = WritableUtils.readVInt(di);
    final int ret = (b1[s1 + x1] != b2[s2 + x2])
      ? b1[s1 + x1] - b2[s2 + x2]
      : super.compare(b1, s1, x1, b2, s2, x2);
    di.reset(reset, 0, 0);
    return ret;
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

protected void readObject(Writable obj, DataInputStream inStream) throws IOException {
  int numBytes = WritableUtils.readVInt(inStream);
  byte[] buffer;
  // For BytesWritable and Text, use the specified length to set the length
  // this causes the "obvious" translations to work. So that if you emit
  // a string "abc" from C++, it shows up as "abc".
  if (obj instanceof BytesWritable) {
    buffer = new byte[numBytes];
    inStream.readFully(buffer);
    ((BytesWritable) obj).set(buffer, 0, numBytes);
  } else if (obj instanceof Text) {
    buffer = new byte[numBytes];
    inStream.readFully(buffer);
    ((Text) obj).set(buffer);
  } else {
    obj.readFields(inStream);
  }
}
 

@Override
public void readFields(DataInput input) throws IOException {
    int encodedByteLengthAndBool = WritableUtils.readVInt(input);
    this.isDeterministic = encodedByteLengthAndBool > 0;
    int byteLength = Math.abs(encodedByteLengthAndBool)-1;
    this.byteValue = new byte[byteLength];
    input.readFully(byteValue, 0, byteLength);
    columnModifier = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
    int typeOrdinal = WritableUtils.readVInt(input);
    if (typeOrdinal < 0) {
        this.type = null;
    } else {
        this.type = PDataType.values()[typeOrdinal];
    }
    if (this.byteValue.length == 0) {
        this.value = null;
    } else {
        this.value = this.type.toObject(byteValue, 0, byteValue.length, this.type, columnModifier);
    }
    // Only to prevent continual reallocations of Integer
    this.byteSize = this.byteValue.length;
}
 

/** deserialization method */
public void readFields(DataInput in) throws IOException {
  block.readFields(in);
  int len = WritableUtils.readVInt(in); // variable length integer
  datanodeIDs = new String[len];
  for(int i=0; i<len; i++) {
    datanodeIDs[i] = Text.readString(in);
  }
}
 

@Override
public void readFields(DataInput input) throws IOException {
    int len = WritableUtils.readVInt(input);
    List<Expression>children = new ArrayList<Expression>(len);
    for (int i = 0; i < len; i++) {
        Expression child = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
        child.readFields(input);
        children.add(child);
    }
    init(children);
}
 

@Override
public void readFields(DataInput in) throws IOException {
  rec_in = WritableUtils.readVLong(in);
  rec_out = WritableUtils.readVLong(in);
  bytes_out = WritableUtils.readVLong(in);
  sizeOfResourceUsageMetrics =  WritableUtils.readVInt(in);
  if (sizeOfResourceUsageMetrics > 0) {
    metrics = new ResourceUsageMetrics();
    metrics.readFields(in);
  }
}
 

protected void readKeyValueLength(DataInput dIn) throws IOException {
  currentKeyLength = WritableUtils.readVInt(dIn);
  currentValueLength = WritableUtils.readVInt(dIn);
  if (currentKeyLength != RLE_MARKER) {
    // original key length
    originalKeyLength = currentKeyLength;
  }
  bytesRead +=
      WritableUtils.getVIntSize(currentKeyLength)
          + WritableUtils.getVIntSize(currentValueLength);
}
 

@Override
public void readFields(DataInput input) throws IOException {
    // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to save space
    int typeAndFlag = WritableUtils.readVInt(input);
    isNullable = (typeAndFlag & 0x01) != 0;
    if ((typeAndFlag & 0x02) != 0) {
        scale = WritableUtils.readVInt(input);
    }
    if ((typeAndFlag & 0x04) != 0) {
        maxLength = WritableUtils.readVInt(input);
    }
    type = PDataType.values()[typeAndFlag >>> 3];
    sortOrder = SortOrder.fromSystemValue(WritableUtils.readVInt(input));
}
 

@Override
public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) {
    InputStream is = new ByteArrayInputStream(ptr.get(), ptr.getOffset() + 1, ptr.getLength() - 1);
    try {
        if (Bytes.equals(ptr.get(), ptr.getOffset(), 1, DistinctValueWithCountServerAggregator.COMPRESS_MARKER, 0,
                1)) {
            InputStream decompressionStream = DistinctValueWithCountServerAggregator.COMPRESS_ALGO
                    .createDecompressionStream(is,
                            DistinctValueWithCountServerAggregator.COMPRESS_ALGO.getDecompressor(), 0);
            is = decompressionStream;
        }
        DataInputStream in = new DataInputStream(is);
        int mapSize = WritableUtils.readVInt(in);
        for (int i = 0; i < mapSize; i++) {
            int keyLen = WritableUtils.readVInt(in);
            byte[] keyBytes = new byte[keyLen];
            in.read(keyBytes, 0, keyLen);
            ImmutableBytesPtr key = new ImmutableBytesPtr(keyBytes);
            int value = WritableUtils.readVInt(in);
            Integer curCount = valueVsCount.get(key);
            if (curCount == null) {
                valueVsCount.put(key, value);
            } else {
                valueVsCount.put(key, curCount + value);
            }
            totalCount += value;
        }
    } catch (IOException ioe) {
        throw new RuntimeException(ioe); // Impossible as we're using a ByteArrayInputStream
    }
    if (buffer == null) {
        initBuffer();
    }
}
 

private HashCacheImpl(byte[] hashCacheBytes, MemoryChunk memoryChunk, int clientVersion) {
    try {
        this.memoryChunk = memoryChunk;
        this.clientVersion = clientVersion;
        byte[] hashCacheByteArray = hashCacheBytes;
        int offset = 0;
        ByteArrayInputStream input = new ByteArrayInputStream(hashCacheByteArray, offset, hashCacheBytes.length);
        DataInputStream dataInput = new DataInputStream(input);
        int nExprs = dataInput.readInt();
        List<Expression> onExpressions = new ArrayList<Expression>(nExprs);
        for (int i = 0; i < nExprs; i++) {
            int expressionOrdinal = WritableUtils.readVInt(dataInput);
            Expression expression = ExpressionType.values()[expressionOrdinal].newInstance();
            expression.readFields(dataInput);
            onExpressions.add(expression);                        
        }
        boolean singleValueOnly = false;
        int exprSizeAndSingleValueOnly = dataInput.readInt();
        int exprSize = exprSizeAndSingleValueOnly;
        if (exprSize < 0) {
            exprSize *= -1;
            singleValueOnly = true;
        }
        this.singleValueOnly = singleValueOnly;
        offset += exprSize;
        int nRows = dataInput.readInt();
        long estimatedSize = SizedUtil.sizeOfMap(nRows, SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE, SizedUtil.RESULT_SIZE) + hashCacheBytes.length;
        this.memoryChunk.resize(estimatedSize);
        HashMap<ImmutableBytesPtr,List<Tuple>> hashCacheMap = new HashMap<ImmutableBytesPtr,List<Tuple>>(nRows * 5 / 4);
        offset += Bytes.SIZEOF_INT;
        // Build Map with evaluated hash key as key and row as value
        for (int i = 0; i < nRows; i++) {
            int resultSize = (int)Bytes.readVLong(hashCacheByteArray, offset);
            offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]);
            ImmutableBytesWritable value = new ImmutableBytesWritable(hashCacheByteArray,offset,resultSize);
            Tuple result = new ResultTuple(ResultUtil.toResult(value));
            ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, onExpressions);
            List<Tuple> tuples = hashCacheMap.get(key);
            if (tuples == null) {
                tuples = new LinkedList<Tuple>();
                hashCacheMap.put(key, tuples);
            }
            tuples.add(result);
            offset += resultSize;
        }
        this.hashCache = Collections.unmodifiableMap(hashCacheMap);
    } catch (IOException e) { // Not possible with ByteArrayInputStream
        throw new RuntimeException(e);
    }
}
 

public void binaryProtocolStub() {
  try {

    initSoket();

    //should be 5
    //RUN_REDUCE boolean 
    WritableUtils.readVInt(dataInput);
    WritableUtils.readVInt(dataInput);
    int intValue = WritableUtils.readVInt(dataInput);
    System.out.println("getIsJavaRecordWriter:" + intValue);

    // reduce key
    WritableUtils.readVInt(dataInput);
    // value of reduce key
    BooleanWritable value = new BooleanWritable();
    readObject(value, dataInput);
    System.out.println("reducer key :" + value);
    // reduce value code:

    // reduce values
    while ((intValue = WritableUtils.readVInt(dataInput)) == 7) {
      Text txt = new Text();
      // value
      readObject(txt, dataInput);
      System.out.println("reduce value  :" + txt);
    }


    // done
    WritableUtils.writeVInt(dataOut, 54);

    dataOut.flush();
    dataOut.close();

  } catch (Exception x) {
    x.printStackTrace();
  } finally {
    closeSoket();

  }
}
 

public static Bytes read(BytesBuilder bb, DataInputStream in) throws IOException {
  bb.setLength(0);
  int len = WritableUtils.readVInt(in);
  bb.append(in, len);
  return bb.toBytes();
}
 

@Override
public void readFields(DataInput input) throws IOException {
    super.readFields(input);
    this.returnType = PDataType.values()[WritableUtils.readVInt(input)];
}
 

@Override
public void readFields(DataInput input) throws IOException {
    super.readFields(input);
    this.returnType = PDataType.values()[WritableUtils.readVInt(input)];
}
 

public void readFields(DataInput in) throws IOException {
  bytes = WritableUtils.readVLong(in);
  records = WritableUtils.readVLong(in);
  checksum = WritableUtils.readVInt(in);
}