org.apache.hadoop.io.WritableUtils Java Examples

@Override
public void readFields(DataInput in) throws IOException {
  size = WritableUtils.readVInt(in);
  int payload = size - WritableUtils.getVIntSize(size);
  if (payload > Long.SIZE / Byte.SIZE) {
    seed = in.readLong();
    payload -= Long.SIZE / Byte.SIZE;
  } else {
    Arrays.fill(literal, (byte)0);
    in.readFully(literal, 0, payload);
    dib.reset(literal, 0, literal.length);
    seed = dib.readLong();
    payload = 0;
  }
  final int vBytes = in.skipBytes(payload);
  if (vBytes != payload) {
    throw new EOFException("Expected " + payload + ", read " + vBytes);
  }
}
 

/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
  super.readFields(in);

  //TODO: move it to DatanodeID once DatanodeID is not stored in FSImage
  this.ipcPort = in.readShort() & 0x0000ffff;

  this.capacity = in.readLong();
  this.dfsUsed = in.readLong();
  this.remaining = in.readLong();
  this.lastUpdate = in.readLong();
  this.xceiverCount = in.readInt();
  this.location = Text.readString(in);
  this.hostName = Text.readString(in);
  setAdminState(WritableUtils.readEnum(in, AdminStates.class));
}
 

@Override
public void readFields(DataInput input) throws IOException {
    // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to save space
    int typeAndFlag = WritableUtils.readVInt(input);
    isNullable = (typeAndFlag & 0x01) != 0;
    if ((typeAndFlag & 0x02) != 0) {
        scale = WritableUtils.readVInt(input);
    }
    if ((typeAndFlag & 0x04) != 0) {
        maxLength = WritableUtils.readVInt(input);
    }
    type = PDataType.values()[typeAndFlag >>> 3];
    if (type.isFixedWidth() && type.getByteSize() == null) {
        byteSize = WritableUtils.readVInt(input);
    }
    columnModifier = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
}
 

private long writeRecords(int count, boolean knownKeyLength,
    boolean knownValueLength, boolean close) throws IOException {
  long rawDataSize = 0;
  for (int nx = 0; nx < count; nx++) {
    String key = TestDTFileByteArrays.composeSortedKey("key", nx);
    DataOutputStream outKey =
        writer.prepareAppendKey(knownKeyLength ? key.length() : -1);
    outKey.write(key.getBytes());
    outKey.close();
    String value = "value" + nx;
    DataOutputStream outValue =
        writer.prepareAppendValue(knownValueLength ? value.length() : -1);
    outValue.write(value.getBytes());
    outValue.close();
    rawDataSize +=
        WritableUtils.getVIntSize(key.getBytes().length)
            + key.getBytes().length
            + WritableUtils.getVIntSize(value.getBytes().length)
            + value.getBytes().length;
  }
  if (close) {
    closeOutput();
  }
  return rawDataSize;
}
 

@Override
public void write(DataOutput out) throws IOException {
  Text.writeString(out, queueName);
  WritableUtils.writeEnum(out, queueState);
  
  if(schedulingInfo!= null) {
    Text.writeString(out, schedulingInfo);
  }else {
    Text.writeString(out, "N/A");
  }
  out.writeInt(stats.length);
  for (JobStatus stat : stats) {
    stat.write(out);
  }
  out.writeInt(children.size());
  for(QueueInfo childQueueInfo : children) {
    childQueueInfo.write(out);
  }
}
 

public boolean next(DataInputBuffer key, DataInputBuffer value)
    throws IOException {
  MemoryBlockIndex memBlkIdx = keyValueIterator.next();
  if (memBlkIdx != null) {
    int pos = memBlkIdx.getIndex();
    MemoryBlock memBlk = memBlkIdx.getMemoryBlock();
    int offset = memBlk.offsets[pos];
    int keyLen = memBlk.keyLenArray[pos];
    int valLen = memBlk.valueLenArray[pos];
    dataOutputBuffer.reset();
    dataOutputBuffer.writeInt(keyLen);
    dataOutputBuffer.write(kvbuffer, offset, keyLen);
    dataOutputBuffer.writeInt(valLen);
    dataOutputBuffer.write(kvbuffer, offset + keyLen, valLen);
    key.reset(dataOutputBuffer.getData(), 0, keyLen
        + WritableUtils.INT_LENGTH_BYTES);
    value.reset(dataOutputBuffer.getData(), keyLen
        + WritableUtils.INT_LENGTH_BYTES, valLen
        + WritableUtils.INT_LENGTH_BYTES);
    return true;
  }
  return false;
}
 

/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

/**
 * FileSystemGroup ::= #scheme (scheme #counter (key value)*)*
 */
@Override
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, map.size()); // #scheme
  for (Map.Entry<String, Object[]> entry : map.entrySet()) {
    WritableUtils.writeString(out, entry.getKey()); // scheme
    // #counter for the above scheme
    WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
    for (Object counter : entry.getValue()) {
      if (counter == null) continue;
      @SuppressWarnings("unchecked")
      FSCounter c = (FSCounter) ((Counter)counter).getUnderlyingCounter();
      WritableUtils.writeVInt(out, c.key.ordinal());  // key
      WritableUtils.writeVLong(out, c.getValue());    // value
    }
  }
}
 

public void write(DataOutput out) throws IOException {
  Text.writeString(out, jobFile);
  taskId.write(out);
  out.writeInt(partition);
  out.writeInt(numSlotsRequired);
  taskStatus.write(out);
  skipRanges.write(out);
  out.writeBoolean(skipping);
  out.writeBoolean(jobCleanup);
  if (jobCleanup) {
    WritableUtils.writeEnum(out, jobRunStateForCleanup);
  }
  out.writeBoolean(jobSetup);
  out.writeBoolean(writeSkipRecs);
  out.writeBoolean(taskCleanup);
  Text.writeString(out, user);
  out.writeInt(encryptedSpillKey.length);
  extraData.write(out);
  out.write(encryptedSpillKey);
}
 

/**
 * Sets this instance's content from the input.
 */
@Override
public void readFields(DataInput in)
        throws IOException {
    // Read the length as a variable int
    int length = WritableUtils.readVInt(in);
    // If necessary increase the buffer capacity
    if (length > _text.capacity()) {
        _text = ByteBuffer.allocate(length);
    }
    // For efficiency read directly into the buffer's array
    in.readFully(_text.array(), 0, length);
    // Since we bypassed putting into the buffer set the position and limit directly
    _text.position(0);
    _text.limit(length);
    // Set the map to null since the contents may have changed.
    _map = null;
}
 

public void write(DataOutput out) throws IOException {
  taskid.write(out);
  out.writeFloat(progress);
  Text.writeString(out, state);
  out.writeLong(startTime);
  out.writeLong(finishTime);
  out.writeBoolean(runOnGPU);
  WritableUtils.writeStringArray(out, diagnostics);
  counters.write(out);
  WritableUtils.writeEnum(out, currentStatus);
  if (currentStatus == TIPStatus.RUNNING) {
    WritableUtils.writeVInt(out, runningAttempts.size());
    TaskAttemptID t[] = new TaskAttemptID[0];
    t = runningAttempts.toArray(t);
    for (int i = 0; i < t.length; i++) {
      t[i].write(out);
    }
  } else if (currentStatus == TIPStatus.COMPLETE) {
    successfulAttempt.write(out);
  }
}
 

public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
 

public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
 

/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
  super.write(out);

  //TODO: move it to DatanodeID once DatanodeID is not stored in FSImage
  out.writeShort(ipcPort);

  out.writeLong(capacity);
  out.writeLong(dfsUsed);
  out.writeLong(remaining);
  out.writeLong(lastUpdate);
  out.writeInt(xceiverCount);
  Text.writeString(out, location);
  Text.writeString(out, hostName == null? "": hostName);
  WritableUtils.writeEnum(out, getAdminState());
}
 

/**
 * FileSystemGroup ::= #scheme (scheme #counter (key value)*)*
 */
@Override
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, map.size()); // #scheme
  for (Map.Entry<String, Object[]> entry : map.entrySet()) {
    WritableUtils.writeString(out, entry.getKey()); // scheme
    // #counter for the above scheme
    WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
    for (Object counter : entry.getValue()) {
      if (counter == null) continue;
      FSCounter c = (FSCounter) ((TezCounter)counter).getUnderlyingCounter();
      WritableUtils.writeVInt(out, c.key.ordinal());  // key
      WritableUtils.writeVLong(out, c.getValue());    // value
    }
  }
}
 

/**
 * Aggregate all LinkNode objects for a given url.
 */
public void reduce(Text key, Iterator<LinkNode> values,
  OutputCollector<Text, LinkNodes> output, Reporter reporter)
  throws IOException {

  List<LinkNode> nodeList = new ArrayList<LinkNode>();
  int numNodes = 0;

  while (values.hasNext()) {
    LinkNode cur = values.next();
    if (numNodes < maxInlinks) {
      nodeList.add((LinkNode)WritableUtils.clone(cur, conf));
      numNodes++;
    }
    else {
      break;
    }
  }

  LinkNode[] linkNodesAr = nodeList.toArray(new LinkNode[nodeList.size()]);
  LinkNodes linkNodes = new LinkNodes(linkNodesAr);
  output.collect(key, linkNodes);
}
 

/**
 * {@inheritDoc}
 * @throws IOException If the child InputSplit cannot be read, typically
 *                     for faliing access checks.
 */
@SuppressWarnings("unchecked")  // Generic array assignment
public void readFields(DataInput in) throws IOException {
  int card = WritableUtils.readVInt(in);
  if (splits == null || splits.length != card) {
    splits = new InputSplit[card];
  }
  Class<? extends InputSplit>[] cls = new Class[card];
  try {
    for (int i = 0; i < card; ++i) {
      cls[i] =
        Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
    }
    for (int i = 0; i < card; ++i) {
      splits[i] = ReflectionUtils.newInstance(cls[i], null);
      splits[i].readFields(in);
    }
  } catch (ClassNotFoundException e) {
    throw (IOException)new IOException("Failed split init").initCause(e);
  }
}
 

@Override
public void readFields(DataInput input) throws IOException {
    this.emptyCFName = WritableUtils.readCompressedByteArray(input);
    int bitsetLongArraySize = WritableUtils.readVInt(input);
    long[] bitsetLongArray = new long[bitsetLongArraySize];
    for (int i = 0; i < bitsetLongArraySize; i++) {
        bitsetLongArray[i] = WritableUtils.readVLong(input);
    }
    this.trackedColumns = BitSet.valueOf(bitsetLongArray);
    this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)];
    int conditionOnlyCfsSize = WritableUtils.readVInt(input);
    this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
    while (conditionOnlyCfsSize > 0) {
        this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input));
        conditionOnlyCfsSize--;
    }
}
 

public void readFields(DataInput in) throws IOException {
  this.taskid.readFields(in);
  this.progress = in.readFloat();
  this.runState = WritableUtils.readEnum(in, State.class);
  this.diagnosticInfo = Text.readString(in);
  this.stateString = Text.readString(in);
  this.phase = WritableUtils.readEnum(in, Phase.class); 
  this.startTime = in.readLong(); 
  this.finishTime = in.readLong(); 
  counters = new Counters();
  this.includeCounters = in.readBoolean();
  this.outputSize = in.readLong();
  if (includeCounters) {
    counters.readFields(in);
  }
  nextRecordRange.readFields(in);
  this.runOnGPU = in.readBoolean();
}
 

public synchronized void readFields(DataInput in) throws IOException {
  this.jobid = new JobID();
  this.jobid.readFields(in);
  this.setupProgress = in.readFloat();
  this.mapProgress = in.readFloat();
  this.reduceProgress = in.readFloat();
  this.cleanupProgress = in.readFloat();
  this.runState = WritableUtils.readEnum(in, State.class);
  this.startTime = in.readLong();
  this.user = StringInterner.weakIntern(Text.readString(in));
  this.priority = WritableUtils.readEnum(in, JobPriority.class);
  this.schedulingInfo = StringInterner.weakIntern(Text.readString(in));
  this.finishTime = in.readLong();
  this.isRetired = in.readBoolean();
  this.historyFile = StringInterner.weakIntern(Text.readString(in));
  this.jobName = StringInterner.weakIntern(Text.readString(in));
  this.trackingUrl = StringInterner.weakIntern(Text.readString(in));
  this.jobFile = StringInterner.weakIntern(Text.readString(in));
  this.isUber = in.readBoolean();

  // De-serialize the job's ACLs
  int numACLs = in.readInt();
  for (int i = 0; i < numACLs; i++) {
    JobACL aclType = WritableUtils.readEnum(in, JobACL.class);
    AccessControlList acl = new AccessControlList(" ");
    acl.readFields(in);
    this.jobACLs.put(aclType, acl);
  }
}
 

/**
 * check if a token is expired. for unit test only. return true when token is
 * expired, false otherwise
 */
static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
    throws IOException {
  ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
  DataInputStream in = new DataInputStream(buf);
  long expiryDate = WritableUtils.readVLong(in);
  return isExpired(expiryDate);
}
 

@Override
public void write(DataOutput output) throws IOException {
    WritableUtils.writeVInt(output, (type.ordinal() + 1) * (this.isNullable ? -1 : 1));
    WritableUtils.writeVInt(output, count * (columnModifier == null ? 1 : -1));
    if (type.isFixedWidth() && type.getByteSize() == null) {
        WritableUtils.writeVInt(output, byteSize);
    }
}
 

@Override
public void readFields(DataInput in) throws IOException {
    int ordinal = in.readInt();
    contentType = ContentType.valueOf(ordinal);
    int length = WritableUtils.readVInt(in);
    if (length > MAX_BUFFER_SIZE) {
        is = (DataInputStream)in;
        return;
    }
    content = new byte[length];
    in.readFully(content, 0, length);
}
 

@Override
public void readFields(DataInput input) throws IOException {
    try {
        expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
        expression.readFields(input);
        expression.reset(); // Initializes expression tree for partial evaluation
    } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
        ServerUtil.throwIOException("BooleanExpressionFilter failed during reading", t);
    }
}
 

/**
 * Prefix encodes the byte array from offset to length into output stream. 
 * Instead of writing the entire byte array, only the portion of the byte array
 * that differs from the beginning of the previous byte array written is written.
 *  
 * @param out output stream to encode into
 * @param b byte array buffer
 * @param offset offset into byte array to start encoding
 * @param length length of byte array to encode
 * @throws IOException
 */
public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException {
      int i = 0;
      int prevOffset = previous.getOffset();
      byte[] prevBytes = previous.get();
      int prevLength = previous.getLength();
      int minLength = prevLength < b.length ? prevLength : b.length;
      for(i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++);
      WritableUtils.writeVInt(out, i);
      Bytes.writeByteArray(out, b, offset + i, length - i);
      previous.set(b, offset, length);
      if (length > maxLength) {
          maxLength = length;
      }
}
 

/**
 * Read byte-array written with a WritableableUtils.vint prefix.
 * @param in Input to read from.
 * @return byte array read off <code>in</code>
 * @throws java.io.IOException e
 */
public static byte [] readByteArray(final DataInput in)
throws IOException {
  int len = WritableUtils.readVInt(in);
  if (len < 0) {
    throw new NegativeArraySizeException(Integer.toString(len));
  }
  byte [] result = new byte[len];
  in.readFully(result, 0, len);
  return result;
}
 

@Override
public void readFields(DataInput in) throws IOException {
  int numSchemes = WritableUtils.readVInt(in);    // #scheme
  FileSystemCounter[] enums = FileSystemCounter.values();
  for (int i = 0; i < numSchemes; ++i) {
    String scheme = WritableUtils.readString(in); // scheme
    int numCounters = WritableUtils.readVInt(in); // #counter
    for (int j = 0; j < numCounters; ++j) {
      findCounter(scheme, enums[WritableUtils.readVInt(in)])  // key
          .setValue(WritableUtils.readVLong(in)); // value
    }
  }
}
 

public int writeLength() throws IOException {
  int length = 0;
  int mapIdLength = Text.encode(mapId).limit();
  length += mapIdLength;

  length += WritableUtils.getVIntSize(mapIdLength);
  length += WritableUtils.getVIntSize(compressedLength);
  length += WritableUtils.getVIntSize(uncompressedLength);
  length += WritableUtils.getVIntSize(forReduce);

  return length;
}
 

public void readFields(DataInput in) throws IOException {
    activeName = WritableUtils.readString(in);
    sessionID = WritableUtils.readString(in);
    timeTag = in.readLong();
    ip = WritableUtils.readString(in);
    deviceID = WritableUtils.readString(in);
    reqUrl = WritableUtils.readString(in);
    userID = WritableUtils.readString(in);
    productID = WritableUtils.readString(in);
    orderID = WritableUtils.readString(in);
}
 

/** Writes each Writable to <code>out</code>.
 * TupleWritable format:
 * {@code
 *  <count><type1><type2>...<typen><obj1><obj2>...<objn>
 * }
 */
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, values.length);
  WritableUtils.writeVLong(out, written);
  for (int i = 0; i < values.length; ++i) {
    Text.writeString(out, values[i].getClass().getName());
  }
  for (int i = 0; i < values.length; ++i) {
    if (has(i)) {
      values[i].write(out);
    }
  }
}