Java Code Examples for org.apache.hadoop.io.WritableComparable#getClass()

The following examples show how to use org.apache.hadoop.io.WritableComparable#getClass() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SortValidator.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", received " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 2

Source File: SortValidator.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", received " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 3

Source File: SortValidator.java From RDFS with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", recieved " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 4

Source File: SortValidator.java From hadoop-gpu with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", recieved " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}