org.apache.hadoop.io.SortedMapWritable Java Examples

The following examples show how to use org.apache.hadoop.io.SortedMapWritable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MedianAndStandardDeviationCommentLengthByHour.java    From hadoop-map-reduce-patterns with Apache License 2.0 6 votes vote down vote up 
protected void reduce(IntWritable key, Iterable<SortedMapWritable> values, Context context)
		throws IOException, InterruptedException {
	SortedMapWritable outValue = new SortedMapWritable();
	for (SortedMapWritable v : values) {
		for (@SuppressWarnings("rawtypes")
		Entry<WritableComparable, Writable> entry : v.entrySet()) {
			LongWritable count = (LongWritable) outValue.get(entry.getKey());
			if (count != null) {
				count.set(count.get() + ((LongWritable) entry.getValue()).get());
			} else {
				outValue.put(entry.getKey(),
						new LongWritable(((LongWritable) entry.getValue()).get()));
			}
		}
	}
	context.write(key, outValue);
}
Example #2
Source File: TypedBytesWritableOutput.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void write(Writable w) throws IOException {
  if (w instanceof TypedBytesWritable) {
    writeTypedBytes((TypedBytesWritable) w);
  } else if (w instanceof BytesWritable) {
    writeBytes((BytesWritable) w);
  } else if (w instanceof ByteWritable) {
    writeByte((ByteWritable) w);
  } else if (w instanceof BooleanWritable) {
    writeBoolean((BooleanWritable) w);
  } else if (w instanceof IntWritable) {
    writeInt((IntWritable) w);
  } else if (w instanceof VIntWritable) {
    writeVInt((VIntWritable) w);
  } else if (w instanceof LongWritable) {
    writeLong((LongWritable) w);
  } else if (w instanceof VLongWritable) {
    writeVLong((VLongWritable) w);
  } else if (w instanceof FloatWritable) {
    writeFloat((FloatWritable) w);
  } else if (w instanceof DoubleWritable) {
    writeDouble((DoubleWritable) w);
  } else if (w instanceof Text) {
    writeText((Text) w);
  } else if (w instanceof ArrayWritable) {
    writeArray((ArrayWritable) w);
  } else if (w instanceof MapWritable) {
    writeMap((MapWritable) w);
  } else if (w instanceof SortedMapWritable) {
    writeSortedMap((SortedMapWritable) w);
  } else if (w instanceof Record) {
    writeRecord((Record) w);
  } else {
    writeWritable(w); // last resort
  }
}
Example #3
Source File: TypedBytesWritableOutput.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void writeSortedMap(SortedMapWritable smw) throws IOException {
  out.writeMapHeader(smw.size());
  for (Map.Entry<WritableComparable, Writable> entry : smw.entrySet()) {
    write(entry.getKey());
    write(entry.getValue());
  }
}
Example #4
Source File: TypedBytesWritableOutput.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void write(Writable w) throws IOException {
  if (w instanceof TypedBytesWritable) {
    writeTypedBytes((TypedBytesWritable) w);
  } else if (w instanceof BytesWritable) {
    writeBytes((BytesWritable) w);
  } else if (w instanceof ByteWritable) {
    writeByte((ByteWritable) w);
  } else if (w instanceof BooleanWritable) {
    writeBoolean((BooleanWritable) w);
  } else if (w instanceof IntWritable) {
    writeInt((IntWritable) w);
  } else if (w instanceof VIntWritable) {
    writeVInt((VIntWritable) w);
  } else if (w instanceof LongWritable) {
    writeLong((LongWritable) w);
  } else if (w instanceof VLongWritable) {
    writeVLong((VLongWritable) w);
  } else if (w instanceof FloatWritable) {
    writeFloat((FloatWritable) w);
  } else if (w instanceof DoubleWritable) {
    writeDouble((DoubleWritable) w);
  } else if (w instanceof Text) {
    writeText((Text) w);
  } else if (w instanceof ArrayWritable) {
    writeArray((ArrayWritable) w);
  } else if (w instanceof MapWritable) {
    writeMap((MapWritable) w);
  } else if (w instanceof SortedMapWritable) {
    writeSortedMap((SortedMapWritable) w);
  } else if (w instanceof Record) {
    writeRecord((Record) w);
  } else {
    writeWritable(w); // last resort
  }
}
Example #5
Source File: TypedBytesWritableOutput.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void writeSortedMap(SortedMapWritable smw) throws IOException {
  out.writeMapHeader(smw.size());
  for (Map.Entry<WritableComparable, Writable> entry : smw.entrySet()) {
    write(entry.getKey());
    write(entry.getValue());
  }
}
Example #6
Source File: MedianAndStandardDeviationCommentLengthByHour.java    From hadoop-map-reduce-patterns with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("deprecation")
public void map(Object key, Text value, Context context) throws IOException,
		InterruptedException {
	Map<String, String> parsed = transformXmlToMap(value.toString());
	// Grab the "CreationDate" field,
	// since it is what we are grouping by
	String strDate = parsed.get("CreationDate");
	// Grab the comment to find the length
	String text = parsed.get("Text");
	// Get the hour this comment was posted in
	if (isNullOrEmpty(strDate) || isNullOrEmpty(text)) {
		return;
	}

	Date creationDate;

	try {
		creationDate = DATE_FORMAT.parse(strDate);
	} catch (ParseException e) {
		e.printStackTrace();
		return;
	}

	outHour.set(creationDate.getHours());
	commentLength.set(text.length());
	SortedMapWritable outCommentLength = new SortedMapWritable();
	outCommentLength.put(commentLength, ONE);
	context.write(outHour, outCommentLength);
}
Example #7
Source File: MedianAndStandardDeviationCommentLengthByHour.java    From hadoop-map-reduce-patterns with Apache License 2.0 5 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
	Configuration conf = new Configuration();
	String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
	if (otherArgs.length != 2) {
		System.err.println("Usage: MedianAndStandardDeviationCommentLengthByHour <in> <out>");
		ToolRunner.printGenericCommandUsage(System.err);
		System.exit(2);
	}

	Job job = new Job(conf,
			"StackOverflow Median and Standard Deviation Comment Length By Hour");
	job.setJarByClass(MedianAndStandardDeviationCommentLengthByHour.class);
	job.setInputFormatClass(TextInputFormat.class);
	job.setMapperClass(MedianStdDevMapper.class);
	job.setCombinerClass(MedianStdDevCombiner.class);
	job.setReducerClass(MedianStdDevReducer.class);
	job.setMapOutputKeyClass(IntWritable.class);
	job.setMapOutputValueClass(SortedMapWritable.class);
	job.setOutputKeyClass(IntWritable.class);
	job.setOutputValueClass(MedianStdDevTuple.class);
	FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
	FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
	boolean success = job.waitForCompletion(true);

	return success ? 0 : 1;
}
Example #8
Source File: TypedBytesWritableInput.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public SortedMapWritable readSortedMap() throws IOException {
  return readSortedMap(null);
}
Example #9
Source File: TypedBytesWritableInput.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public SortedMapWritable readSortedMap() throws IOException {
  return readSortedMap(null);
}