org.apache.hadoop.mapreduce.Mapper.Context Java Examples

public void map(LongWritable key, Writable value, Context context)
  throws IOException, InterruptedException {
  try {
    String str = value.toString();
    if (value instanceof Text) {
      writer.write(str, 0, str.length());
      writer.newLine();
    } else if (value instanceof SqoopRecord) {
      writer.write(str, 0, str.length());
    }
  } catch (Exception e) {
    doExecuteUpdate("DROP TABLE " + tmpTableName);
    cleanup(context);
    throw new IOException(e);
  }
}
 

@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.doSetup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 

@Override
public void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException {

  int counter = 0;
  
  System.out.println("starting mapper");
  System.out.println();
  for (int i = 0; i < numberOfRecords; i++) {
    String keyRoot = StringUtils.leftPad(Integer.toString(r.nextInt(Short.MAX_VALUE)), 5, '0');

    if (i % 1000 == 0) {
      System.out.print(".");
    }

    for (int j = 0; j < 10; j++) {
      hKey.set(Bytes.toBytes(keyRoot + "|" + runID + "|" + taskId));
      kv = new KeyValue(hKey.get(), columnFamily, Bytes.toBytes("C" + j), Bytes.toBytes("counter:" + counter++ ));
      context.write(hKey, kv);
    }
  }

  System.out.println("finished mapper");
}
 

public void map(Map<String, ByteBuffer> keys, Map<String, ByteBuffer> columns, Context context) throws IOException, InterruptedException
{
    for (Entry<String, ByteBuffer> column : columns.entrySet())
    {
        if (!"line".equalsIgnoreCase(column.getKey()))
            continue;

        String value = ByteBufferUtil.string(column.getValue());

        StringTokenizer itr = new StringTokenizer(value);
        while (itr.hasMoreTokens())
        {
            word.set(itr.nextToken());
            context.write(word, one);
        }
    }
}
 

@Override
protected void startAligner(Mapper.Context context) throws IOException, InterruptedException {
    // make command
    String customArgs = HalvadeConf.getCustomArgs(context.getConfiguration(), "bwa", "mem");
    String[] command = CommandGenerator.bwaMem(bin, ref, null, null, isPaired, true, threads, customArgs);
    pbw = new ProcessBuilderWrapper(command, bin);
    // run command
    // needs to be streamed to output otherwise the process blocks ...
    pbw.startProcess(null, System.err);
    // check if alive.
    if(!pbw.isAlive())
        throw new ProcessException("BWA mem", pbw.getExitState());
    pbw.getSTDINWriter();
    // make a SAMstream handler
    ssh = new SAMStreamHandler(instance, context, false);
    ssh.start();
}
 

@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = new Configuration();
    BaseCuboidMapper mapper = new BaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.setup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 

@Override
public void map(LongWritable key, Writable value, Context context)
  throws IOException, InterruptedException {
  line.setLength(0);
  line.append(value.toString());
  if (value instanceof Text) {
    line.append(System.getProperty("line.separator"));
  }
  try {
    byte[]data = line.toString().getBytes("UTF-8");
    copyin.writeToCopy(data, 0, data.length);
  } catch (SQLException ex) {
    LoggingUtils.logAll(LOG, "Unable to execute copy", ex);
    close();
    throw new IOException(ex);
  }
}
 

@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.doSetup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 

/**
 * Constructor for SampleMachineConsumer - needs the Mapper Context
 *
 * @param context A Hadoop MapReduce Mapper.Context to which this consumer
 *                should writer
 */
public SampleMachineConsumer(final Context context) {
    super();

    ContextWriter contextWrite = new ContextWriter(context, template);
    this.addDataWriter(contextWrite);
    this.addDataTransformer(new SampleMachineTransformer());

    exit = new AtomicBoolean(false);
    handler = new JenkinsReportingHandler(exit);

    currentRow = -1;
    finalRow = -2;

    setReportGap(1000);
}
 

@SuppressWarnings("rawtypes")
public static void cleanup(final Context context) {
	closeMap();
	closeCommentMap();
	closeIssuesMap();
	closeCommitMap();
}
 

/**
 * Add logging in map cleanup method
 * 
 * @param context
 *            - map context
 * @param className
 *            - Class which is calling this method
 * @param methodName
 *            - Class Method which is calling this method
 */
@SuppressWarnings(RAW_TYPES)
public static void getMapContextInfoCleanup(Context context,
		String className, String methodName) {
	Counter counter = context.getCounter(MAPRED_COUNTER, MAP_INPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());

	counter = context.getCounter(MAPRED_COUNTER, MAP_OUTPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());
}
 

/**
 * Test map method of Importer
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testKeyValueImporter() throws Throwable {
  CellImporter importer = new CellImporter();
  Configuration configuration = new Configuration();
  Context ctx = mock(Context.class);
  when(ctx.getConfiguration()).thenReturn(configuration);

  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      MapReduceExtendedCell key = (MapReduceExtendedCell) invocation.getArgument(1);
      assertEquals("Key", Bytes.toString(writer.get()));
      assertEquals("row", Bytes.toString(CellUtil.cloneRow(key)));
      return null;
    }
  }).when(ctx).write(any(), any());

  importer.setup(ctx);
  Result value = mock(Result.class);
  KeyValue[] keys = {
      new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
          Bytes.toBytes("value")),
      new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
          Bytes.toBytes("value1")) };
  when(value.rawCells()).thenReturn(keys);
  importer.map(new ImmutableBytesWritable(Bytes.toBytes("Key")), value, ctx);

}
 

/**
 * Test IndexBuilder from examples
 */
@SuppressWarnings("unchecked")
@Test
public void testIndexBuilder() throws Exception {
  Configuration configuration = new Configuration();
  String[] args = { "tableName", "columnFamily", "column1", "column2" };
  IndexBuilder.configureJob(configuration, args);
  assertEquals("tableName", configuration.get("index.tablename"));
  assertEquals("tableName", configuration.get(TableInputFormat.INPUT_TABLE));
  assertEquals("column1,column2", configuration.get("index.fields"));

  Map map = new Map();
  ImmutableBytesWritable rowKey = new ImmutableBytesWritable(Bytes.toBytes("test"));
  Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context ctx =
      mock(Context.class);
  when(ctx.getConfiguration()).thenReturn(configuration);
  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      Put put = (Put) invocation.getArgument(1);
      assertEquals("tableName-column1", Bytes.toString(writer.get()));
      assertEquals("test", Bytes.toString(put.getRow()));
      return null;
    }
  }).when(ctx).write(any(), any());
  Result result = mock(Result.class);
  when(result.getValue(Bytes.toBytes("columnFamily"), Bytes.toBytes("column1"))).thenReturn(
      Bytes.toBytes("test"));
  map.setup(ctx);
  map.map(rowKey, result, ctx);
}
 

public void reduce(IntWritable key, Iterable<Text> values, Context context)
		throws IOException, InterruptedException {
	for (Text value : values) {
		// 排序后再次颠倒k-v，将日期作为key
		System.out.println(value.toString() + ":" + key.get());
		context.write(value, key);
	}
}
 

/**
 * Test SampleUploader from examples
 */
@SuppressWarnings("unchecked")
@Test
public void testSampleUploader() throws Exception {
  Configuration configuration = new Configuration();
  Uploader uploader = new Uploader();
  Mapper<LongWritable, Text, ImmutableBytesWritable, Put>.Context ctx = mock(Context.class);
  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      Put put = (Put) invocation.getArgument(1);
      assertEquals("row", Bytes.toString(writer.get()));
      assertEquals("row", Bytes.toString(put.getRow()));
      return null;
    }
  }).when(ctx).write(any(), any());

  uploader.map(null, new Text("row,family,qualifier,value"), ctx);

  Path dir = util.getDataTestDirOnTestFS("testSampleUploader");

  String[] args = { dir.toString(), "simpleTable" };
  Job job = SampleUploader.configureJob(configuration, args);
  assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());
}
 

public void map(Long key, Row row, Context context) throws IOException, InterruptedException
{
    String value = row.getString("line");
    logger.debug("read {}:{}={} from {}", new Object[] {key, "line", value, context.getInputSplit()});
    StringTokenizer itr = new StringTokenizer(value);
    while (itr.hasMoreTokens())
    {
        word.set(itr.nextToken());
        context.write(word, one);
    }
}
 

@Override
protected void cleanup(Context context)
  throws IOException, InterruptedException {
  try {
    copyin.endCopy();
  } catch (SQLException ex) {
    LoggingUtils.logAll(LOG, "Unable to finalize copy", ex);
    throw new IOException(ex);
  }
  close();
}
 

public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
{
    int sum = 0;
    for (IntWritable val : values)
        sum += val.get();
    context.write(key, new IntWritable(sum));
}
 

public void reduce(Text word, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
{
    int sum = 0;
    for (IntWritable val : values)
        sum += val.get();
    keys.put("word", ByteBufferUtil.bytes(word.toString()));
    context.write(keys, getBindVariables(word, sum));
}
 

@Override
public void setup(Context context) {
  System.out.println("starting setup");

  columnFamily = Bytes.toBytes(context.getConfiguration().get(COLUMN_FAMILY));
  runID = context.getConfiguration().get(RUN_ID);
  taskId = context.getTaskAttemptID().getTaskID().getId();
  numberOfRecords = context.getConfiguration().getInt(NUMBER_OF_RECORDS, 1000) / context.getConfiguration().getInt("nmapinputformat.num.maps", 1);

  System.out.println("finished setup");
}
 

public SAMStreamHandler(AlignerInstance instance, Context context, boolean useCompact) {
    this.is = instance.getSTDOUTStream();
    this.mFileHeader = instance.getFileHeader();
    this.instance = instance;
    this.useCompact = useCompact;
    mCurrentLine = null;
    mFile = null;
    validationStringency = ValidationStringency.LENIENT;
    mReader = new BufferedLineReader(this.is);
    samRecordFactory = new DefaultSAMRecordFactory();
    this.context = context;
    isPaired = HalvadeConf.getIsPaired(context.getConfiguration());
}
 

/**
 * 
 * This BWA instance runs BWA from stdin (custom provided BWA is needed)
 */
private BWAMemInstance(Context context, String bin) throws IOException, URISyntaxException {
    super(context, bin);
    String taskid = context.getTaskAttemptID().toString();
    taskid = taskid.substring(taskid.indexOf("m_"));
    ref = HalvadeFileUtils.downloadBWAIndex(context, taskid);
}
 

static public BWAMemInstance getBWAInstance(Mapper.Context context, String bin) throws IOException, InterruptedException, URISyntaxException {
    if(instance == null) {
        instance = new BWAMemInstance(context, bin);
        instance.startAligner(context);
    }
    BWAMemInstance.context = context;
    return instance;
}
 

public Crawler(String id, String seed, int topN, int deepth, CrawlDB db,
		Context context) {
	this.id = id;
	this.topN = topN;
	this.deepth = deepth;
	this.seed = seed;
	this.db = db;
	this.context = context;
}
 

public NegativeCellIdCounter(
    final Context context,
    final long level,
    final long minLevel,
    final long maxLevel) {
  super(context, level, minLevel, maxLevel);
}
 

public MapContextCellCounter(
    final Context context,
    final long level,
    final long minLevel,
    final long maxLevel) {
  this.context = context;
  this.level = level;
  this.minLevel = minLevel;
  this.maxLevel = maxLevel;
  numLevels = (maxLevel - minLevel) + 1;
}
 

public void recoverParityBlockToFile(FileSystem srcFs, Path srcPath,
		FileSystem parityFs, Path parityPath, long blockSize,
		long blockOffset, File localBlockFile, Context context)
		throws IOException, InterruptedException {
	OutputStream out = new FileOutputStream(localBlockFile);
	fixErasedBlock(srcFs, srcPath, parityFs, parityPath, false, blockSize,
			blockOffset, blockSize, false, out, context, false);
	out.close();
}
 

DecoderInputStream generateAlternateStream(FileSystem srcFs, Path srcFile,
		FileSystem parityFs, Path parityFile, long blockSize,
		long errorOffset, long limit, Context context) {
	configureBuffers(blockSize);
	Progressable reporter = context;
	if (reporter == null) {
		reporter = RaidUtils.NULL_PROGRESSABLE;
	}

	DecoderInputStream decoderInputStream = null;
	
	if(codec.id.equals("crs") || codec.id.equals("lrc")) {
		decoderInputStream = new CRSDecoderInputStream(
				reporter, limit, blockSize, errorOffset, srcFs, srcFile,
				parityFs, parityFile);
	} else {
		decoderInputStream = new DecoderInputStream(
				reporter, limit, blockSize, errorOffset, srcFs, srcFile,
				parityFs, parityFile);
	}
	
	/*
	decoderInputStream = new DecoderInputStream(
			reporter, limit, blockSize, errorOffset, srcFs, srcFile,
			parityFs, parityFile);
	*/
	return decoderInputStream;
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testMap() throws Exception {
    TenantAndIdEmittableKey key = new TenantAndIdEmittableKey();
    ValueMapper m = new MockValueMapper();
    BSONObject entry = new BasicBSONObject("found", "data");
    BSONWritable entity = new BSONWritable(entry);

    Context context = Mockito.mock(Context.class);
    PowerMockito.when(context, "write", Matchers.any(EmittableKey.class),
        Matchers.any(BSONObject.class)).thenAnswer(new Answer<BSONObject>() {

        @Override
        public BSONObject answer(InvocationOnMock invocation) throws Throwable {

            Object[] args = invocation.getArguments();

            assertNotNull(args);
            assertEquals(args.length, 2);

            assertTrue(args[0] instanceof TenantAndIdEmittableKey);
            assertTrue(args[1] instanceof ContentSummary);

            TenantAndIdEmittableKey id = (TenantAndIdEmittableKey) args[0];
            assertNotNull(id);

            ContentSummary e = (ContentSummary) args[1];
            assertEquals(e.getLength(), 1);
            assertEquals(e.getFileCount(), 2);
            assertEquals(e.getDirectoryCount(), 3);

            return null;
        }
    });

    m.map(key, entity, context);
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testMapValueNotFound() throws Exception {
    TenantAndIdEmittableKey key = new TenantAndIdEmittableKey();
    ValueMapper m = new MockValueMapper();
    BSONObject entry = new BasicBSONObject("not_found", "data");
    BSONWritable entity = new BSONWritable(entry);

    Context context = Mockito.mock(Context.class);
    PowerMockito.when(context, "write", Matchers.any(TenantAndIdEmittableKey.class),
        Matchers.any(BSONObject.class)).thenAnswer(new Answer<BSONObject>() {

        @Override
        public BSONObject answer(InvocationOnMock invocation) throws Throwable {

            Object[] args = invocation.getArguments();

            assertNotNull(args);
            assertEquals(args.length, 2);

            assertTrue(args[0] instanceof TenantAndIdEmittableKey);
            assertTrue(args[1] instanceof NullWritable);

            return null;
        }
    });

    m.map(key, entity, context);
}