Java Code Examples for org.apache.hadoop.io.MapWritable#put()

@Override
protected void reduce(StatsUserDimension key, Iterable<TimeOutputValue> values, Context context)
    throws IOException, InterruptedException {
  this.unique.clear();
  //开始计算uuid的个数
  for (TimeOutputValue value : values) {
    this.unique.add(value.getId());
  }

  MapWritable map = new MapWritable();
  map.put(new IntWritable(-1), new IntWritable(this.unique.size()));

  //设置kpi名称
  String kpiName = key.getStatsCommon().getKpi().getKpiName();
  if (KpiType.NEW_INSTALL_USER.name.equals(kpiName)) {
    //计算stats_user表中的新增用户
    outputValue.setKpi(KpiType.NEW_INSTALL_USER);

  } else if (KpiType.BROWSER_NEW_INSTALL_USER.name.equals(kpiName)) {
    //计算stats_device_browser的新增用户
    outputValue.setKpi(KpiType.BROWSER_NEW_INSTALL_USER);
  }
  outputValue.setValue(map);
  context.write(key, outputValue);
}
 

@Override
protected void reduce(StatsUserDimension key, Iterable<TimeOutputValue> values, Context context) throws IOException, InterruptedException {
    this.unique.clear();

    // 开始计算uuid的个数
    for (TimeOutputValue value : values) {
        this.unique.add(value.getId());//uid,用户ID
    }
    
    MapWritable map = new MapWritable();//相当于java中HashMap
    map.put(new IntWritable(-1), new IntWritable(this.unique.size()));
    outputValue.setValue(map);

    // 设置kpi名称
    String kpiName = key.getStatsCommon().getKpi().getKpiName();
    if (KpiType.NEW_INSTALL_USER.name.equals(kpiName)) {
        // 计算stats_user表中的新增用户
        outputValue.setKpi(KpiType.NEW_INSTALL_USER);
    } else if (KpiType.BROWSER_NEW_INSTALL_USER.name.equals(kpiName)) {
        // 计算stats_device_browser表中的新增用户
        outputValue.setKpi(KpiType.BROWSER_NEW_INSTALL_USER);
    }
    context.write(key, outputValue);
}
 

@Override
protected void reduce(StatsUserDimension key, Iterable<TimeOutputValue> values, Context context) throws IOException, InterruptedException {
    this.unique.clear();

    // 开始计算uuid的个数
    for (TimeOutputValue value : values) {
        this.unique.add(value.getId());
    }
    MapWritable map = new MapWritable();
    map.put(new IntWritable(-1), new IntWritable(this.unique.size()));
    outputValue.setValue(map);

    // 设置kpi名称
    String kpiName = key.getStatsCommon().getKpi().getKpiName();
    if (KpiType.NEW_INSTALL_USER.name.equals(kpiName)) {
        // 计算stats_user表中的新增用户
        outputValue.setKpi(KpiType.NEW_INSTALL_USER);
    } else if (KpiType.BROWSER_NEW_INSTALL_USER.name.equals(kpiName)) {
        // 计算stats_device_browser表中的新增用户
        outputValue.setKpi(KpiType.BROWSER_NEW_INSTALL_USER);
    }
    context.write(key, outputValue);
}
 

private void configureGenericRecordExportInputFormat(Job job, String tableName)
    throws IOException {
  ConnManager connManager = context.getConnManager();
  Map<String, Integer> columnTypeInts;
  if (options.getCall() == null) {
    columnTypeInts = connManager.getColumnTypes(
        tableName,
        options.getSqlQuery());
  } else {
    columnTypeInts = connManager.getColumnTypesForProcedure(
        options.getCall());
  }
  String[] specifiedColumns = options.getColumns();
  MapWritable columnTypes = new MapWritable();
  for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
    String column = e.getKey();
    column = (specifiedColumns == null) ? column : options.getColumnNameCaseInsensitive(column);
    if (column != null) {
      Text columnName = new Text(column);
      Text columnType = new Text(connManager.toJavaType(tableName, column, e.getValue()));
      columnTypes.put(columnName, columnType);
    }
  }
  DefaultStringifier.store(job.getConfiguration(), columnTypes,
      AvroExportMapper.AVRO_COLUMN_TYPES_MAP);
}
 

private void populateMap(SortedMap<ByteBuffer, IColumn> cvalue, MapWritable value)
{
  for (Map.Entry<ByteBuffer, IColumn> e : cvalue.entrySet())
  {
    ByteBuffer k = e.getKey();
    IColumn    v = e.getValue();

    if (!v.isLive()) {
      continue;
    }

    BytesWritable newKey   = convertByteBuffer(k);
    BytesWritable newValue = convertByteBuffer(v.value());

    value.put(newKey, newValue);
  }
}
 

@Test
public void generateEventWritable() throws Exception {
    MapWritable document = new MapWritable();
    document.put(new Text("field"), new Text("value"));

    SerializationEventConverter eventConverter = new SerializationEventConverter();

    SerializationFailure iaeFailure = new SerializationFailure(new IllegalArgumentException("garbage"), document, new ArrayList<String>());

    String rawEvent = eventConverter.getRawEvent(iaeFailure);
    assertThat(rawEvent, Matchers.startsWith("org.apache.hadoop.io.MapWritable@"));
    String timestamp = eventConverter.getTimestamp(iaeFailure);
    assertTrue(StringUtils.hasText(timestamp));
    assertTrue(DateUtils.parseDate(timestamp).getTime().getTime() > 1L);
    String exceptionType = eventConverter.renderExceptionType(iaeFailure);
    assertEquals("illegal_argument_exception", exceptionType);
    String exceptionMessage = eventConverter.renderExceptionMessage(iaeFailure);
    assertEquals("garbage", exceptionMessage);
    String eventMessage = eventConverter.renderEventMessage(iaeFailure);
    assertEquals("Could not construct bulk entry from record", eventMessage);
}
 

/**
 * Converts a Map of Strings into a Writable and writes it.
 *
 * @param map
 * @param output
 * @throws IOException
 */
public static void writeMap(Map<String,String> map, DataOutput output) throws IOException {
    MapWritable mw = new MapWritable();
    
    for (Map.Entry<String,String> entry : map.entrySet()) {
        mw.put(new Text(entry.getKey()), new Text(entry.getValue()));
    }
    
    mw.write(output);
}
 

@Test
public void testMapFieldExtractorNested() throws Exception {
    ConstantFieldExtractor cfe = new MapWritableFieldExtractor();
    Map<Writable, Writable> m = new MapWritable();
    MapWritable nested = new MapWritable();
    nested.put(new Text("bar"), new Text("found"));
    m.put(new Text("foo"), nested);
    assertEquals(new Text("found"), extract(cfe, "foo.bar", m));
}
 

public static Writable toWritable(Object o) throws IOException {
  if (o instanceof Long) {
    return new LongWritable(((Long) o).longValue());
  } else if (o instanceof String) {
    return new Text(o.toString());
  } else if (o instanceof byte[]) {
    return new BytesWritable((byte[]) o);
  } else if (o instanceof Integer) {
    return new IntWritable(((Integer) o).intValue());
  } else if (o instanceof Short) {
    return new ShortWritable(((Short) o).shortValue());
  } else if (o instanceof Byte) {
    return new ByteWritable(((Byte) o).byteValue());
  } else if (o instanceof Double) {
    return new DoubleWritable(((Double) o).doubleValue());
  } else if (o instanceof Float) {
    return new FloatWritable(((Float) o).floatValue());
  } else if (o instanceof Boolean) {
    return new BooleanWritable(((Boolean) o).booleanValue());
  } else if (o instanceof List) {
    Writable[] a = new Writable[((List) o).size()];
    for (int i = 0; i < a.length; i++) {
      a[i] = new ObjectWritable(toWritable(((List) o).get(i)));
    }
    return new ArrayWritable(ObjectWritable.class, a);
  } else if (o instanceof Map) {
    MapWritable map = new MapWritable();
    for (Entry<Object,Object> entry: ((Map<Object,Object>) o).entrySet()) {
      map.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
    }
    return map;
  } else if (null == o) {
    return NullWritable.get();
  } else {
    ObjectWritable ow = new ObjectWritable();
    ow.set(o);
    return ow;
  }// else {
  //  throw new IOException("Unsupported type " + o.getClass());
  //}
}