org.apache.hadoop.io.ShortWritable Java Examples

/**
 * All Writable objects passed in here are guaranteed to be primitive writable objects.
 */
private boolean objCastHelper(Object javaObj, Writable obj) {
  if (obj instanceof IntWritable) {
    return ((IntWritable) obj).get() == (Integer) javaObj;
  } else if (obj instanceof Text) {
    return (obj).toString().equals(javaObj);
  } else if (obj instanceof LongWritable) {
    return ((LongWritable) obj).get() == (Long) javaObj;
  } else if (obj instanceof ShortWritable) {
    return ((ShortWritable) obj).get() == (Short) javaObj;
  } else if (obj instanceof DoubleWritable) {
    return ((DoubleWritable) obj).get() == (Double) javaObj;
  } else {
    throw new RuntimeException("Cannot recognize the writable type, please enrich the castHelper function");
  }
}
 

private void setupMapper(String intermediateTable) throws IOException {
//        FileInputFormat.setInputPaths(job, input);

        String[] dbTableNames = HadoopUtil.parseHiveTableName(intermediateTable);
        HCatInputFormat.setInput(job, dbTableNames[0],
                dbTableNames[1]);
        
        job.setInputFormatClass(HCatInputFormat.class);
        job.setMapperClass(FactDistinctColumnsMapper.class);
        job.setCombinerClass(FactDistinctColumnsCombiner.class);
        job.setMapOutputKeyClass(ShortWritable.class);
        job.setMapOutputValueClass(Text.class);
    }
 

@Override
public void write(DataOutput out) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.write(out);

    // 1. num, int1, int2
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.set(num[i]);
        intw.write(out);
    }

    intw.set(int1);
    intw.write(out);

    intw.set(int2);
    intw.write(out);

    // 2. st1
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.set(strings[i]);
        txt.write(out);
    }

    txt.set(st1);
    txt.write(out);

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.set(dubs[i]);
        dw.write(out);
    }

    dw.set(db);
    dw.write(out);

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.set(fts[i]);
        fw.write(out);
    }

    fw.set(ft);
    fw.write(out);

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.set(lngs[i]);
        lw.write(out);
    }
    lw.set(lng);
    lw.write(out);

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.set(bools[i]);
        bw.write(out);
    }
    bw.set(bool);
    bw.write(out);

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.set(shrts[i]);
        sw.write(out);
    }
    sw.set(shrt);
    sw.write(out);

    // 8. bytes
    // BytesWritable btsw = new BytesWritable(bts);
    // btsw.write(out);
    BytesWritable btsw = new BytesWritable();
    btsw.setCapacity(bts.length);
    btsw.setSize(bts.length);
    btsw.set(bts, 0, bts.length);
    btsw.write(out);
}
 

@Override
public void readFields(DataInput in) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.readFields(in);
    tms = tms_text.toString();

    // 1. integers
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.readFields(in);
        num[i] = intw.get();
    }

    intw.readFields(in);
    int1 = intw.get();

    intw.readFields(in);
    int2 = intw.get();

    // 2. strings
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.readFields(in);
        strings[i] = txt.toString();
    }

    txt.readFields(in);
    st1 = txt.toString();

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.readFields(in);
        dubs[i] = dw.get();
    }

    dw.readFields(in);
    db = dw.get();

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.readFields(in);
        fts[i] = fw.get();
    }

    fw.readFields(in);
    ft = fw.get();

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.readFields(in);
        lngs[i] = lw.get();
    }

    lw.readFields(in);
    lng = lw.get();

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.readFields(in);
        bools[i] = bw.get();
    }

    bw.readFields(in);
    bool = bw.get();

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.readFields(in);
        shrts[i] = sw.get();
    }
    sw.readFields(in);
    shrt = sw.get();

    // 8. bytes
    BytesWritable btsw = new BytesWritable();
    btsw.readFields(in);
    byte[] buffer = btsw.getBytes();
    bts = new byte[btsw.getLength()];
    for (int i = 0; i < btsw.getLength(); i++) {
        bts[i] = buffer[i];
    }
}
 

/** read short value */
static short readShort(DataInput in) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.readFields(in);
  return uShort.get();
}
 

/** write short value */
static void writeShort(short value, DataOutputStream out) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.set(value);
  uShort.write(out);
}
 

/** read short value */
static short readShort(DataInput in) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.readFields(in);
  return uShort.get();
}
 

/** write short value */
static void writeShort(short value, DataOutputStream out) throws IOException {
  ShortWritable uShort = TL_DATA.get().U_SHORT;
  uShort.set(value);
  uShort.write(out);
}
 

public static Writable toWritable(Object o) throws IOException {
  if (o instanceof Long) {
    return new LongWritable(((Long) o).longValue());
  } else if (o instanceof String) {
    return new Text(o.toString());
  } else if (o instanceof byte[]) {
    return new BytesWritable((byte[]) o);
  } else if (o instanceof Integer) {
    return new IntWritable(((Integer) o).intValue());
  } else if (o instanceof Short) {
    return new ShortWritable(((Short) o).shortValue());
  } else if (o instanceof Byte) {
    return new ByteWritable(((Byte) o).byteValue());
  } else if (o instanceof Double) {
    return new DoubleWritable(((Double) o).doubleValue());
  } else if (o instanceof Float) {
    return new FloatWritable(((Float) o).floatValue());
  } else if (o instanceof Boolean) {
    return new BooleanWritable(((Boolean) o).booleanValue());
  } else if (o instanceof List) {
    Writable[] a = new Writable[((List) o).size()];
    for (int i = 0; i < a.length; i++) {
      a[i] = new ObjectWritable(toWritable(((List) o).get(i)));
    }
    return new ArrayWritable(ObjectWritable.class, a);
  } else if (o instanceof Map) {
    MapWritable map = new MapWritable();
    for (Entry<Object,Object> entry: ((Map<Object,Object>) o).entrySet()) {
      map.put(toWritable(entry.getKey()), toWritable(entry.getValue()));
    }
    return map;
  } else if (null == o) {
    return NullWritable.get();
  } else {
    ObjectWritable ow = new ObjectWritable();
    ow.set(o);
    return ow;
  }// else {
  //  throw new IOException("Unsupported type " + o.getClass());
  //}
}
 

/**
 * Copy the value of {@param from} object into {@param to} with supporting of type-widening that ORC allowed.
 */
public static void handlePrimitiveWritableComparable(WritableComparable from, WritableComparable to) {
  if (from instanceof ByteWritable) {
    if (to instanceof ByteWritable) {
      ((ByteWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ByteWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ByteWritable) from).get());
      return;
    }
  } else if (from instanceof ShortWritable) {
    if (to instanceof ShortWritable) {
      ((ShortWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof IntWritable) {
      ((IntWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((ShortWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((ShortWritable) from).get());
      return;
    }
  } else if (from instanceof IntWritable) {
    if (to instanceof IntWritable) {
      ((IntWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof LongWritable) {
      ((LongWritable) to).set(((IntWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((IntWritable) from).get());
      return;
    }
  } else if (from instanceof LongWritable) {
    if (to instanceof LongWritable) {
      ((LongWritable) to).set(((LongWritable) from).get());
      return;
    } else if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((LongWritable) from).get());
      return;
    }
    // Following from this branch, type-widening is not allowed and only value-copy will happen.
  } else if (from instanceof DoubleWritable) {
    if (to instanceof DoubleWritable) {
      ((DoubleWritable) to).set(((DoubleWritable) from).get());
      return;
    }
  } else if (from instanceof BytesWritable) {
    if (to instanceof BytesWritable) {
      ((BytesWritable) to).set((BytesWritable) from);
      return;
    }
  } else if (from instanceof FloatWritable) {
    if (to instanceof FloatWritable) {
      ((FloatWritable) to).set(((FloatWritable) from).get());
      return;
    }
  } else if (from instanceof Text) {
    if (to instanceof Text) {
      ((Text) to).set((Text) from);
      return;
    }
  } else if (from instanceof DateWritable) {
    if (to instanceof DateWritable) {
      ((DateWritable) to).set(((DateWritable) from).get());
      return;
    }
  } else if (from instanceof OrcTimestamp && to instanceof OrcTimestamp) {
    ((OrcTimestamp) to).set(((OrcTimestamp) from).toString());
    return;
  } else if (from instanceof HiveDecimalWritable && to instanceof HiveDecimalWritable) {
    ((HiveDecimalWritable) to).set(((HiveDecimalWritable) from).getHiveDecimal());
    return;
  } else if (from instanceof BooleanWritable && to instanceof BooleanWritable) {
    ((BooleanWritable) to).set(((BooleanWritable) from).get());
    return;
  }
  throw new UnsupportedOperationException(String
      .format("The conversion of primitive-type WritableComparable object from %s to %s is not supported",
          from.getClass(), to.getClass()));
}
 

/**
 * Fill in value in OrcStruct with given schema, assuming {@param w} contains the same schema as {@param schema}.
 * {@param schema} is still necessary to given given {@param w} do contains schema information itself, because the
 * actual value type is only available in {@link TypeDescription} but not {@link org.apache.orc.mapred.OrcValue}.
 *
 * For simplicity here are some assumptions:
 * - We only give 3 primitive values and use them to construct compound values. To make it work for different types that
 * can be widened or shrunk to each other, please use value within small range.
 * - For List, Map or Union, make sure there's at least one entry within the record-container.
 * you may want to try createValueRecursively(TypeDescription) instead of {@link OrcStruct#createValue(TypeDescription)}
 */
public static void fillOrcStructWithFixedValue(WritableComparable w, TypeDescription schema, int unionTag,
    int intValue, String stringValue, boolean booleanValue) {
  switch (schema.getCategory()) {
    case BOOLEAN:
      ((BooleanWritable) w).set(booleanValue);
      break;
    case BYTE:
      ((ByteWritable) w).set((byte) intValue);
      break;
    case SHORT:
      ((ShortWritable) w).set((short) intValue);
      break;
    case INT:
      ((IntWritable) w).set(intValue);
      break;
    case LONG:
      ((LongWritable) w).set(intValue);
      break;
    case FLOAT:
      ((FloatWritable) w).set(intValue * 1.0f);
      break;
    case DOUBLE:
      ((DoubleWritable) w).set(intValue * 1.0);
      break;
    case STRING:
    case CHAR:
    case VARCHAR:
      ((Text) w).set(stringValue);
      break;
    case BINARY:
      throw new UnsupportedOperationException("Binary type is not supported in random orc data filler");
    case DECIMAL:
      throw new UnsupportedOperationException("Decimal type is not supported in random orc data filler");
    case DATE:
    case TIMESTAMP:
    case TIMESTAMP_INSTANT:
      throw new UnsupportedOperationException(
          "Timestamp and its derived types is not supported in random orc data filler");
    case LIST:
      OrcList castedList = (OrcList) w;
      // Here it is not trivial to create typed-object in element-type. So this method expect the value container
      // to at least contain one element, or the traversing within the list will be skipped.
      for (Object i : castedList) {
        fillOrcStructWithFixedValue((WritableComparable) i, schema.getChildren().get(0), unionTag, intValue,
            stringValue, booleanValue);
      }
      break;
    case MAP:
      OrcMap castedMap = (OrcMap) w;
      for (Object entry : castedMap.entrySet()) {
        Map.Entry<WritableComparable, WritableComparable> castedEntry =
            (Map.Entry<WritableComparable, WritableComparable>) entry;
        fillOrcStructWithFixedValue(castedEntry.getKey(), schema.getChildren().get(0), unionTag, intValue,
            stringValue, booleanValue);
        fillOrcStructWithFixedValue(castedEntry.getValue(), schema.getChildren().get(1), unionTag, intValue,
            stringValue, booleanValue);
      }
      break;
    case STRUCT:
      OrcStruct castedStruct = (OrcStruct) w;
      int fieldIdx = 0;
      for (TypeDescription child : schema.getChildren()) {
        fillOrcStructWithFixedValue(castedStruct.getFieldValue(fieldIdx), child, unionTag, intValue, stringValue,
            booleanValue);
        fieldIdx += 1;
      }
      break;
    case UNION:
      OrcUnion castedUnion = (OrcUnion) w;
      TypeDescription targetMemberSchema = schema.getChildren().get(unionTag);
      castedUnion.set(unionTag, OrcUtils.createValueRecursively(targetMemberSchema));
      fillOrcStructWithFixedValue((WritableComparable) castedUnion.getObject(), targetMemberSchema, unionTag,
          intValue, stringValue, booleanValue);
      break;
    default:
      throw new IllegalArgumentException("Unknown type " + schema.toString());
  }
}
 

private void setupMapper() throws IOException {

        String tableName = job.getConfiguration().get(BatchConstants.TABLE_NAME);
        String[] dbTableNames = HadoopUtil.parseHiveTableName(tableName);

        log.info("setting hcat input format, db name {} , table name {}", dbTableNames[0],dbTableNames[1]);

        HCatInputFormat.setInput(job, dbTableNames[0], dbTableNames[1]);

        job.setInputFormatClass(HCatInputFormat.class);

        job.setMapperClass(IIDistinctColumnsMapper.class);
        job.setCombinerClass(IIDistinctColumnsCombiner.class);
        job.setMapOutputKeyClass(ShortWritable.class);
        job.setMapOutputValueClass(Text.class);
    }