org.apache.hadoop.util.GenericsUtil Java Examples

The following examples show how to use org.apache.hadoop.util.GenericsUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Chain.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
Example #2
Source File: DefaultStringifier.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * Restores the array of objects from the configuration.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use
 * @param keyName the name of the key to use
 * @param itemClass the class of the item
 * @return restored object
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.
 */
public static <K> K[] loadArray(Configuration conf, String keyName,
    Class<K> itemClass) throws IOException {
  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      itemClass);
  try {
    String itemStr = conf.get(keyName);
    ArrayList<K> list = new ArrayList<K>();
    String[] parts = itemStr.split(SEPARATOR);

    for (String part : parts) {
      if (!part.equals(""))
        list.add(stringifier.fromString(part));
    }

    return GenericsUtil.toArray(itemClass, list);
  }
  finally {
    stringifier.close();
  }
}
Example #3
Source File: DefaultStringifier.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * Stores the array of items in the configuration with the given keyName.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use 
 * @param items the objects to be stored
 * @param keyName the name of the key to use
 * @throws IndexOutOfBoundsException if the items array is empty
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.         
 */
public static <K> void storeArray(Configuration conf, K[] items,
    String keyName) throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
      GenericsUtil.getClass(items[0]));
  try {
    StringBuilder builder = new StringBuilder();
    for (K item : items) {
      builder.append(stringifier.toString(item)).append(SEPARATOR);
    }
    conf.set(keyName, builder.toString());
  }
  finally {
    stringifier.close();
  }
}
Example #4
Source File: TestWritableSerialization.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
/**
 * A utility that tests serialization/deserialization. 
 * @param <K> the class of the item
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static<K> K testSerialization(Configuration conf, K before) 
  throws Exception {
  
  SerializationFactory factory = new SerializationFactory(conf);
  Serializer<K> serializer 
    = factory.getSerializer(GenericsUtil.getClass(before));
  Deserializer<K> deserializer 
    = factory.getDeserializer(GenericsUtil.getClass(before));
 
  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(before);
  serializer.close();
  
  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  
  assertEquals(before, after);
  return after;
}
Example #5
Source File: TestWritableJobConf.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
private <K> K serDeser(K conf) throws Exception {
  SerializationFactory factory = new SerializationFactory(CONF);
  Serializer<K> serializer =
    factory.getSerializer(GenericsUtil.getClass(conf));
  Deserializer<K> deserializer =
    factory.getDeserializer(GenericsUtil.getClass(conf));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(conf);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #6
Source File: Chain.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
Example #7
Source File: DefaultStringifier.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * Restores the array of objects from the configuration.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use
 * @param keyName the name of the key to use
 * @param itemClass the class of the item
 * @return restored object
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.
 */
public static <K> K[] loadArray(Configuration conf, String keyName,
    Class<K> itemClass) throws IOException {
  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      itemClass);
  try {
    String itemStr = conf.get(keyName);
    ArrayList<K> list = new ArrayList<K>();
    String[] parts = itemStr.split(SEPARATOR);

    for (String part : parts) {
      if (!part.equals(""))
        list.add(stringifier.fromString(part));
    }

    return GenericsUtil.toArray(itemClass, list);
  }
  finally {
    stringifier.close();
  }
}
Example #8
Source File: DefaultStringifier.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * Stores the array of items in the configuration with the given keyName.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use 
 * @param items the objects to be stored
 * @param keyName the name of the key to use
 * @throws IndexOutOfBoundsException if the items array is empty
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.         
 */
public static <K> void storeArray(Configuration conf, K[] items,
    String keyName) throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
      GenericsUtil.getClass(items[0]));
  try {
    StringBuilder builder = new StringBuilder();
    for (K item : items) {
      builder.append(stringifier.toString(item)).append(SEPARATOR);
    }
    conf.set(keyName, builder.toString());
  }
  finally {
    stringifier.close();
  }
}
Example #9
Source File: TestWritableSerialization.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
/**
 * A utility that tests serialization/deserialization. 
 * @param <K> the class of the item
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static<K> K testSerialization(Configuration conf, K before) 
  throws Exception {
  
  SerializationFactory factory = new SerializationFactory(conf);
  Serializer<K> serializer 
    = factory.getSerializer(GenericsUtil.getClass(before));
  Deserializer<K> deserializer 
    = factory.getDeserializer(GenericsUtil.getClass(before));
 
  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(before);
  serializer.close();
  
  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  
  assertEquals(before, after);
  return after;
}
Example #10
Source File: TestWritableJobConf.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
private <K> K serDeser(K conf) throws Exception {
  SerializationFactory factory = new SerializationFactory(CONF);
  Serializer<K> serializer =
    factory.getSerializer(GenericsUtil.getClass(conf));
  Deserializer<K> deserializer =
    factory.getDeserializer(GenericsUtil.getClass(conf));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(conf);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #11
Source File: TestWritableJobConf.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private <K> K serDeser(K conf) throws Exception {
  SerializationFactory factory = new SerializationFactory(CONF);
  Serializer<K> serializer =
    factory.getSerializer(GenericsUtil.getClass(conf));
  Deserializer<K> deserializer =
    factory.getDeserializer(GenericsUtil.getClass(conf));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(conf);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #12
Source File: SerializationTestUtil.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * A utility that tests serialization/deserialization. 
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization
 * @param <K> the class of the item
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static <K> K testSerialization(Configuration conf, K before)
	throws Exception {

  SerializationFactory factory = new SerializationFactory(conf);
  Serializer<K> serializer 
    = factory.getSerializer(GenericsUtil.getClass(before));
  Deserializer<K> deserializer 
    = factory.getDeserializer(GenericsUtil.getClass(before));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(before);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #13
Source File: DefaultStringifier.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Restores the array of objects from the configuration.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use
 * @param keyName the name of the key to use
 * @param itemClass the class of the item
 * @return restored object
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.
 */
public static <K> K[] loadArray(Configuration conf, String keyName,
    Class<K> itemClass) throws IOException {
  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      itemClass);
  try {
    String itemStr = conf.get(keyName);
    ArrayList<K> list = new ArrayList<K>();
    String[] parts = itemStr.split(SEPARATOR);

    for (String part : parts) {
      if (!part.isEmpty())
        list.add(stringifier.fromString(part));
    }

    return GenericsUtil.toArray(itemClass, list);
  }
  finally {
    stringifier.close();
  }
}
Example #14
Source File: DefaultStringifier.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Stores the array of items in the configuration with the given keyName.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use 
 * @param items the objects to be stored
 * @param keyName the name of the key to use
 * @throws IndexOutOfBoundsException if the items array is empty
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.         
 */
public static <K> void storeArray(Configuration conf, K[] items,
    String keyName) throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
      GenericsUtil.getClass(items[0]));
  try {
    StringBuilder builder = new StringBuilder();
    for (K item : items) {
      builder.append(stringifier.toString(item)).append(SEPARATOR);
    }
    conf.set(keyName, builder.toString());
  }
  finally {
    stringifier.close();
  }
}
Example #15
Source File: Chain.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
Example #16
Source File: TestWritableJobConf.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private <K> K serDeser(K conf) throws Exception {
  SerializationFactory factory = new SerializationFactory(CONF);
  Serializer<K> serializer =
    factory.getSerializer(GenericsUtil.getClass(conf));
  Deserializer<K> deserializer =
    factory.getDeserializer(GenericsUtil.getClass(conf));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(conf);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #17
Source File: SerializationTestUtil.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * A utility that tests serialization/deserialization. 
 * @param conf configuration to use, "io.serializations" is read to 
 * determine the serialization
 * @param <K> the class of the item
 * @param before item to (de)serialize
 * @return deserialized item
 */
public static <K> K testSerialization(Configuration conf, K before)
	throws Exception {

  SerializationFactory factory = new SerializationFactory(conf);
  Serializer<K> serializer 
    = factory.getSerializer(GenericsUtil.getClass(before));
  Deserializer<K> deserializer 
    = factory.getDeserializer(GenericsUtil.getClass(before));

  DataOutputBuffer out = new DataOutputBuffer();
  serializer.open(out);
  serializer.serialize(before);
  serializer.close();

  DataInputBuffer in = new DataInputBuffer();
  in.reset(out.getData(), out.getLength());
  deserializer.open(in);
  K after = deserializer.deserialize(null);
  deserializer.close();
  return after;
}
Example #18
Source File: DefaultStringifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Restores the array of objects from the configuration.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use
 * @param keyName the name of the key to use
 * @param itemClass the class of the item
 * @return restored object
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.
 */
public static <K> K[] loadArray(Configuration conf, String keyName,
    Class<K> itemClass) throws IOException {
  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      itemClass);
  try {
    String itemStr = conf.get(keyName);
    ArrayList<K> list = new ArrayList<K>();
    String[] parts = itemStr.split(SEPARATOR);

    for (String part : parts) {
      if (!part.isEmpty())
        list.add(stringifier.fromString(part));
    }

    return GenericsUtil.toArray(itemClass, list);
  }
  finally {
    stringifier.close();
  }
}
Example #19
Source File: DefaultStringifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Stores the array of items in the configuration with the given keyName.
 * 
 * @param <K> the class of the item
 * @param conf the configuration to use 
 * @param items the objects to be stored
 * @param keyName the name of the key to use
 * @throws IndexOutOfBoundsException if the items array is empty
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes.         
 */
public static <K> void storeArray(Configuration conf, K[] items,
    String keyName) throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf, 
      GenericsUtil.getClass(items[0]));
  try {
    StringBuilder builder = new StringBuilder();
    for (K item : items) {
      builder.append(stringifier.toString(item)).append(SEPARATOR);
    }
    conf.set(keyName, builder.toString());
  }
  finally {
    stringifier.close();
  }
}
Example #20
Source File: Chain.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private <E> E makeCopyForPassByValue(Serialization<E> serialization,
                                      E obj) throws IOException {
  Serializer<E> ser =
    serialization.getSerializer(GenericsUtil.getClass(obj));
  Deserializer<E> deser =
    serialization.getDeserializer(GenericsUtil.getClass(obj));

  DataOutputBuffer dof = threadLocalDataOutputBuffer.get();

  dof.reset();
  ser.open(dof);
  ser.serialize(obj);
  ser.close();
  obj = ReflectionUtils.newInstance(GenericsUtil.getClass(obj),
                                    getChainJobConf());
  ByteArrayInputStream bais =
    new ByteArrayInputStream(dof.getData(), 0, dof.getLength());
  deser.open(bais);
  deser.deserialize(obj);
  deser.close();
  return obj;
}
Example #21
Source File: DefaultStringifier.java    From big-c with Apache License 2.0 3 votes vote down vote up 
/**
 * Stores the item in the configuration with the given keyName.
 * 
 * @param <K>  the class of the item
 * @param conf the configuration to store
 * @param item the object to be stored
 * @param keyName the name of the key to use
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes. 
 */
public static <K> void store(Configuration conf, K item, String keyName)
throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      GenericsUtil.getClass(item));
  conf.set(keyName, stringifier.toString(item));
  stringifier.close();
}
Example #22
Source File: DefaultStringifier.java    From RDFS with Apache License 2.0 3 votes vote down vote up 
/**
 * Stores the item in the configuration with the given keyName.
 * 
 * @param <K>  the class of the item
 * @param conf the configuration to store
 * @param item the object to be stored
 * @param keyName the name of the key to use
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes. 
 */
public static <K> void store(Configuration conf, K item, String keyName)
throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      GenericsUtil.getClass(item));
  conf.set(keyName, stringifier.toString(item));
  stringifier.close();
}
Example #23
Source File: DefaultStringifier.java    From hadoop-gpu with Apache License 2.0 3 votes vote down vote up 
/**
 * Stores the item in the configuration with the given keyName.
 * 
 * @param <K>  the class of the item
 * @param conf the configuration to store
 * @param item the object to be stored
 * @param keyName the name of the key to use
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes. 
 */
public static <K> void store(Configuration conf, K item, String keyName)
throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      GenericsUtil.getClass(item));
  conf.set(keyName, stringifier.toString(item));
  stringifier.close();
}
Example #24
Source File: DefaultStringifier.java    From hadoop with Apache License 2.0 3 votes vote down vote up 
/**
 * Stores the item in the configuration with the given keyName.
 * 
 * @param <K>  the class of the item
 * @param conf the configuration to store
 * @param item the object to be stored
 * @param keyName the name of the key to use
 * @throws IOException : forwards Exceptions from the underlying 
 * {@link Serialization} classes. 
 */
public static <K> void store(Configuration conf, K item, String keyName)
throws IOException {

  DefaultStringifier<K> stringifier = new DefaultStringifier<K>(conf,
      GenericsUtil.getClass(item));
  conf.set(keyName, stringifier.toString(item));
  stringifier.close();
}