org.apache.avro.io.ResolvingDecoder Java Examples

/**
 * Called to read a record instance. Overridden to read a pig tuple.
 */
@Override
protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException {

    // find out the order in which we will receive fields from the ResolvingDecoder
    Field[] readOrderedFields = in.readFieldOrder();

    /* create an empty tuple */
    Tuple tuple = TupleFactory.getInstance().newTuple(readOrderedFields.length);

    /* read fields and put in output order in tuple
     * The ResolvingDecoder figures out the writer schema to reader schema mapping for us
     */
    for (Field f : readOrderedFields) {
        tuple.set(f.pos(), read(old, f.schema(), in));
    }

    return tuple;
}
 

/**
 * Called to read a record instance. Overridden to read a pig tuple.
 */
@Override
protected Object readRecord(Object old, Schema expected, ResolvingDecoder in) throws IOException {

    // find out the order in which we will receive fields from the ResolvingDecoder
    Field[] readOrderedFields = in.readFieldOrder();

    /* create an empty tuple */
    Tuple tuple = TupleFactory.getInstance().newTuple(readOrderedFields.length);

    /* read fields and put in output order in tuple
     * The ResolvingDecoder figures out the writer schema to reader schema mapping for us
     */
    for (Field f : readOrderedFields) {
        tuple.set(f.pos(), read(old, f.schema(), in));
    }

    return tuple;
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

public ReadAvroContainer(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {   
  super(builder, config, parent, child, context);

  String schemaString = getConfigs().getString(config, "readerSchemaString", null);
  if (schemaString != null) {
    this.readerSchema = new Parser().parse(schemaString);
  } else {        
    String schemaFile = getConfigs().getString(config, "readerSchemaFile", null);
    if (schemaFile != null) {
      try { 
        this.readerSchema = new Parser().parse(new File(schemaFile));
      } catch (IOException e) {
        throw new MorphlineCompilationException("Cannot parse external Avro reader schema file: " + schemaFile, config, e);
      }
    } else {
      this.readerSchema = null;
    }
  }
  
  if (getClass() == ReadAvroContainer.class) {
    resolverCache = new BoundedLRUHashMap<ByteArrayKey, ResolvingDecoder>(
        getConfigs().getInt(config, "schemaCacheCapacity", 100));
    
    validateArguments();
  } else {
    resolverCache = null;
  }
}
 

/**
 * Called to read a map instance. Overridden to read a pig map.
 */
protected Object readMap(Object old, Schema expected, ResolvingDecoder in) throws IOException {
    Schema eValue = expected.getValueType();
    long l = in.readMapStart();
    Object map = newMap(old, (int) l);
    if (l > 0) {
        do {
            for (int i = 0; i < l; i++) {
                addToMap(map, readString(null, AvroStorageUtils.StringSchema, in),
                                  read(null, eValue, in));
            }
        } while ((l = in.mapNext()) > 0);
    }
    return map;
}
 

/**
 * Called to read a map instance. Overridden to read a pig map.
 */
protected Object readMap(Object old, Schema expected, ResolvingDecoder in) throws IOException {
    Schema eValue = expected.getValueType();
    long l = in.readMapStart();
    Object map = newMap(old, (int) l);
    if (l > 0) {
        do {
            for (int i = 0; i < l; i++) {
                addToMap(map, readString(null, AvroStorageUtils.StringSchema, in),
                                  read(null, eValue, in));
            }
        } while ((l = in.mapNext()) > 0);
    }
    return map;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

@Override
public InternalRow read(InternalRow reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  InternalRow row = reader.read(resolver, reuse);
  resolver.drain();
  return row;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

private ResolvingDecoder resolve(Decoder decoder) throws IOException {
  Map<Schema, Map<Schema, ResolvingDecoder>> cache = DECODER_CACHES.get();
  Map<Schema, ResolvingDecoder> fileSchemaToResolver = cache
      .computeIfAbsent(readSchema, k -> new HashMap<>());

  ResolvingDecoder resolver = fileSchemaToResolver.get(fileSchema);
  if (resolver == null) {
    resolver = newResolver();
    fileSchemaToResolver.put(fileSchema, resolver);
  }

  resolver.configure(decoder);

  return resolver;
}
 

@Override
public T read(T reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  T value = reader.read(resolver, reuse);
  resolver.drain();
  return value;
}
 

private ResolvingDecoder newResolver() {
  try {
    return DecoderFactory.get().resolvingDecoder(fileSchema, readSchema, null);
  } catch (IOException e) {
    throw new RuntimeIOException(e);
  }
}
 

@Override
public InternalRow read(InternalRow reuse, Decoder decoder) throws IOException {
  ResolvingDecoder resolver = resolve(decoder);
  InternalRow row = reader.read(resolver, reuse);
  resolver.drain();
  return row;
}
 

protected ResolvingDecoder createResolver(Schema writerSchema, Schema readerSchema) throws IOException {
  return DecoderFactory.get().resolvingDecoder(
      Schema.applyAliases(writerSchema, readerSchema), readerSchema, null);
}
 

/** Call this whenever appropriate */
public void setResolver(ResolvingDecoder resolver) {
  this.resolver = resolver;
}