org.embulk.spi.Column Java Examples

private void buildOutputInputColumnMap()
{
    for (Column outputColumn : outputSchema.getColumns()) {
        String name    = outputColumn.getName();
        String srcName = getSrc(name, task.getColumns());
        if (srcName == null) {
            srcName = getSrc(name, task.getAddColumns());
        }
        if (srcName == null) {
            srcName = name;
        }
        Column inputColumn;
        try {
            inputColumn = inputSchema.lookupColumn(srcName);
        }
        catch (SchemaConfigException ex) {
            inputColumn = null;
        }
        outputInputColumnMap.put(outputColumn, inputColumn); // NOTE: inputColumn would be null
    }
}
 

@Override
public void stringColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
            pageBuilder.setString(column, string);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
        }
    }
}
 

@Override
public void doubleColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
            pageBuilder.setDouble(column, doubleValue);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
        }
    }
}
 

@Override
public void longColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
            pageBuilder.setLong(column, longValue);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
        }
    }
}
 

@Override
public void booleanColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
            pageBuilder.setBoolean(column, booleanValue);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
        }
    }
}
 

@Override
public void timestampColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
            pageBuilder.setTimestamp(column, timestamp);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
        }
    }
}
 

@Override
public void transaction(ConfigSource configSource, Control control)
{
    PluginTask task = configSource.loadConfig(PluginTask.class);

    if (! task.getColumnOptions().isEmpty()) {
        log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true).");
    }

    SchemaConfig schemaConfig = getSchemaConfig(task);
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
        ColumnConfig columnConfig = schemaConfig.getColumn(i);
        Type type = getType(task, columnConfig);
        columns.add(new Column(i, columnConfig.getName(), type));
    }
    control.run(task.dump(), new Schema(columns.build()));
}
 

@Override
public void timestampColumn(Column outputColumn)
{
    Column inputColumn = outputInputColumnMap.get(outputColumn);
    if (inputColumn == null || pageReader.isNull(inputColumn)) {
        Timestamp defaultValue = (Timestamp) outputDefaultMap.get(outputColumn);
        if (defaultValue == null) {
            pageBuilder.setNull(outputColumn);
        }
        else {
            pageBuilder.setTimestamp(outputColumn, defaultValue);
        }
    }
    else {
        pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
    }
}
 

@Override
public void stringColumn(Column outputColumn)
{
    Column inputColumn = outputInputColumnMap.get(outputColumn);
    if (inputColumn == null || pageReader.isNull(inputColumn)) {
        String defaultValue = (String) outputDefaultMap.get(outputColumn);
        if (defaultValue == null) {
            pageBuilder.setNull(outputColumn);
        }
        else {
            pageBuilder.setString(outputColumn, defaultValue);
        }
    }
    else {
        pageBuilder.setString(outputColumn, pageReader.getString(inputColumn));
    }
}
 

@Override
public void doubleColumn(Column outputColumn)
{
    Column inputColumn = outputInputColumnMap.get(outputColumn);
    if (inputColumn == null || pageReader.isNull(inputColumn)) {
        Double defaultValue = (Double) outputDefaultMap.get(outputColumn);
        if (defaultValue == null) {
            pageBuilder.setNull(outputColumn);
        }
        else {
            pageBuilder.setDouble(outputColumn, defaultValue.doubleValue());
        }
    }
    else {
        pageBuilder.setDouble(outputColumn, pageReader.getDouble(inputColumn));
    }
}
 

@Override
public void longColumn(Column outputColumn)
{
    Column inputColumn = outputInputColumnMap.get(outputColumn);
    if (inputColumn == null || pageReader.isNull(inputColumn)) {
        Long defaultValue = (Long) outputDefaultMap.get(outputColumn);
        if (defaultValue == null) {
            pageBuilder.setNull(outputColumn);
        }
        else {
            pageBuilder.setLong(outputColumn, defaultValue.longValue());
        }
    }
    else {
        pageBuilder.setLong(outputColumn, pageReader.getLong(inputColumn));
    }
}
 

@Override
public void booleanColumn(Column outputColumn)
{
    Column inputColumn = outputInputColumnMap.get(outputColumn);
    if (inputColumn == null || pageReader.isNull(inputColumn)) {
        Boolean defaultValue = (Boolean) outputDefaultMap.get(outputColumn);
        if (defaultValue == null) {
            pageBuilder.setNull(outputColumn);
        }
        else {
            pageBuilder.setBoolean(outputColumn, defaultValue.booleanValue());
        }
    }
    else {
        pageBuilder.setBoolean(outputColumn, pageReader.getBoolean(inputColumn));
    }
}
 

@Override
public void timestampColumn(Column column)
{
    if (!record.isNull(column)) {
        Timestamp t = record.getTimestamp(column);
        String formatted = timestampFormatters[column.getIndex()].format(t);
        consumer.addBinary(Binary.fromString(formatted));
    }
}
 

@Override
public void stringColumn(Column column)
{
    if (!record.isNull(column)) {
        consumer.addBinary(Binary.fromString(record.getString(column)));
    }
}
 

@Override
public void doubleColumn(Column column)
{
    if (!record.isNull(column)) {
        consumer.addDouble(record.getDouble(column));
    }
}
 

@Override
public void longColumn(Column column)
{
    if (!record.isNull(column)) {
        consumer.addLong(record.getLong(column));
    }
}
 

@Override
public void booleanColumn(Column column)
{
    if (!record.isNull(column)) {
        consumer.addBoolean(record.getBoolean(column));
    }
}
 

@Override
public void write(PageReader record)
{
    final ColumnVisitor visitor = new ParquetColumnVisitor(record, consumer);
    consumer.startMessage();
    for (Column c : schema.getColumns()) {
        if (!record.isNull(c)) {
            consumer.startField(c.getName(), c.getIndex());
            c.visit(visitor);
            consumer.endField(c.getName(), c.getIndex());
        }
    }
    consumer.endMessage();
}
 

@Override
public void jsonColumn(Column column)
{
    if (isNil(value)) {
        pageBuilder.setNull(column);
    }
    else {
        try {
            pageBuilder.setJson(column, value);
        }
        catch (MessageTypeException e) {
            throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
        }
    }
}
 

private void buildOutputDefaultMap()
{
    for (Column outputColumn : outputSchema.getColumns()) {
        String name = outputColumn.getName();
        Type type = outputColumn.getType();

        Object defaultValue = getDefault(task, name, type, task.getColumns());
        if (defaultValue == null) {
            defaultValue = getDefault(task, name, type, task.getAddColumns());
        }
        if (defaultValue != null) {
            outputDefaultMap.put(outputColumn, defaultValue);
        }
    }
}
 

private void setColumnNameValues(Schema schema)
{
    ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
    for (Column column : schema.getColumns()) {
        String name = column.getName();
        builder.put(name, newString(name));
    }
    columnNameValues = builder.build();
}
 

@Test(expected = ConfigException.class)
public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
{
    ConfigSource config = configFromYamlString(
            "type: column",
            "columns:",
            "- {name: a}",
            "drop_columns:",
            "- {name: a}");
    Schema inputSchema = schema(
            new Column(0, "a", STRING),
            new Column(1, "b", STRING));

    transaction(config, inputSchema);
}
 

@Test
public void buildOutputSchema_AddColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
            "  - {name: string, type: string, default: string}",
            "  - {name: boolean, type: boolean, default: true}",
            "  - {name: long, type: long, default: 0}",
            "  - {name: double, type: double, default: 0.5}",
            "  - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
    Schema inputSchema = Schema.builder()
            .add("keep_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(7, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("keep_me", column.getName());
    }
    {
        column = outputSchema.getColumn(1);
        assertEquals("timestamp", column.getName());
    }
}
 

@Test
public void buildOutputSchema_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("keep_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(1, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("keep_me", column.getName());
    }
}
 

@Test
public void buildOutputSchema_Columns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("remove_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(6, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("timestamp", column.getName());
    }
}
 

@Override
public void timestampColumn(Column column)
{
    // formatted as string
    fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName(), OriginalType.UTF8));
}
 

private Value getColumnNameValue(Column column)
{
    return columnNameValues.get(column.getName());
}
 

@Override
public void stringColumn(Column column)
{
    fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName(), OriginalType.UTF8));
}
 

@Override
public void timestampColumn(Column column)
{
    // formatted as string
    fields.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveTypeName.BINARY, column.getName()));
}
 

@Override
public void jsonColumn(Column column)
{
    throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
}