org.embulk.spi.Schema Java Examples

The following examples show how to use org.embulk.spi.Schema. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JsonlParserPlugin.java    From embulk-parser-jsonl with MIT License 6 votes vote down vote up 
@Override
public void transaction(ConfigSource configSource, Control control)
{
    PluginTask task = configSource.loadConfig(PluginTask.class);

    if (! task.getColumnOptions().isEmpty()) {
        log.warn("embulk-parser-jsonl: \"column_options\" option is deprecated, specify type directly to \"columns\" option with typecast: true (default: true).");
    }

    SchemaConfig schemaConfig = getSchemaConfig(task);
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    for (int i = 0; i < schemaConfig.getColumnCount(); i++) {
        ColumnConfig columnConfig = schemaConfig.getColumn(i);
        Type type = getType(task, columnConfig);
        columns.add(new Column(i, columnConfig.getName(), type));
    }
    control.run(task.dump(), new Schema(columns.build()));
}
Example #2
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_withColumnNameIncludingSingleQuotes()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$['\\\\'json1']['k1']\"}");
    Schema inputSchema = Schema.builder()
            .add("'json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":"v"}
    Value k1 = ValueFactory.newString("k1");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(k1, v);

    MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
    assertEquals("{\"k1\":\"v\"}", visited.toString());
}
Example #3
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_withComplexRename()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$.json1['k____1']\", src: \"$.json1['k.-=+1']\"}",
            " - {name: \"$.json1['k____1'][0]['k____1']\", src: \"$.json1['k____1'][0]['k.-=+1']\"}",
            " - {name: \"$['json1']['k_2']\", src: \"$['json1']['k.2']\"}",
            " - {name: \"$['json1']['k_2']['k_2']\", src: \"$['json1']['k_2']['k.2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k.-=+1");
    Value k2 = ValueFactory.newString("k.2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
Example #4
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_withSingleQuotesAndDoubleQuotes()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}",
            " - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}",
            " - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}",
            " - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k.1");
    Value k2 = ValueFactory.newString("k.2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
Example #5
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_withDotAndBracket()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            " - {name: \"$.json1['k_1']\"}",
            " - {name: \"$.json1['k_1'][0]['k_1']\"}",
            " - {name: \"$['json1']['k_2']\"}",
            " - {name: \"$['json1']['k_2']['k_2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
    Value k1 = ValueFactory.newString("k_1");
    Value k2 = ValueFactory.newString("k_2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
}
Example #6
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitArray_columnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
            "  - {name: \"$['json1']['k2'][0]\"}",
            "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
}
Example #7
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitArray_addColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
            "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"}],"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
}
Example #8
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitArray_dropColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: \"$['json1']['k1'][0]['k1']\"}",
            "  - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
            k2, ValueFactory.newArray(v, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
}
Example #9
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitMap_columnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: \"$['json1']['k1']\"}",
            "  - {name: \"$['json1']['k2']['k2']\"}",
            "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
            "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
Example #10
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitMap_addColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
            "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
Example #11
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitMap_dropColumnsUsingBracketNotation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: \"$['json1']['k1']['k1']\"}",
            "  - {name: \"$['json1']['k2']\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{}}", visited.toString());
}
Example #12
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_AddColumns_WithSrc()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: copy, src: src}");
    Schema inputSchema = Schema.builder()
            .add("src", STRING)
            .build();
    List<Object[]> records = filter(task, inputSchema,
            "src");

    assertEquals(1, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(2, record.length);
        assertEquals("src", record[0]);
        assertEquals("src", record[1]);
    }
}
Example #13
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visit_Columns_WithSrc()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: copy, src: src}");
    Schema inputSchema = Schema.builder()
            .add("src", STRING)
            .build();
    List<Object[]> records = filter(task, inputSchema,
            "src");

    assertEquals(1, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(1, record.length);
        assertEquals("src", record[0]);
    }
}
Example #14
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitMap_AddColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: $.json1.k3.k3, type: string, default: v}",
            "  - {name: $.json1.k4, src: $.json1.k2}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
}
Example #15
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void visitMap_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: $.json1.k1.k1}",
            "  - {name: $.json1.k2}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
    Value k1 = ValueFactory.newString("k1");
    Value k2 = ValueFactory.newString("k2");
    Value v = ValueFactory.newString("v");
    Value map = ValueFactory.newMap(
            k1, ValueFactory.newMap(k1, v),
            k2, ValueFactory.newMap(k2, v));

    MapValue visited = subject.visit("$['json1']", map).asMapValue();
    assertEquals("{\"k1\":{}}", visited.toString());
}
Example #16
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
@Test
public void buildJsonSchema()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: $.json1.a.default}",
            "add_columns:",
            "  - {name: $.json1.a.copy, src: $.json1.a.src}",
            "columns:",
            "  - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .add("json2", JSON)
            .build();
    JsonVisitor subject = jsonVisitor(task, inputSchema);

    assertFalse(subject.jsonDropColumns.isEmpty());
    assertFalse(subject.jsonAddColumns.isEmpty());
    assertTrue(subject.jsonColumns.isEmpty()); // drop_columns overcome columns
}
Example #17
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 6 votes vote down vote up 
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects)
{
    MockPageOutput output = new MockPageOutput();
    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
    PageReader pageReader = new PageReader(inputSchema);
    ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);

    List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
    for (Page page : pages) {
        pageReader.setPage(page);

        while (pageReader.nextRecord()) {
            outputSchema.visitColumns(visitor);
            pageBuilder.addRecord();
        }
    }
    pageBuilder.finish();
    pageBuilder.close();
    return Pages.toObjects(outputSchema, output.pages);
}
Example #18
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
private void transaction(ConfigSource config, Schema inputSchema)
{
    plugin.transaction(config, inputSchema, new FilterPlugin.Control() {
        @Override
        public void run(TaskSource taskSource, Schema outputSchema)
        {
        }
    });
}
Example #19
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void visit_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("keep_me", STRING)
            .build();
    List<Object[]> records = filter(task, inputSchema,
            Timestamp.ofEpochSecond(1436745600), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "keep_me",
            null, null, null, null, null, null, "keep_me");

    assertEquals(2, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(1, record.length);
        assertEquals("keep_me", record[0]);
    }
    {
        record = records.get(1);
        assertEquals(1, record.length);
        assertEquals("keep_me", record[0]);
    }
}
Example #20
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void visit_Columns_WithDrop()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("remove_me", STRING)
            .build();
    List<Object[]> records = filter(task, inputSchema,
            Timestamp.ofEpochSecond(0), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "remove_me",
            Timestamp.ofEpochSecond(0), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "remove_me");

    assertEquals(2, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(6, record.length);
        assertEquals(Timestamp.ofEpochSecond(0), record[0]);
        assertEquals("string", record[1]);
        assertEquals(Boolean.valueOf(true), record[2]);
        assertEquals(Long.valueOf(0), record[3]);
        assertEquals(Double.valueOf(0.5), record[4]);
        assertEquals(ValueFactory.newString("json"), record[5]);
    }
}
Example #21
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test(expected = ConfigException.class)
public void configure_EitherOfColumnsOrDropColumnsCanBeSpecified()
{
    ConfigSource config = configFromYamlString(
            "type: column",
            "columns:",
            "- {name: a}",
            "drop_columns:",
            "- {name: a}");
    Schema inputSchema = schema(
            new Column(0, "a", STRING),
            new Column(1, "b", STRING));

    transaction(config, inputSchema);
}
Example #22
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void buildOutputSchema_AddColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
            "  - {name: string, type: string, default: string}",
            "  - {name: boolean, type: boolean, default: true}",
            "  - {name: long, type: long, default: 0}",
            "  - {name: double, type: double, default: 0.5}",
            "  - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
    Schema inputSchema = Schema.builder()
            .add("keep_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(7, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("keep_me", column.getName());
    }
    {
        column = outputSchema.getColumn(1);
        assertEquals("timestamp", column.getName());
    }
}
Example #23
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void buildOutputSchema_DropColumns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "drop_columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("keep_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(1, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("keep_me", column.getName());
    }
}
Example #24
Source File: TestColumnFilterPlugin.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void buildOutputSchema_Columns()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "  - {name: timestamp}",
            "  - {name: string}",
            "  - {name: boolean}",
            "  - {name: long}",
            "  - {name: double}",
            "  - {name: json}");
    Schema inputSchema = Schema.builder()
            .add("timestamp", TIMESTAMP)
            .add("string", STRING)
            .add("boolean", BOOLEAN)
            .add("long", LONG)
            .add("double", DOUBLE)
            .add("json", JSON)
            .add("remove_me", STRING)
            .build();

    Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
    assertEquals(6, outputSchema.size());

    Column column;
    {
        column = outputSchema.getColumn(0);
        assertEquals("timestamp", column.getName());
    }
}
Example #25
Source File: TestJsonVisitor.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test(expected = ConfigException.class)
public void configException_MultiIndexOperation()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "columns:",
            "- name: \"$.json1[0,1]\"");
    Schema inputSchema = Schema.builder()
            .add("json1", JSON)
            .build();
    jsonVisitor(task, inputSchema);
}
Example #26
Source File: TestColumnVisitorImpl.java    From embulk-filter-column with MIT License 5 votes vote down vote up 
@Test
public void visit_AddColumns_WithDefault()
{
    PluginTask task = taskFromYamlString(
            "type: column",
            "add_columns:",
            "  - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
            "  - {name: string, type: string, default: string}",
            "  - {name: boolean, type: boolean, default: true}",
            "  - {name: long, type: long, default: 0}",
            "  - {name: double, type: double, default: 0.5}",
            "  - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
    Schema inputSchema = Schema.builder()
            .add("keep_me", STRING)
            .build();
    List<Object[]> records = filter(task, inputSchema,
            "keep_me",
            "keep_me");

    assertEquals(2, records.size());

    Object[] record;
    {
        record = records.get(0);
        assertEquals(7, record.length);
        assertEquals("keep_me", record[0]);
        assertEquals(Timestamp.ofEpochSecond(1436745600), record[1]);
        assertEquals("string", record[2]);
        assertEquals(Boolean.valueOf(true), record[3]);
        assertEquals(Long.valueOf(0), record[4]);
        assertEquals(Double.valueOf(0.5), record[5]);
        assertEquals("{\"foo\":\"bar\"}", record[6].toString());
    }
}
Example #27
Source File: JsonlParserPlugin.java    From embulk-parser-jsonl with MIT License 5 votes vote down vote up 
private void setColumnNameValues(Schema schema)
{
    ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
    for (Column column : schema.getColumns()) {
        String name = column.getName();
        builder.put(name, newString(name));
    }
    columnNameValues = builder.build();
}
Example #28
Source File: ColumnVisitorImpl.java    From embulk-parser-jsonl with MIT License 5 votes vote down vote up 
public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
{
    this.task = task;
    this.schema = schema;
    this.pageBuilder = pageBuilder;
    this.timestampParsers = timestampParsers;
    this.autoTypecasts = new Boolean[schema.size()];
    buildAutoTypecasts();
}
Example #29
Source File: TestJsonlParserPlugin.java    From embulk-parser-jsonl with MIT License 5 votes vote down vote up 
private void transaction(ConfigSource config, final FileInput input)
{
    plugin.transaction(config, new ParserPlugin.Control()
    {
        @Override
        public void run(TaskSource taskSource, Schema schema)
        {
            plugin.run(taskSource, schema, input, output);
        }
    });
}
Example #30
Source File: ParquetOutputPlugin.java    From embulk-output-parquet with MIT License 5 votes vote down vote up 
public ConfigDiff transaction(ConfigSource config,
                              Schema schema, int processorCount,
                              OutputPlugin.Control control)
{
    PluginTask task = config.loadConfig(PluginTask.class);

    //TODO


    try (@SuppressWarnings("unchecked") ClassLoaderSwap clswp = new ClassLoaderSwap(this.getClass())) {
        control.run(task.dump());
    }
    return Exec.newConfigDiff();
}