Java Code Examples for org.datavec.api.transform.TransformProcess#execute()

The following examples show how to use org.datavec.api.transform.TransformProcess#execute() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestPythonTransformProcess.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test()
public void testStringConcat() throws Exception{
    Builder schemaBuilder = new Builder();
    schemaBuilder
            .addColumnString("col1")
            .addColumnString("col2");

    Schema initialSchema = schemaBuilder.build();
    schemaBuilder.addColumnString("col3");
    Schema finalSchema = schemaBuilder.build();

    String pythonCode = "col3 = col1 + col2";

    TransformProcess tp = new TransformProcess.Builder(initialSchema).transform(
            PythonTransform.builder().code(pythonCode)
                    .outputSchema(finalSchema)
                    .build()
    ).build();

    List<Writable> inputs = Arrays.asList((Writable)new Text("Hello "), new Text("World!"));

    List<Writable> outputs = tp.execute(inputs);
    assertEquals((outputs.get(0)).toString(), "Hello ");
    assertEquals((outputs.get(1)).toString(), "World!");
    assertEquals((outputs.get(2)).toString(), "Hello World!");

}
Example 2
Source File: TestPythonTransformProcess.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 60000L)
public void testMixedTypes() throws Exception{
    Builder schemaBuilder = new Builder();
    schemaBuilder
            .addColumnInteger("col1")
            .addColumnFloat("col2")
            .addColumnString("col3")
            .addColumnDouble("col4");


    Schema initialSchema = schemaBuilder.build();
    schemaBuilder.addColumnInteger("col5");
    Schema finalSchema = schemaBuilder.build();

    String pythonCode = "col5  = (int(col3) + col1 + int(col2)) * int(col4)";

    TransformProcess tp = new TransformProcess.Builder(initialSchema).transform(
            PythonTransform.builder().code(pythonCode)
                    .outputSchema(finalSchema)
                    .inputSchema(initialSchema)
                    .build()        ).build();

    List<Writable> inputs = Arrays.asList((Writable)new IntWritable(10),
            new FloatWritable(3.5f),
            new Text("5"),
            new DoubleWritable(2.0)
    );

    List<Writable> outputs = tp.execute(inputs);
    assertEquals(((LongWritable)outputs.get(4)).get(), 36);
}
Example 3
Source File: TestPythonTransformProcess.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 60000L)
public void testNDArray() throws Exception{
    long[] shape = new long[]{3, 2};
    INDArray arr1 = Nd4j.rand(shape);
    INDArray arr2 = Nd4j.rand(shape);

    INDArray expectedOutput = arr1.add(arr2);

    Builder schemaBuilder = new Builder();
    schemaBuilder
            .addColumnNDArray("col1", shape)
            .addColumnNDArray("col2", shape);

    Schema initialSchema = schemaBuilder.build();
    schemaBuilder.addColumnNDArray("col3", shape);
    Schema finalSchema = schemaBuilder.build();

    String pythonCode = "col3 = col1 + col2";
    TransformProcess tp = new TransformProcess.Builder(initialSchema).transform(
            PythonTransform.builder().code(pythonCode)
                    .outputSchema(finalSchema)
                    .build()        ).build();

    List<Writable> inputs = Arrays.asList(
            (Writable)
            new NDArrayWritable(arr1),
            new NDArrayWritable(arr2)
    );

    List<Writable> outputs = tp.execute(inputs);
    assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get());
    assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get());
    assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get());

}
Example 4
Source File: TestPythonTransformProcess.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 60000L)
public void testNDArray2() throws Exception{
    long[] shape = new long[]{3, 2};
    INDArray arr1 = Nd4j.rand(shape);
    INDArray arr2 = Nd4j.rand(shape);

    INDArray expectedOutput = arr1.add(arr2);

    Builder schemaBuilder = new Builder();
    schemaBuilder
            .addColumnNDArray("col1", shape)
            .addColumnNDArray("col2", shape);

    Schema initialSchema = schemaBuilder.build();
    schemaBuilder.addColumnNDArray("col3", shape);
    Schema finalSchema = schemaBuilder.build();

    String pythonCode = "col3 = col1 + col2";
    TransformProcess tp = new TransformProcess.Builder(initialSchema).transform(
            PythonTransform.builder().code(pythonCode)
                    .outputSchema(finalSchema)
                    .build()        ).build();

    List<Writable> inputs = Arrays.asList(
            (Writable)
            new NDArrayWritable(arr1),
            new NDArrayWritable(arr2)
    );

    List<Writable> outputs = tp.execute(inputs);
    assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get());
    assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get());
    assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get());

}
Example 5
Source File: TestPythonTransformProcess.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 60000L)
public void testNDArrayMixed() throws Exception{
    long[] shape = new long[]{3, 2};
    INDArray arr1 = Nd4j.rand(DataType.DOUBLE, shape);
    INDArray arr2 = Nd4j.rand(DataType.DOUBLE, shape);
    INDArray expectedOutput = arr1.add(arr2.castTo(DataType.DOUBLE));

    Builder schemaBuilder = new Builder();
    schemaBuilder
            .addColumnNDArray("col1", shape)
            .addColumnNDArray("col2", shape);

    Schema initialSchema = schemaBuilder.build();
    schemaBuilder.addColumnNDArray("col3", shape);
    Schema finalSchema = schemaBuilder.build();

    String pythonCode = "col3 = col1 + col2";
    TransformProcess tp = new TransformProcess.Builder(initialSchema).transform(
            PythonTransform.builder().code(pythonCode)
                    .outputSchema(finalSchema)
                    .build()
    ).build();

    List<Writable> inputs = Arrays.asList(
            (Writable)
            new NDArrayWritable(arr1),
            new NDArrayWritable(arr2)
    );

    List<Writable> outputs = tp.execute(inputs);
    assertEquals(arr1, ((NDArrayWritable)outputs.get(0)).get());
    assertEquals(arr2, ((NDArrayWritable)outputs.get(1)).get());
    assertEquals(expectedOutput,((NDArrayWritable)outputs.get(2)).get());

}
Example 6
Source File: TestNDArrayWritableTransforms.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNDArrayWritableBasic() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10}).addColumnString("col2")
                    .build();


    TransformProcess tp = new TransformProcess.Builder(s).ndArrayScalarOpTransform("col1", MathOp.Add, 100).build();

    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new Text("str0"));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0),
                    new NDArrayWritable(Nd4j.linspace(0, 9, 10).addi(100)), new Text("str0"));

    assertEquals(exp, out);

}
Example 7
Source File: TestNDArrayWritableTransforms.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testNDArrayWritableBasic() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10}).addColumnString("col2")
                    .build();


    TransformProcess tp = new TransformProcess.Builder(s).ndArrayScalarOpTransform("col1", MathOp.Add, 100).build();

    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new Text("str0"));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0),
                    new NDArrayWritable(Nd4j.linspace(0, 9, 10).addi(100)), new Text("str0"));

    assertEquals(exp, out);

}
Example 8
Source File: TestNDArrayWritableTransforms.java    From DataVec with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayColumnsMathOpTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s)
                    .ndArrayColumnsMathOpTransform("myCol", MathOp.Add, "col1", "col2").build();

    List<String> expColNames = Arrays.asList("col0", "col1", "col2", "myCol");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());


    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0)));
    List<Writable> out = tp.execute(in);

    List<Writable> exp =
                    Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0)),
                                    new NDArrayWritable(Nd4j.linspace(0, 9, 10).addi(2.0)));

    assertEquals(exp, out);
}
Example 9
Source File: TestNDArrayWritableTransforms.java    From DataVec with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayMathFunctionTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s).ndArrayMathFunctionTransform("col1", MathFunction.SIN)
                    .ndArrayMathFunctionTransform("col2", MathFunction.SQRT).build();



    List<String> expColNames = Arrays.asList("col0", "col1", "col2");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());


    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0)));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0),
                    new NDArrayWritable(Transforms.sin(Nd4j.linspace(0, 9, 10))),
                    new NDArrayWritable(Transforms.sqrt(Nd4j.valueArrayOf(1, 10, 2.0))));

    assertEquals(exp, out);
}
Example 10
Source File: TestNDArrayWritableTransforms.java    From DataVec with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayDistanceTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s)
                    .ndArrayDistanceTransform("dist", Distance.COSINE, "col1", "col2").build();



    List<String> expColNames = Arrays.asList("col0", "col1", "col2", "dist");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());

    Nd4j.getRandom().setSeed(12345);
    INDArray arr1 = Nd4j.rand(1, 10);
    INDArray arr2 = Nd4j.rand(1, 10);
    double cosine = Transforms.cosineSim(arr1, arr2);

    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(arr1.dup()),
                    new NDArrayWritable(arr2.dup()));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(arr1),
                    new NDArrayWritable(arr2), new DoubleWritable(cosine));

    assertEquals(exp, out);
}
Example 11
Source File: TestNDArrayWritableTransforms.java    From deeplearning4j with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayColumnsMathOpTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s)
                    .ndArrayColumnsMathOpTransform("myCol", MathOp.Add, "col1", "col2").build();

    List<String> expColNames = Arrays.asList("col0", "col1", "col2", "myCol");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());


    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(DataType.DOUBLE,0, 10, 1).reshape(1,10)),
                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0).castTo(DataType.DOUBLE)));
    List<Writable> out = tp.execute(in);

    List<Writable> exp =
                    Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(DataType.DOUBLE,0, 10, 1).reshape(1,10)),
                                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0).castTo(DataType.DOUBLE)),
                                    new NDArrayWritable(Nd4j.linspace(DataType.DOUBLE, 0, 10, 1).addi(2.0).reshape(1,10)));

    assertEquals(exp, out);
}
Example 12
Source File: TestNDArrayWritableTransforms.java    From deeplearning4j with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayMathFunctionTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s).ndArrayMathFunctionTransform("col1", MathFunction.SIN)
                    .ndArrayMathFunctionTransform("col2", MathFunction.SQRT).build();



    List<String> expColNames = Arrays.asList("col0", "col1", "col2");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());


    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(Nd4j.linspace(0, 9, 10)),
                    new NDArrayWritable(Nd4j.valueArrayOf(1, 10, 2.0)));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0),
                    new NDArrayWritable(Transforms.sin(Nd4j.linspace(0, 9, 10))),
                    new NDArrayWritable(Transforms.sqrt(Nd4j.valueArrayOf(1, 10, 2.0))));

    assertEquals(exp, out);
}
Example 13
Source File: TestNDArrayWritableTransforms.java    From deeplearning4j with Apache License 2.0 3 votes vote down vote up 
@Test
public void testNDArrayDistanceTransform() {

    Schema s = new Schema.Builder()

                    .addColumnDouble("col0").addColumnNDArray("col1", new long[] {1, 10})
                    .addColumnNDArray("col2", new long[] {1, 10}).build();


    TransformProcess tp = new TransformProcess.Builder(s)
                    .ndArrayDistanceTransform("dist", Distance.COSINE, "col1", "col2").build();



    List<String> expColNames = Arrays.asList("col0", "col1", "col2", "dist");
    assertEquals(expColNames, tp.getFinalSchema().getColumnNames());

    Nd4j.getRandom().setSeed(12345);
    INDArray arr1 = Nd4j.rand(1, 10);
    INDArray arr2 = Nd4j.rand(1, 10);
    double cosine = Transforms.cosineSim(arr1, arr2);

    List<Writable> in = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(arr1.dup()),
                    new NDArrayWritable(arr2.dup()));
    List<Writable> out = tp.execute(in);

    List<Writable> exp = Arrays.<Writable>asList(new DoubleWritable(0), new NDArrayWritable(arr1),
                    new NDArrayWritable(arr2), new DoubleWritable(cosine));

    assertEquals(exp, out);
}