Java Code Examples for org.datavec.api.transform.TransformProcess#toJson()

The following examples show how to use org.datavec.api.transform.TransformProcess#toJson() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SerializationExample.java    From Java-Deep-Learning-Cookbook with MIT License 6 votes vote down vote up 
public static void main(String[] args) {
    Schema schema  =  new Schema.Builder()
            .addColumnsString("Name", "Subject")
            .addColumnInteger("Score")
            .addColumnCategorical("Grade", Arrays.asList("A","B","C","D"))
            .addColumnInteger("Passed").build();

    TransformProcess transformProcess = new TransformProcess.Builder(schema)
            .removeColumns("Name")
            .transform(new ConvertToDouble("Score"))
            .categoricalToInteger("Grade").build();

    String json = transformProcess.toJson();
    System.out.println(json);

    String yaml = transformProcess.toYaml();
    System.out.println(yaml);
}
Example 2
Source File: SerializationExample.java    From Java-Deep-Learning-Cookbook with MIT License 6 votes vote down vote up 
public static void main(String[] args) {
    Schema schema  =  new Schema.Builder()
            .addColumnsString("Name", "Subject")
            .addColumnInteger("Score")
            .addColumnCategorical("Grade", Arrays.asList("A","B","C","D"))
            .addColumnInteger("Passed").build();

    TransformProcess transformProcess = new TransformProcess.Builder(schema)
            .removeColumns("Name")
            .transform(new ConvertToDouble("Score"))
            .categoricalToInteger("Grade").build();

    String json = transformProcess.toJson();
    System.out.println(json);

    String yaml = transformProcess.toYaml();
    System.out.println(yaml);
}
Example 3
Source File: TestYamlJsonSerde.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testTransformProcessAndSchema() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnNDArray("nd1a", new long[] {1, 10})
                    .addColumnNDArray("nd1b", new long[] {1, 10}).addColumnNDArray("nd2", new long[] {1, 100})
                    .addColumnNDArray("nd3", new long[] {-1, -1}).build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .ndArrayColumnsMathOpTransform("added", MathOp.Add, "nd1a", "nd1b")
                    .ndArrayMathFunctionTransform("nd2", MathFunction.SQRT)
                    .ndArrayScalarOpTransform("nd3", MathOp.Multiply, 2.0).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
Example 4
Source File: TestCustomTransformJsonYaml.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCustomTransform() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnDouble("secondCol").build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .transform(new CustomTransform("secondCol", 3.14159))
                    .doubleMathOp("secondCol", MathOp.Multiply, 2.0).filter(new CustomFilter(123))
                    .filter(new CustomCondition("someArg")).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
Example 5
Source File: TestYamlJsonSerde.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testTransformProcessAndSchema() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnNDArray("nd1a", new long[] {1, 10})
                    .addColumnNDArray("nd1b", new long[] {1, 10}).addColumnNDArray("nd2", new long[] {1, 100})
                    .addColumnNDArray("nd3", new long[] {-1, -1}).build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .ndArrayColumnsMathOpTransform("added", MathOp.Add, "nd1a", "nd1b")
                    .ndArrayMathFunctionTransform("nd2", MathFunction.SQRT)
                    .ndArrayScalarOpTransform("nd3", MathOp.Multiply, 2.0).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
Example 6
Source File: TestCustomTransformJsonYaml.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCustomTransform() {

    Schema schema = new Schema.Builder().addColumnInteger("firstCol").addColumnDouble("secondCol").build();

    TransformProcess tp = new TransformProcess.Builder(schema).integerMathOp("firstCol", MathOp.Add, 1)
                    .transform(new CustomTransform("secondCol", 3.14159))
                    .doubleMathOp("secondCol", MathOp.Multiply, 2.0).filter(new CustomFilter(123))
                    .filter(new CustomCondition("someArg")).build();

    String asJson = tp.toJson();
    String asYaml = tp.toYaml();

    TransformProcess fromJson = TransformProcess.fromJson(asJson);
    TransformProcess fromYaml = TransformProcess.fromYaml(asYaml);

    assertEquals(tp, fromJson);
    assertEquals(tp, fromYaml);
}
Example 7
Source File: DataVecTransformClient.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
/**
 * @param transformProcess
 */
@Override
public void setCSVTransformProcess(TransformProcess transformProcess) {
    try {
        String s = transformProcess.toJson();
        Unirest.post(url + "/transformprocess").header("accept", "application/json")
                .header("Content-Type", "application/json").body(s).asJson();

    } catch (UnirestException e) {
        log.error("Error in setCSVTransformProcess()", e);
        e.printStackTrace();
    }
}
Example 8
Source File: DataVecTransformClient.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
/**
 * @param transformProcess
 */
@Override
public void setCSVTransformProcess(TransformProcess transformProcess) {
    try {
        String s = transformProcess.toJson();
        Unirest.post(url + "/transformprocess").header("accept", "application/json")
                .header("Content-Type", "application/json").body(s).asJson();

    } catch (UnirestException e) {
        log.error("Error in setCSVTransformProcess()", e);
    }
}
Example 9
Source File: TestGazeteerTransform.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGazeteerTransform(){

    String[] corpus = {
            "hello I like apple".toLowerCase(),
            "cherry date eggplant potato".toLowerCase()
    };

    //Gazeteer transform: basically 0/1 if word is present. Assumes already tokenized input
    List<String> words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant");

    GazeteerTransform t = new GazeteerTransform("words", "out", words);

    SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder()
            .addColumnString("words").build();

    TransformProcess tp = new TransformProcess.Builder(schema)
            .transform(t)
            .build();

    List<List<List<Writable>>> input = new ArrayList<>();
    for(String s : corpus){
        String[] split = s.split(" ");
        List<List<Writable>> seq = new ArrayList<>();
        for(String s2 : split){
            seq.add(Collections.<Writable>singletonList(new Text(s2)));
        }
        input.add(seq);
    }

    List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp);

    INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get();
    INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get();

    INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0});
    INDArray exp1 = Nd4j.create(new float[]{0, 0, 1, 1, 1});

    assertEquals(exp0, arr0);
    assertEquals(exp1, arr1);


    String json = tp.toJson();
    TransformProcess tp2 = TransformProcess.fromJson(json);
    assertEquals(tp, tp2);

    List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp);
    INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get();
    INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get();

    assertEquals(exp0, arr0a);
    assertEquals(exp1, arr1a);
}
Example 10
Source File: TestMultiNLPTransform.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void test(){

    List<String> words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant");
    GazeteerTransform t1 = new GazeteerTransform("words", "out", words);
    GazeteerTransform t2 = new GazeteerTransform("out", "out", words);


    MultiNlpTransform multi = new MultiNlpTransform("text", new BagOfWordsTransform[]{t1, t2}, "out");

    String[] corpus = {
            "hello I like apple".toLowerCase(),
            "date eggplant potato".toLowerCase()
    };

    List<List<List<Writable>>> input = new ArrayList<>();
    for(String s : corpus){
        String[] split = s.split(" ");
        List<List<Writable>> seq = new ArrayList<>();
        for(String s2 : split){
            seq.add(Collections.<Writable>singletonList(new Text(s2)));
        }
        input.add(seq);
    }

    SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder()
            .addColumnString("text").build();

    TransformProcess tp = new TransformProcess.Builder(schema)
            .transform(multi)
            .build();

    List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp);

    INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get();
    INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get();

    INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0, 1, 0, 0, 0, 0});
    INDArray exp1 = Nd4j.create(new float[]{0, 0, 0, 1, 1, 0, 0, 0, 1, 1});

    assertEquals(exp0, arr0);
    assertEquals(exp1, arr1);


    String json = tp.toJson();
    TransformProcess tp2 = TransformProcess.fromJson(json);
    assertEquals(tp, tp2);

    List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp);
    INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get();
    INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get();

    assertEquals(exp0, arr0a);
    assertEquals(exp1, arr1a);

}