Java Code Examples for org.datavec.local.transforms.LocalTransformExecutor#executeSequenceToSequence()

The following examples show how to use org.datavec.local.transforms.LocalTransformExecutor#executeSequenceToSequence() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ExecutionTest.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Test
public void testExecutionSequence() {

    Schema schema = new SequenceSchema.Builder().addColumnInteger("col0")
            .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
            .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<List<Writable>>> inputSequences = new ArrayList<>();
    List<List<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));
    List<List<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1)));
    seq2.add(Arrays.<Writable>asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1)));

    inputSequences.add(seq1);
    inputSequences.add(seq2);

    List<List<List<Writable>>> rdd =  (inputSequences);

    List<List<List<Writable>>> out = LocalTransformExecutor.executeSequenceToSequence(rdd, tp);

    Collections.sort(out, new Comparator<List<List<Writable>>>() {
        @Override
        public int compare(List<List<Writable>> o1, List<List<Writable>> o2) {
            return -Integer.compare(o1.size(), o2.size());
        }
    });

    List<List<List<Writable>>> expectedSequence = new ArrayList<>();
    List<List<Writable>> seq1e = new ArrayList<>();
    seq1e.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));
    List<List<Writable>> seq2e = new ArrayList<>();
    seq2e.add(Arrays.<Writable>asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1)));
    seq2e.add(Arrays.<Writable>asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1)));

    expectedSequence.add(seq1e);
    expectedSequence.add(seq2e);

    assertEquals(expectedSequence, out);
}
Example 2
Source File: TestGazeteerTransform.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGazeteerTransform(){

    String[] corpus = {
            "hello I like apple".toLowerCase(),
            "cherry date eggplant potato".toLowerCase()
    };

    //Gazeteer transform: basically 0/1 if word is present. Assumes already tokenized input
    List<String> words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant");

    GazeteerTransform t = new GazeteerTransform("words", "out", words);

    SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder()
            .addColumnString("words").build();

    TransformProcess tp = new TransformProcess.Builder(schema)
            .transform(t)
            .build();

    List<List<List<Writable>>> input = new ArrayList<>();
    for(String s : corpus){
        String[] split = s.split(" ");
        List<List<Writable>> seq = new ArrayList<>();
        for(String s2 : split){
            seq.add(Collections.<Writable>singletonList(new Text(s2)));
        }
        input.add(seq);
    }

    List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp);

    INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get();
    INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get();

    INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0});
    INDArray exp1 = Nd4j.create(new float[]{0, 0, 1, 1, 1});

    assertEquals(exp0, arr0);
    assertEquals(exp1, arr1);


    String json = tp.toJson();
    TransformProcess tp2 = TransformProcess.fromJson(json);
    assertEquals(tp, tp2);

    List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp);
    INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get();
    INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get();

    assertEquals(exp0, arr0a);
    assertEquals(exp1, arr1a);
}
Example 3
Source File: TestMultiNLPTransform.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void test(){

    List<String> words = Arrays.asList("apple", "banana", "cherry", "date", "eggplant");
    GazeteerTransform t1 = new GazeteerTransform("words", "out", words);
    GazeteerTransform t2 = new GazeteerTransform("out", "out", words);


    MultiNlpTransform multi = new MultiNlpTransform("text", new BagOfWordsTransform[]{t1, t2}, "out");

    String[] corpus = {
            "hello I like apple".toLowerCase(),
            "date eggplant potato".toLowerCase()
    };

    List<List<List<Writable>>> input = new ArrayList<>();
    for(String s : corpus){
        String[] split = s.split(" ");
        List<List<Writable>> seq = new ArrayList<>();
        for(String s2 : split){
            seq.add(Collections.<Writable>singletonList(new Text(s2)));
        }
        input.add(seq);
    }

    SequenceSchema schema = (SequenceSchema) new SequenceSchema.Builder()
            .addColumnString("text").build();

    TransformProcess tp = new TransformProcess.Builder(schema)
            .transform(multi)
            .build();

    List<List<List<Writable>>> execute = LocalTransformExecutor.executeSequenceToSequence(input, tp);

    INDArray arr0 = ((NDArrayWritable)execute.get(0).get(0).get(0)).get();
    INDArray arr1 = ((NDArrayWritable)execute.get(0).get(1).get(0)).get();

    INDArray exp0 = Nd4j.create(new float[]{1, 0, 0, 0, 0, 1, 0, 0, 0, 0});
    INDArray exp1 = Nd4j.create(new float[]{0, 0, 0, 1, 1, 0, 0, 0, 1, 1});

    assertEquals(exp0, arr0);
    assertEquals(exp1, arr1);


    String json = tp.toJson();
    TransformProcess tp2 = TransformProcess.fromJson(json);
    assertEquals(tp, tp2);

    List<List<List<Writable>>> execute2 = LocalTransformExecutor.executeSequenceToSequence(input, tp);
    INDArray arr0a = ((NDArrayWritable)execute2.get(0).get(0).get(0)).get();
    INDArray arr1a = ((NDArrayWritable)execute2.get(0).get(1).get(0)).get();

    assertEquals(exp0, arr0a);
    assertEquals(exp1, arr1a);

}
Example 4
Source File: ExecutionTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testExecutionSequence() {

    Schema schema = new SequenceSchema.Builder().addColumnInteger("col0")
            .addColumnCategorical("col1", "state0", "state1", "state2").addColumnDouble("col2").build();

    TransformProcess tp = new TransformProcess.Builder(schema).categoricalToInteger("col1")
            .doubleMathOp("col2", MathOp.Add, 10.0).build();

    List<List<List<Writable>>> inputSequences = new ArrayList<>();
    List<List<Writable>> seq1 = new ArrayList<>();
    seq1.add(Arrays.<Writable>asList(new IntWritable(0), new Text("state2"), new DoubleWritable(0.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(1), new Text("state1"), new DoubleWritable(1.1)));
    seq1.add(Arrays.<Writable>asList(new IntWritable(2), new Text("state0"), new DoubleWritable(2.1)));
    List<List<Writable>> seq2 = new ArrayList<>();
    seq2.add(Arrays.<Writable>asList(new IntWritable(3), new Text("state0"), new DoubleWritable(3.1)));
    seq2.add(Arrays.<Writable>asList(new IntWritable(4), new Text("state1"), new DoubleWritable(4.1)));

    inputSequences.add(seq1);
    inputSequences.add(seq2);

    List<List<List<Writable>>> rdd =  (inputSequences);

    List<List<List<Writable>>> out = LocalTransformExecutor.executeSequenceToSequence(rdd, tp);

    Collections.sort(out, new Comparator<List<List<Writable>>>() {
        @Override
        public int compare(List<List<Writable>> o1, List<List<Writable>> o2) {
            return -Integer.compare(o1.size(), o2.size());
        }
    });

    List<List<List<Writable>>> expectedSequence = new ArrayList<>();
    List<List<Writable>> seq1e = new ArrayList<>();
    seq1e.add(Arrays.<Writable>asList(new IntWritable(0), new IntWritable(2), new DoubleWritable(10.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(1), new IntWritable(1), new DoubleWritable(11.1)));
    seq1e.add(Arrays.<Writable>asList(new IntWritable(2), new IntWritable(0), new DoubleWritable(12.1)));
    List<List<Writable>> seq2e = new ArrayList<>();
    seq2e.add(Arrays.<Writable>asList(new IntWritable(3), new IntWritable(0), new DoubleWritable(13.1)));
    seq2e.add(Arrays.<Writable>asList(new IntWritable(4), new IntWritable(1), new DoubleWritable(14.1)));

    expectedSequence.add(seq1e);
    expectedSequence.add(seq2e);

    assertEquals(expectedSequence, out);
}