Java Code Examples for org.datavec.api.records.reader.SequenceRecordReader#reset()

The following examples show how to use org.datavec.api.records.reader.SequenceRecordReader#reset() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 2
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith(file.getName()));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 3
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 4
Source File: CSVNLinesSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCSVNlinesSequenceRecordReaderMetaData() throws Exception {
    int nLinesPerSequence = 10;

    SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
    seqRR.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    CSVRecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    List<List<List<Writable>>> out = new ArrayList<>();
    while (seqRR.hasNext()) {
        List<List<Writable>> next = seqRR.sequenceRecord();
        out.add(next);
    }

    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (seqRR.hasNext()) {
        SequenceRecord seq = seqRR.nextSequence();
        out2.add(seq.getSequenceRecord());
        meta.add(seq.getMetaData());
        out3.add(seq);
    }

    assertEquals(out, out2);

    List<SequenceRecord> out4 = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(out3, out4);
}
Example 5
Source File: RegexRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRegexSequenceRecordReader() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    String path = new ClassPathResource("/logtestdata/logtestfile0.txt").getFile().toURI().toString();
    path = path.replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<Writable>> exp0 = new ArrayList<>();
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), new Text("DEBUG"),
                    new Text("First entry message!")));
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), new Text("INFO"),
                    new Text("Second entry message!")));
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), new Text("WARN"),
                    new Text("Third entry message!")));


    List<List<Writable>> exp1 = new ArrayList<>();
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.011"), new Text("11"), new Text("DEBUG"),
                    new Text("First entry message!")));
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.012"), new Text("12"), new Text("INFO"),
                    new Text("Second entry message!")));
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.013"), new Text("13"), new Text("WARN"),
                    new Text("Third entry message!")));

    assertEquals(exp0, rr.sequenceRecord());
    assertEquals(exp1, rr.sequenceRecord());
    assertFalse(rr.hasNext());

    //Test resetting:
    rr.reset();
    assertEquals(exp0, rr.sequenceRecord());
    assertEquals(exp1, rr.sequenceRecord());
    assertFalse(rr.hasNext());
}
Example 6
Source File: RegexRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRegexSequenceRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    String path = new ClassPathResource("/logtestdata/logtestfile0.txt").getFile().toURI().toString();
    path = path.replace("0", "%d");
    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<List<Writable>>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.sequenceRecord());
    }

    assertEquals(2, out.size());
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    while (rr.hasNext()) {
        SequenceRecord seqr = rr.nextSequence();
        out2.add(seqr.getSequenceRecord());
        out3.add(seqr);
        meta.add(seqr.getMetaData());
    }

    List<SequenceRecord> fromMeta = rr.loadSequenceFromMetaData(meta);

    assertEquals(out, out2);
    assertEquals(out3, fromMeta);
}
Example 7
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 8
Source File: CSVNLinesSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCSVNlinesSequenceRecordReaderMetaData() throws Exception {
    int nLinesPerSequence = 10;

    SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
    seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    CSVRecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    List<List<List<Writable>>> out = new ArrayList<>();
    while (seqRR.hasNext()) {
        List<List<Writable>> next = seqRR.sequenceRecord();
        out.add(next);
    }

    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (seqRR.hasNext()) {
        SequenceRecord seq = seqRR.nextSequence();
        out2.add(seq.getSequenceRecord());
        meta.add(seq.getMetaData());
        out3.add(seq);
    }

    assertEquals(out, out2);

    List<SequenceRecord> out4 = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(out3, out4);
}
Example 9
Source File: RegexRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRegexSequenceRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    ClassPathResource cpr = new ClassPathResource("datavec-api/logtestdata/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "logtestfile%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<List<Writable>>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.sequenceRecord());
    }

    assertEquals(2, out.size());
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    while (rr.hasNext()) {
        SequenceRecord seqr = rr.nextSequence();
        out2.add(seqr.getSequenceRecord());
        out3.add(seqr);
        meta.add(seqr.getMetaData());
    }

    List<SequenceRecord> fromMeta = rr.loadSequenceFromMetaData(meta);

    assertEquals(out, out2);
    assertEquals(out3, fromMeta);
}
Example 10
Source File: CSVLineSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void test() throws Exception {

    File f = testDir.newFolder();
    File source = new File(f, "temp.csv");
    String str = "a,b,c\n1,2,3,4";
    FileUtils.writeStringToFile(source, str, StandardCharsets.UTF_8);

    SequenceRecordReader rr = new CSVLineSequenceRecordReader();
    rr.initialize(new FileSplit(source));

    List<List<Writable>> exp0 = Arrays.asList(
            Collections.<Writable>singletonList(new Text("a")),
            Collections.<Writable>singletonList(new Text("b")),
            Collections.<Writable>singletonList(new Text("c")));

    List<List<Writable>> exp1 = Arrays.asList(
            Collections.<Writable>singletonList(new Text("1")),
            Collections.<Writable>singletonList(new Text("2")),
            Collections.<Writable>singletonList(new Text("3")),
            Collections.<Writable>singletonList(new Text("4")));

    for( int i=0; i<3; i++ ) {
        int count = 0;
        while (rr.hasNext()) {
            List<List<Writable>> next = rr.sequenceRecord();
            if (count++ == 0) {
                assertEquals(exp0, next);
            } else {
                assertEquals(exp1, next);
            }
        }

        assertEquals(2, count);

        rr.reset();
    }
}
Example 11
Source File: TestCollectionRecordReaders.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCollectionSequenceRecordReader() throws Exception {

    List<List<List<Writable>>> listOfSequences = new ArrayList<>();

    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new IntWritable(0), new IntWritable(1)));
    sequence1.add(Arrays.asList((Writable) new IntWritable(2), new IntWritable(3)));
    listOfSequences.add(sequence1);

    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new IntWritable(4), new IntWritable(5)));
    sequence2.add(Arrays.asList((Writable) new IntWritable(6), new IntWritable(7)));
    listOfSequences.add(sequence2);

    SequenceRecordReader seqRR = new CollectionSequenceRecordReader(listOfSequences);
    assertTrue(seqRR.hasNext());

    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    seqRR.reset();
    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    //Test metadata:
    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> seq = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();

    while (seqRR.hasNext()) {
        SequenceRecord r = seqRR.nextSequence();
        out2.add(r.getSequenceRecord());
        seq.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(listOfSequences, out2);

    List<SequenceRecord> fromMeta = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(seq, fromMeta);
}
Example 12
Source File: CSVMultiSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testEqualLength() throws Exception {

    for( int i=0; i<3; i++ ) {

        String seqSep;
        String seqSepRegex;
        switch (i) {
            case 0:
                seqSep = "";
                seqSepRegex = "^$";
                break;
            case 1:
                seqSep = "---";
                seqSepRegex = seqSep;
                break;
            case 2:
                seqSep = "&";
                seqSepRegex = seqSep;
                break;
            default:
                throw new RuntimeException();
        }

        String str = "a,b\n1,2\nx,y\n" + seqSep + "\nA\nB\nC";
        File f = testDir.newFile();
        FileUtils.writeStringToFile(f, str, StandardCharsets.UTF_8);

        SequenceRecordReader seqRR = new CSVMultiSequenceRecordReader(seqSepRegex, CSVMultiSequenceRecordReader.Mode.EQUAL_LENGTH);
        seqRR.initialize(new FileSplit(f));


        List<List<Writable>> exp0 = Arrays.asList(
                Arrays.<Writable>asList(new Text("a"), new Text("1"), new Text("x")),
                Arrays.<Writable>asList(new Text("b"), new Text("2"), new Text("y")));

        List<List<Writable>> exp1 = Collections.singletonList(Arrays.<Writable>asList(new Text("A"), new Text("B"), new Text("C")));

        assertEquals(exp0, seqRR.sequenceRecord());
        assertEquals(exp1, seqRR.sequenceRecord());
        assertFalse(seqRR.hasNext());

        seqRR.reset();

        assertEquals(exp0, seqRR.sequenceRecord());
        assertEquals(exp1, seqRR.sequenceRecord());
        assertFalse(seqRR.hasNext());
    }
}
Example 13
Source File: CSVMultiSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPadding() throws Exception {

    for( int i=0; i<3; i++ ) {

        String seqSep;
        String seqSepRegex;
        switch (i) {
            case 0:
                seqSep = "";
                seqSepRegex = "^$";
                break;
            case 1:
                seqSep = "---";
                seqSepRegex = seqSep;
                break;
            case 2:
                seqSep = "&";
                seqSepRegex = seqSep;
                break;
            default:
                throw new RuntimeException();
        }

        String str = "a,b\n1\nx\n" + seqSep + "\nA\nB\nC";
        File f = testDir.newFile();
        FileUtils.writeStringToFile(f, str, StandardCharsets.UTF_8);

        SequenceRecordReader seqRR = new CSVMultiSequenceRecordReader(seqSepRegex, CSVMultiSequenceRecordReader.Mode.PAD, new Text("PAD"));
        seqRR.initialize(new FileSplit(f));


        List<List<Writable>> exp0 = Arrays.asList(
                Arrays.<Writable>asList(new Text("a"), new Text("1"), new Text("x")),
                Arrays.<Writable>asList(new Text("b"), new Text("PAD"), new Text("PAD")));

        List<List<Writable>> exp1 = Collections.singletonList(Arrays.<Writable>asList(new Text("A"), new Text("B"), new Text("C")));

        assertEquals(exp0, seqRR.sequenceRecord());
        assertEquals(exp1, seqRR.sequenceRecord());
        assertFalse(seqRR.hasNext());

        seqRR.reset();

        assertEquals(exp0, seqRR.sequenceRecord());
        assertEquals(exp1, seqRR.sequenceRecord());
        assertFalse(seqRR.hasNext());
    }
}
Example 14
Source File: RegexRecordReaderTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRegexSequenceRecordReader() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    ClassPathResource cpr = new ClassPathResource("datavec-api/logtestdata/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "logtestfile%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<Writable>> exp0 = new ArrayList<>();
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"), new Text("DEBUG"),
                    new Text("First entry message!")));
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"), new Text("INFO"),
                    new Text("Second entry message!")));
    exp0.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"), new Text("WARN"),
                    new Text("Third entry message!")));


    List<List<Writable>> exp1 = new ArrayList<>();
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.011"), new Text("11"), new Text("DEBUG"),
                    new Text("First entry message!")));
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.012"), new Text("12"), new Text("INFO"),
                    new Text("Second entry message!")));
    exp1.add(Arrays.asList((Writable) new Text("2016-01-01 23:59:59.013"), new Text("13"), new Text("WARN"),
                    new Text("Third entry message!")));

    assertEquals(exp0, rr.sequenceRecord());
    assertEquals(exp1, rr.sequenceRecord());
    assertFalse(rr.hasNext());

    //Test resetting:
    rr.reset();
    assertEquals(exp0, rr.sequenceRecord());
    assertEquals(exp1, rr.sequenceRecord());
    assertFalse(rr.hasNext());
}
Example 15
Source File: TestCollectionRecordReaders.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCollectionSequenceRecordReader() throws Exception {

    List<List<List<Writable>>> listOfSequences = new ArrayList<>();

    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new IntWritable(0), new IntWritable(1)));
    sequence1.add(Arrays.asList((Writable) new IntWritable(2), new IntWritable(3)));
    listOfSequences.add(sequence1);

    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new IntWritable(4), new IntWritable(5)));
    sequence2.add(Arrays.asList((Writable) new IntWritable(6), new IntWritable(7)));
    listOfSequences.add(sequence2);

    SequenceRecordReader seqRR = new CollectionSequenceRecordReader(listOfSequences);
    assertTrue(seqRR.hasNext());

    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    seqRR.reset();
    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    //Test metadata:
    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> seq = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();

    while (seqRR.hasNext()) {
        SequenceRecord r = seqRR.nextSequence();
        out2.add(r.getSequenceRecord());
        seq.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(listOfSequences, out2);

    List<SequenceRecord> fromMeta = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(seq, fromMeta);
}