Java Code Examples for org.datavec.api.records.reader.SequenceRecordReader#nextSequence()

The following examples show how to use org.datavec.api.records.reader.SequenceRecordReader#nextSequence() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 2
Source File: CSVSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCsvSeqAndNumberedFileSplit() throws Exception {
    File baseDir = tempDir.newFolder();
    //Simple sanity check unit test
    for (int i = 0; i < 3; i++) {
        new org.nd4j.linalg.io.ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(baseDir);
    }

    //Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator
    org.nd4j.linalg.io.ClassPathResource resource = new org.nd4j.linalg.io.ClassPathResource("csvsequence_0.txt");
    String featuresPath = new File(baseDir, "csvsequence_%d.txt").getAbsolutePath();

    SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ",");
    featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2));

    while(featureReader.hasNext()){
        featureReader.nextSequence();
    }

}
Example 3
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith(file.getName()));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 4
Source File: CSVSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCsvSeqAndNumberedFileSplit() throws Exception {
    File baseDir = tempDir.newFolder();
    //Simple sanity check unit test
    for (int i = 0; i < 3; i++) {
        new ClassPathResource(String.format("csvsequence_%d.txt", i)).getTempFileFromArchive(baseDir);
    }

    //Load time series from CSV sequence files; compare to SequenceRecordReaderDataSetIterator
    ClassPathResource resource = new ClassPathResource("csvsequence_0.txt");
    String featuresPath = new File(baseDir, "csvsequence_%d.txt").getAbsolutePath();

    SequenceRecordReader featureReader = new CSVSequenceRecordReader(1, ",");
    featureReader.initialize(new NumberedFileInputSplit(featuresPath, 0, 2));

    while(featureReader.hasNext()){
        featureReader.nextSequence();
    }

}
Example 5
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 6
Source File: CSVNLinesSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCSVNlinesSequenceRecordReaderMetaData() throws Exception {
    int nLinesPerSequence = 10;

    SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
    seqRR.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    CSVRecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    List<List<List<Writable>>> out = new ArrayList<>();
    while (seqRR.hasNext()) {
        List<List<Writable>> next = seqRR.sequenceRecord();
        out.add(next);
    }

    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (seqRR.hasNext()) {
        SequenceRecord seq = seqRR.nextSequence();
        out2.add(seq.getSequenceRecord());
        meta.add(seq.getMetaData());
        out3.add(seq);
    }

    assertEquals(out, out2);

    List<SequenceRecord> out4 = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(out3, out4);
}
Example 7
Source File: RegexRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRegexSequenceRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    String path = new ClassPathResource("/logtestdata/logtestfile0.txt").getFile().toURI().toString();
    path = path.replace("0", "%d");
    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<List<Writable>>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.sequenceRecord());
    }

    assertEquals(2, out.size());
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    while (rr.hasNext()) {
        SequenceRecord seqr = rr.nextSequence();
        out2.add(seqr.getSequenceRecord());
        out3.add(seqr);
        meta.add(seqr.getMetaData());
    }

    List<SequenceRecord> fromMeta = rr.loadSequenceFromMetaData(meta);

    assertEquals(out, out2);
    assertEquals(out3, fromMeta);
}
Example 8
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 9
Source File: CSVNLinesSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCSVNlinesSequenceRecordReaderMetaData() throws Exception {
    int nLinesPerSequence = 10;

    SequenceRecordReader seqRR = new CSVNLinesSequenceRecordReader(nLinesPerSequence);
    seqRR.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    CSVRecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    List<List<List<Writable>>> out = new ArrayList<>();
    while (seqRR.hasNext()) {
        List<List<Writable>> next = seqRR.sequenceRecord();
        out.add(next);
    }

    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (seqRR.hasNext()) {
        SequenceRecord seq = seqRR.nextSequence();
        out2.add(seq.getSequenceRecord());
        meta.add(seq.getMetaData());
        out3.add(seq);
    }

    assertEquals(out, out2);

    List<SequenceRecord> out4 = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(out3, out4);
}
Example 10
Source File: RegexRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRegexSequenceRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    ClassPathResource cpr = new ClassPathResource("datavec-api/logtestdata/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "logtestfile%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 1);

    SequenceRecordReader rr = new RegexSequenceRecordReader(regex, 1);
    rr.initialize(is);

    List<List<List<Writable>>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.sequenceRecord());
    }

    assertEquals(2, out.size());
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    while (rr.hasNext()) {
        SequenceRecord seqr = rr.nextSequence();
        out2.add(seqr.getSequenceRecord());
        out3.add(seqr);
        meta.add(seqr.getMetaData());
    }

    List<SequenceRecord> fromMeta = rr.loadSequenceFromMetaData(meta);

    assertEquals(out, out2);
    assertEquals(out3, fromMeta);
}
Example 11
Source File: TestCollectionRecordReaders.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCollectionSequenceRecordReader() throws Exception {

    List<List<List<Writable>>> listOfSequences = new ArrayList<>();

    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new IntWritable(0), new IntWritable(1)));
    sequence1.add(Arrays.asList((Writable) new IntWritable(2), new IntWritable(3)));
    listOfSequences.add(sequence1);

    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new IntWritable(4), new IntWritable(5)));
    sequence2.add(Arrays.asList((Writable) new IntWritable(6), new IntWritable(7)));
    listOfSequences.add(sequence2);

    SequenceRecordReader seqRR = new CollectionSequenceRecordReader(listOfSequences);
    assertTrue(seqRR.hasNext());

    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    seqRR.reset();
    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    //Test metadata:
    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> seq = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();

    while (seqRR.hasNext()) {
        SequenceRecord r = seqRR.nextSequence();
        out2.add(r.getSequenceRecord());
        seq.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(listOfSequences, out2);

    List<SequenceRecord> fromMeta = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(seq, fromMeta);
}
Example 12
Source File: TestCollectionRecordReaders.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCollectionSequenceRecordReader() throws Exception {

    List<List<List<Writable>>> listOfSequences = new ArrayList<>();

    List<List<Writable>> sequence1 = new ArrayList<>();
    sequence1.add(Arrays.asList((Writable) new IntWritable(0), new IntWritable(1)));
    sequence1.add(Arrays.asList((Writable) new IntWritable(2), new IntWritable(3)));
    listOfSequences.add(sequence1);

    List<List<Writable>> sequence2 = new ArrayList<>();
    sequence2.add(Arrays.asList((Writable) new IntWritable(4), new IntWritable(5)));
    sequence2.add(Arrays.asList((Writable) new IntWritable(6), new IntWritable(7)));
    listOfSequences.add(sequence2);

    SequenceRecordReader seqRR = new CollectionSequenceRecordReader(listOfSequences);
    assertTrue(seqRR.hasNext());

    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    seqRR.reset();
    assertEquals(sequence1, seqRR.sequenceRecord());
    assertEquals(sequence2, seqRR.sequenceRecord());
    assertFalse(seqRR.hasNext());

    //Test metadata:
    seqRR.reset();
    List<List<List<Writable>>> out2 = new ArrayList<>();
    List<SequenceRecord> seq = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();

    while (seqRR.hasNext()) {
        SequenceRecord r = seqRR.nextSequence();
        out2.add(r.getSequenceRecord());
        seq.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(listOfSequences, out2);

    List<SequenceRecord> fromMeta = seqRR.loadSequenceFromMetaData(meta);
    assertEquals(seq, fromMeta);
}