Java Code Examples for org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader#initialize()

The following examples show how to use org.datavec.api.records.reader.impl.csv.CSVSequenceRecordReader#initialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CSVSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReset() throws Exception {
    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    int nTests = 5;
    for (int i = 0; i < nTests; i++) {
        seqReader.reset();

        int sequenceCount = 0;
        while (seqReader.hasNext()) {
            List<List<Writable>> sequence = seqReader.sequenceRecord();
            assertEquals(4, sequence.size()); //4 lines, plus 1 header line

            Iterator<List<Writable>> timeStepIter = sequence.iterator();
            int lineCount = 0;
            while (timeStepIter.hasNext()) {
                timeStepIter.next();
                lineCount++;
            }
            sequenceCount++;
            assertEquals(4, lineCount);
        }
        assertEquals(3, sequenceCount);
    }
}
Example 2
Source File: CSVSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testReset() throws Exception {
    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    int nTests = 5;
    for (int i = 0; i < nTests; i++) {
        seqReader.reset();

        int sequenceCount = 0;
        while (seqReader.hasNext()) {
            List<List<Writable>> sequence = seqReader.sequenceRecord();
            assertEquals(4, sequence.size()); //4 lines, plus 1 header line

            Iterator<List<Writable>> timeStepIter = sequence.iterator();
            int lineCount = 0;
            while (timeStepIter.hasNext()) {
                timeStepIter.next();
                lineCount++;
            }
            sequenceCount++;
            assertEquals(4, lineCount);
        }
        assertEquals(3, sequenceCount);
    }
}
Example 3
Source File: CSVSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void test() throws Exception {

    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    int sequenceCount = 0;
    while (seqReader.hasNext()) {
        List<List<Writable>> sequence = seqReader.sequenceRecord();
        assertEquals(4, sequence.size()); //4 lines, plus 1 header line

        Iterator<List<Writable>> timeStepIter = sequence.iterator();
        int lineCount = 0;
        while (timeStepIter.hasNext()) {
            List<Writable> timeStep = timeStepIter.next();
            assertEquals(3, timeStep.size());
            Iterator<Writable> lineIter = timeStep.iterator();
            int countInLine = 0;
            while (lineIter.hasNext()) {
                Writable entry = lineIter.next();
                int expValue = 100 * sequenceCount + 10 * lineCount + countInLine;
                assertEquals(String.valueOf(expValue), entry.toString());
                countInLine++;
            }
            lineCount++;
        }
        sequenceCount++;
    }
}
Example 4
Source File: CSVSequenceRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMetaData() throws Exception {
    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    List<List<List<Writable>>> l = new ArrayList<>();
    while (seqReader.hasNext()) {
        List<List<Writable>> sequence = seqReader.sequenceRecord();
        assertEquals(4, sequence.size()); //4 lines, plus 1 header line

        Iterator<List<Writable>> timeStepIter = sequence.iterator();
        int lineCount = 0;
        while (timeStepIter.hasNext()) {
            timeStepIter.next();
            lineCount++;
        }
        assertEquals(4, lineCount);

        l.add(sequence);
    }

    List<SequenceRecord> l2 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    seqReader.reset();
    while (seqReader.hasNext()) {
        SequenceRecord sr = seqReader.nextSequence();
        l2.add(sr);
        meta.add(sr.getMetaData());
    }
    assertEquals(3, l2.size());

    List<SequenceRecord> fromMeta = seqReader.loadSequenceFromMetaData(meta);
    for (int i = 0; i < 3; i++) {
        assertEquals(l.get(i), l2.get(i).getSequenceRecord());
        assertEquals(l.get(i), fromMeta.get(i).getSequenceRecord());
    }
}
Example 5
Source File: CSVSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void test() throws Exception {

    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    int sequenceCount = 0;
    while (seqReader.hasNext()) {
        List<List<Writable>> sequence = seqReader.sequenceRecord();
        assertEquals(4, sequence.size()); //4 lines, plus 1 header line

        Iterator<List<Writable>> timeStepIter = sequence.iterator();
        int lineCount = 0;
        while (timeStepIter.hasNext()) {
            List<Writable> timeStep = timeStepIter.next();
            assertEquals(3, timeStep.size());
            Iterator<Writable> lineIter = timeStep.iterator();
            int countInLine = 0;
            while (lineIter.hasNext()) {
                Writable entry = lineIter.next();
                int expValue = 100 * sequenceCount + 10 * lineCount + countInLine;
                assertEquals(String.valueOf(expValue), entry.toString());
                countInLine++;
            }
            lineCount++;
        }
        sequenceCount++;
    }
}
Example 6
Source File: CSVSequenceRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMetaData() throws Exception {
    CSVSequenceRecordReader seqReader = new CSVSequenceRecordReader(1, ",");
    seqReader.initialize(new TestInputSplit());

    List<List<List<Writable>>> l = new ArrayList<>();
    while (seqReader.hasNext()) {
        List<List<Writable>> sequence = seqReader.sequenceRecord();
        assertEquals(4, sequence.size()); //4 lines, plus 1 header line

        Iterator<List<Writable>> timeStepIter = sequence.iterator();
        int lineCount = 0;
        while (timeStepIter.hasNext()) {
            timeStepIter.next();
            lineCount++;
        }
        assertEquals(4, lineCount);

        l.add(sequence);
    }

    List<SequenceRecord> l2 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    seqReader.reset();
    while (seqReader.hasNext()) {
        SequenceRecord sr = seqReader.nextSequence();
        l2.add(sr);
        meta.add(sr.getMetaData());
    }
    assertEquals(3, l2.size());

    List<SequenceRecord> fromMeta = seqReader.loadSequenceFromMetaData(meta);
    for (int i = 0; i < 3; i++) {
        assertEquals(l.get(i), l2.get(i).getSequenceRecord());
        assertEquals(l.get(i), fromMeta.get(i).getSequenceRecord());
    }
}
Example 7
Source File: TestStreamInputSplit.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCsvSequenceSimple() throws Exception {

    File dir = testDir.newFolder();
    File f1 = new File(dir, "file1.txt");
    File f2 = new File(dir, "file2.txt");

    FileUtils.writeStringToFile(f1, "a,b,c\nd,e,f", StandardCharsets.UTF_8);
    FileUtils.writeStringToFile(f2, "1,2,3", StandardCharsets.UTF_8);

    List<URI> uris = Arrays.asList(f1.toURI(), f2.toURI());

    CSVSequenceRecordReader rr = new CSVSequenceRecordReader();

    TestStreamFunction fn = new TestStreamFunction();
    InputSplit is = new StreamInputSplit(uris, fn);
    rr.initialize(is);

    List<List<List<Writable>>> exp = new ArrayList<>();
    exp.add(Arrays.asList(
            Arrays.<Writable>asList(new Text("a"), new Text("b"), new Text("c")),
            Arrays.<Writable>asList(new Text("d"), new Text("e"), new Text("f"))));
    exp.add(Arrays.asList(
            Arrays.<Writable>asList(new Text("1"), new Text("2"), new Text("3"))));

    List<List<List<Writable>>> act = new ArrayList<>();
    while (rr.hasNext()) {
        act.add(rr.sequenceRecord());
    }

    assertEquals(exp, act);

    //Check that the specified stream loading function was used, not the default:
    assertEquals(uris, fn.calledWithUris);

    rr.reset();
    int count = 0;
    while(rr.hasNext()) {
        count++;
        rr.sequenceRecord();
    }
    assertEquals(2, count);
}
Example 8
Source File: TestStreamInputSplit.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testShuffle() throws Exception {
    File dir = testDir.newFolder();
    File f1 = new File(dir, "file1.txt");
    File f2 = new File(dir, "file2.txt");
    File f3 = new File(dir, "file3.txt");

    FileUtils.writeStringToFile(f1, "a,b,c", StandardCharsets.UTF_8);
    FileUtils.writeStringToFile(f2, "1,2,3", StandardCharsets.UTF_8);
    FileUtils.writeStringToFile(f3, "x,y,z", StandardCharsets.UTF_8);

    List<URI> uris = Arrays.asList(f1.toURI(), f2.toURI(), f3.toURI());

    CSVSequenceRecordReader rr = new CSVSequenceRecordReader();

    TestStreamFunction fn = new TestStreamFunction();
    InputSplit is = new StreamInputSplit(uris, fn, new Random(12345));
    rr.initialize(is);

    List<List<List<Writable>>> act = new ArrayList<>();
    while (rr.hasNext()) {
        act.add(rr.sequenceRecord());
    }

    rr.reset();
    List<List<List<Writable>>> act2 = new ArrayList<>();
    while (rr.hasNext()) {
        act2.add(rr.sequenceRecord());
    }

    rr.reset();
    List<List<List<Writable>>> act3 = new ArrayList<>();
    while (rr.hasNext()) {
        act3.add(rr.sequenceRecord());
    }

    assertEquals(3, act.size());
    assertEquals(3, act2.size());
    assertEquals(3, act3.size());

    /*
    System.out.println(act);
    System.out.println("---------");
    System.out.println(act2);
    System.out.println("---------");
    System.out.println(act3);
    */

    //Check not the same. With this RNG seed, results are different for first 3 resets
    assertNotEquals(act, act2);
    assertNotEquals(act2, act3);
    assertNotEquals(act, act3);
}