Java Code Examples for org.datavec.api.records.reader.impl.csv.CSVRecordReader#reset()

@Test
public void testReset() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    int nResets = 5;
    for (int i = 0; i < nResets; i++) {

        int lineCount = 0;
        while (rr.hasNext()) {
            List<Writable> line = rr.next();
            assertEquals(5, line.size());
            lineCount++;
        }
        assertFalse(rr.hasNext());
        assertEquals(150, lineCount);
        rr.reset();
    }
}
 

@Test
public void testResetWithSkipLines() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(10, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));
    int lineCount = 0;
    while (rr.hasNext()) {
        rr.next();
        ++lineCount;
    }
    assertEquals(140, lineCount);
    rr.reset();
    lineCount = 0;
    while (rr.hasNext()) {
        rr.next();
        ++lineCount;
    }
    assertEquals(140, lineCount);
}
 

@Test(expected = NoSuchElementException.class)
public void testCsvSkipAllLines() throws IOException, InterruptedException {
    final int numLines = 4;
    final List<Writable> lineList = Arrays.asList((Writable) new IntWritable(numLines - 1),
                    (Writable) new Text("one"), (Writable) new Text("two"), (Writable) new Text("three"));
    String header = ",one,two,three";
    List<String> lines = new ArrayList<>();
    for (int i = 0; i < numLines; i++)
        lines.add(Integer.toString(i) + header);
    File tempFile = File.createTempFile("csvSkipLines", ".csv");
    FileUtils.writeLines(tempFile, lines);

    CSVRecordReader rr = new CSVRecordReader(numLines, ',');
    rr.initialize(new FileSplit(tempFile));
    rr.reset();
    assertTrue(!rr.hasNext());
    rr.next();
}
 

@Test
public void testCsvSkipAllButOneLine() throws IOException, InterruptedException {
    final int numLines = 4;
    final List<Writable> lineList = Arrays.<Writable>asList(new Text(Integer.toString(numLines - 1)),
            new Text("one"), new Text("two"), new Text("three"));
    String header = ",one,two,three";
    List<String> lines = new ArrayList<>();
    for (int i = 0; i < numLines; i++)
        lines.add(Integer.toString(i) + header);
    File tempFile = File.createTempFile("csvSkipLines", ".csv");
    FileUtils.writeLines(tempFile, lines);

    CSVRecordReader rr = new CSVRecordReader(numLines - 1, ',');
    rr.initialize(new FileSplit(tempFile));
    rr.reset();
    assertTrue(rr.hasNext());
    assertEquals(rr.next(), lineList);
}
 

@Test
public void testStreamReset() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new InputStreamInputSplit(new ClassPathResource("iris.dat").getInputStream()));

    int count = 0;
    while(rr.hasNext()){
        assertNotNull(rr.next());
        count++;
    }
    assertEquals(150, count);

    assertFalse(rr.resetSupported());

    try{
        rr.reset();
        fail("Expected exception");
    } catch (Exception e){
        e.printStackTrace();
    }
}
 

@Test
public void testReset() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    int nResets = 5;
    for (int i = 0; i < nResets; i++) {

        int lineCount = 0;
        while (rr.hasNext()) {
            List<Writable> line = rr.next();
            assertEquals(5, line.size());
            lineCount++;
        }
        assertFalse(rr.hasNext());
        assertEquals(150, lineCount);
        rr.reset();
    }
}
 

@Test
public void testResetWithSkipLines() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(10, ',');
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));
    int lineCount = 0;
    while (rr.hasNext()) {
        rr.next();
        ++lineCount;
    }
    assertEquals(140, lineCount);
    rr.reset();
    lineCount = 0;
    while (rr.hasNext()) {
        rr.next();
        ++lineCount;
    }
    assertEquals(140, lineCount);
}
 

@Test(expected = NoSuchElementException.class)
public void testCsvSkipAllLines() throws IOException, InterruptedException {
    final int numLines = 4;
    final List<Writable> lineList = Arrays.asList((Writable) new IntWritable(numLines - 1),
                    (Writable) new Text("one"), (Writable) new Text("two"), (Writable) new Text("three"));
    String header = ",one,two,three";
    List<String> lines = new ArrayList<>();
    for (int i = 0; i < numLines; i++)
        lines.add(Integer.toString(i) + header);
    File tempFile = File.createTempFile("csvSkipLines", ".csv");
    FileUtils.writeLines(tempFile, lines);

    CSVRecordReader rr = new CSVRecordReader(numLines, ',');
    rr.initialize(new FileSplit(tempFile));
    rr.reset();
    assertTrue(!rr.hasNext());
    rr.next();
}
 

@Test
public void testCsvSkipAllButOneLine() throws IOException, InterruptedException {
    final int numLines = 4;
    final List<Writable> lineList = Arrays.<Writable>asList(new Text(Integer.toString(numLines - 1)),
            new Text("one"), new Text("two"), new Text("three"));
    String header = ",one,two,three";
    List<String> lines = new ArrayList<>();
    for (int i = 0; i < numLines; i++)
        lines.add(Integer.toString(i) + header);
    File tempFile = File.createTempFile("csvSkipLines", ".csv");
    FileUtils.writeLines(tempFile, lines);

    CSVRecordReader rr = new CSVRecordReader(numLines - 1, ',');
    rr.initialize(new FileSplit(tempFile));
    rr.reset();
    assertTrue(rr.hasNext());
    assertEquals(rr.next(), lineList);
}
 

@Test
    public void testStreamReset() throws Exception {
        CSVRecordReader rr = new CSVRecordReader(0, ',');
        rr.initialize(new InputStreamInputSplit(new ClassPathResource("datavec-api/iris.dat").getInputStream()));

        int count = 0;
        while(rr.hasNext()){
            assertNotNull(rr.next());
            count++;
        }
        assertEquals(150, count);

        assertFalse(rr.resetSupported());

        try{
            rr.reset();
            fail("Expected exception");
        } catch (Exception e){
            String msg = e.getMessage();
            String msg2 = e.getCause().getMessage();
            assertTrue(msg, msg.contains("Error during LineRecordReader reset"));
            assertTrue(msg2, msg2.contains("Reset not supported from streams"));
//            e.printStackTrace();
        }
    }
 

@Test
public void testCsvSimple() throws Exception {
    File dir = testDir.newFolder();
    File f1 = new File(dir, "file1.txt");
    File f2 = new File(dir, "file2.txt");

    FileUtils.writeStringToFile(f1, "a,b,c\nd,e,f", StandardCharsets.UTF_8);
    FileUtils.writeStringToFile(f2, "1,2,3", StandardCharsets.UTF_8);

    List<URI> uris = Arrays.asList(f1.toURI(), f2.toURI());

    CSVRecordReader rr = new CSVRecordReader();

    TestStreamFunction fn = new TestStreamFunction();
    InputSplit is = new StreamInputSplit(uris, fn);
    rr.initialize(is);

    List<List<Writable>> exp = new ArrayList<>();
    exp.add(Arrays.<Writable>asList(new Text("a"), new Text("b"), new Text("c")));
    exp.add(Arrays.<Writable>asList(new Text("d"), new Text("e"), new Text("f")));
    exp.add(Arrays.<Writable>asList(new Text("1"), new Text("2"), new Text("3")));

    List<List<Writable>> act = new ArrayList<>();
    while(rr.hasNext()){
        act.add(rr.next());
    }

    assertEquals(exp, act);

    //Check that the specified stream loading function was used, not the default:
    assertEquals(uris, fn.calledWithUris);

    rr.reset();
    int count = 0;
    while(rr.hasNext()) {
        count++;
        rr.next();
    }
    assertEquals(3, count);
}
 

@Test
public void testMeta() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    int lineCount = 0;
    List<RecordMetaData> metaList = new ArrayList<>();
    List<List<Writable>> writables = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        assertEquals(5, r.getRecord().size());
        lineCount++;
        RecordMetaData meta = r.getMetaData();
        //            System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI());

        metaList.add(meta);
        writables.add(r.getRecord());
    }
    assertFalse(rr.hasNext());
    assertEquals(150, lineCount);
    rr.reset();


    System.out.println("\n\n\n--------------------------------");
    List<Record> contents = rr.loadFromMetaData(metaList);
    assertEquals(150, contents.size());
    //        for(Record r : contents ){
    //            System.out.println(r);
    //        }

    List<RecordMetaData> meta2 = new ArrayList<>();
    meta2.add(metaList.get(100));
    meta2.add(metaList.get(90));
    meta2.add(metaList.get(80));
    meta2.add(metaList.get(70));
    meta2.add(metaList.get(60));

    List<Record> contents2 = rr.loadFromMetaData(meta2);
    assertEquals(writables.get(100), contents2.get(0).getRecord());
    assertEquals(writables.get(90), contents2.get(1).getRecord());
    assertEquals(writables.get(80), contents2.get(2).getRecord());
    assertEquals(writables.get(70), contents2.get(3).getRecord());
    assertEquals(writables.get(60), contents2.get(4).getRecord());
}
 

@Test
public void testMeta() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    int lineCount = 0;
    List<RecordMetaData> metaList = new ArrayList<>();
    List<List<Writable>> writables = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        assertEquals(5, r.getRecord().size());
        lineCount++;
        RecordMetaData meta = r.getMetaData();
        //            System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI());

        metaList.add(meta);
        writables.add(r.getRecord());
    }
    assertFalse(rr.hasNext());
    assertEquals(150, lineCount);
    rr.reset();


    System.out.println("\n\n\n--------------------------------");
    List<Record> contents = rr.loadFromMetaData(metaList);
    assertEquals(150, contents.size());
    //        for(Record r : contents ){
    //            System.out.println(r);
    //        }

    List<RecordMetaData> meta2 = new ArrayList<>();
    meta2.add(metaList.get(100));
    meta2.add(metaList.get(90));
    meta2.add(metaList.get(80));
    meta2.add(metaList.get(70));
    meta2.add(metaList.get(60));

    List<Record> contents2 = rr.loadFromMetaData(meta2);
    assertEquals(writables.get(100), contents2.get(0).getRecord());
    assertEquals(writables.get(90), contents2.get(1).getRecord());
    assertEquals(writables.get(80), contents2.get(2).getRecord());
    assertEquals(writables.get(70), contents2.get(3).getRecord());
    assertEquals(writables.get(60), contents2.get(4).getRecord());
}