Java Code Examples for org.datavec.api.records.reader.RecordReader#reset()

private static void testJacksonRecordReader(RecordReader rr) {

        List<Writable> json0 = rr.next();
        List<Writable> exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"));
        assertEquals(exp0, json0);

        List<Writable> json1 = rr.next();
        List<Writable> exp1 =
                        Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"));
        assertEquals(exp1, json1);

        List<Writable> json2 = rr.next();
        List<Writable> exp2 =
                        Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"));
        assertEquals(exp2, json2);

        assertFalse(rr.hasNext());

        //Test reset
        rr.reset();
        assertEquals(exp0, rr.next());
        assertEquals(exp1, rr.next());
        assertEquals(exp2, rr.next());
        assertFalse(rr.hasNext());
    }
 

@Test
public void testRegexLineRecordReader() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    RecordReader rr = new RegexLineRecordReader(regex, 1);
    rr.initialize(new FileSplit(new ClassPathResource("/logtestdata/logtestfile0.txt").getFile()));

    List<Writable> exp0 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"),
                    new Text("DEBUG"), new Text("First entry message!"));
    List<Writable> exp1 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"),
                    new Text("INFO"), new Text("Second entry message!"));
    List<Writable> exp2 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"),
                    new Text("WARN"), new Text("Third entry message!"));
    assertEquals(exp0, rr.next());
    assertEquals(exp1, rr.next());
    assertEquals(exp2, rr.next());
    assertFalse(rr.hasNext());

    //Test reset:
    rr.reset();
    assertEquals(exp0, rr.next());
    assertEquals(exp1, rr.next());
    assertEquals(exp2, rr.next());
    assertFalse(rr.hasNext());
}
 

@Test
public void testRegexLineRecordReader() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    RecordReader rr = new RegexLineRecordReader(regex, 1);
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/logtestdata/logtestfile0.txt").getFile()));

    List<Writable> exp0 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.001"), new Text("1"),
                    new Text("DEBUG"), new Text("First entry message!"));
    List<Writable> exp1 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.002"), new Text("2"),
                    new Text("INFO"), new Text("Second entry message!"));
    List<Writable> exp2 = Arrays.asList((Writable) new Text("2016-01-01 23:59:59.003"), new Text("3"),
                    new Text("WARN"), new Text("Third entry message!"));
    assertEquals(exp0, rr.next());
    assertEquals(exp1, rr.next());
    assertEquals(exp2, rr.next());
    assertFalse(rr.hasNext());

    //Test reset:
    rr.reset();
    assertEquals(exp0, rr.next());
    assertEquals(exp1, rr.next());
    assertEquals(exp2, rr.next());
    assertFalse(rr.hasNext());
}
 

@Test
public void testRegexLineRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    RecordReader rr = new RegexLineRecordReader(regex, 1);
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/logtestdata/logtestfile0.txt").getFile()));

    List<List<Writable>> list = new ArrayList<>();
    while (rr.hasNext()) {
        list.add(rr.next());
    }
    assertEquals(3, list.size());

    List<Record> list2 = new ArrayList<>();
    List<List<Writable>> list3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    int count = 1; //Start by skipping 1 line
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        list2.add(r);
        list3.add(r.getRecord());
        meta.add(r.getMetaData());

        assertEquals(count++, ((RecordMetaDataLine) r.getMetaData()).getLineNumber());
    }

    List<Record> fromMeta = rr.loadFromMetaData(meta);

    assertEquals(list, list3);
    assertEquals(list2, fromMeta);
}
 

@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

@Test
public void testRegexLineRecordReaderMeta() throws Exception {
    String regex = "(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (\\d+) ([A-Z]+) (.*)";

    RecordReader rr = new RegexLineRecordReader(regex, 1);
    rr.initialize(new FileSplit(new ClassPathResource("/logtestdata/logtestfile0.txt").getFile()));

    List<List<Writable>> list = new ArrayList<>();
    while (rr.hasNext()) {
        list.add(rr.next());
    }
    assertEquals(3, list.size());

    List<Record> list2 = new ArrayList<>();
    List<List<Writable>> list3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    rr.reset();
    int count = 1; //Start by skipping 1 line
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        list2.add(r);
        list3.add(r.getRecord());
        meta.add(r.getMetaData());

        assertEquals(count++, ((RecordMetaDataLine) r.getMetaData()).getLineNumber());
    }

    List<Record> fromMeta = rr.loadFromMetaData(meta);

    assertEquals(list, list3);
    assertEquals(list2, fromMeta);
}
 

/**
 * Analyse the specified data - returns a DataAnalysis object with summary information about each column
 *
 * @param schema Schema for data
 * @param rr     Data to analyze
 * @return DataAnalysis for data
 */
public static DataAnalysis analyze(Schema schema, RecordReader rr, int maxHistogramBuckets){
    AnalysisAddFunction addFn = new AnalysisAddFunction(schema);
    List<AnalysisCounter> counters = null;
    while(rr.hasNext()){
        counters = addFn.apply(counters, rr.next());
    }

    double[][] minsMaxes = new double[counters.size()][2];

    List<ColumnType> columnTypes = schema.getColumnTypes();
    List<ColumnAnalysis> list = DataVecAnalysisUtils.convertCounters(counters, minsMaxes, columnTypes);


    //Do another pass collecting histogram values:
    List<HistogramCounter> histogramCounters = null;
    HistogramAddFunction add = new HistogramAddFunction(maxHistogramBuckets, schema, minsMaxes);
    if(rr.resetSupported()){
        rr.reset();
        while(rr.hasNext()){
            histogramCounters = add.apply(histogramCounters, rr.next());
        }

        DataVecAnalysisUtils.mergeCounters(list, histogramCounters);
    }

    return new DataAnalysis(schema, list);
}
 

@Override
public void reset() {
	if (!resetSupported) {
		throw new IllegalStateException("Cannot reset iterator - reset not supported (resetSupported() == false):"
				+ " one or more underlying (sequence) record readers do not support resetting");
	}

	for (RecordReader rr : recordReaders.values())
		rr.reset();
}
 

@Test
public void testAnalysisBasic() throws Exception {

    RecordReader rr = new CSVRecordReader();
    rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getFile()));

    Schema s = new Schema.Builder()
            .addColumnsDouble("0", "1", "2", "3")
            .addColumnInteger("label")
            .build();

    DataAnalysis da = AnalyzeLocal.analyze(s, rr);

    System.out.println(da);

    //Compare:
    List<List<Writable>> list = new ArrayList<>();
    rr.reset();
    while(rr.hasNext()){
        list.add(rr.next());
    }

    INDArray arr = RecordConverter.toMatrix(DataType.DOUBLE, list);
    INDArray mean = arr.mean(0);
    INDArray std = arr.std(0);

    for( int i=0; i<5; i++ ){
        double m = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getMean();
        double stddev = ((NumericalColumnAnalysis)da.getColumnAnalysis().get(i)).getSampleStdev();
        assertEquals(mean.getDouble(i), m, 1e-3);
        assertEquals(std.getDouble(i), stddev, 1e-3);
    }

}
 

@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("datavec-api/json/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "json_test_%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

@Test(timeout = 300000)
public void testIrisFitMultiDataSetIterator() throws Exception {

    RecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(Resources.asFile("iris.txt")));

    MultiDataSetIterator iter = new RecordReaderMultiDataSetIterator.Builder(10).addReader("iris", rr)
            .addInput("iris", 0, 3).addOutputOneHot("iris", 4, 3).build();

    ComputationGraphConfiguration config = new NeuralNetConfiguration.Builder()
            .updater(new Sgd(0.1))
            .graphBuilder().addInputs("in")
            .addLayer("dense", new DenseLayer.Builder().nIn(4).nOut(2).build(), "in").addLayer("out",
                    new OutputLayer.Builder(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).nIn(2).nOut(3)
                            .build(),
                    "dense")
            .setOutputs("out").build();

    ComputationGraph cg = new ComputationGraph(config);
    cg.init();

    cg.fit(iter);


    rr.reset();
    iter = new RecordReaderMultiDataSetIterator.Builder(10).addReader("iris", rr).addInput("iris", 0, 3)
            .addOutputOneHot("iris", 4, 3).build();
    while (iter.hasNext()) {
        cg.fit(iter.next());
    }
}
 

@Test
public void testLocalFilter(){

    List<List<Writable>> in = new ArrayList<>();
    in.add(Arrays.asList(new Text("Keep"), new IntWritable(0)));
    in.add(Arrays.asList(new Text("Remove"), new IntWritable(1)));
    in.add(Arrays.asList(new Text("Keep"), new IntWritable(2)));
    in.add(Arrays.asList(new Text("Remove"), new IntWritable(3)));

    Schema s = new Schema.Builder()
            .addColumnCategorical("cat", "Keep", "Remove")
            .addColumnInteger("int")
            .build();

    TransformProcess tp = new TransformProcess.Builder(s)
            .filter(new CategoricalColumnCondition("cat", ConditionOp.Equal, "Remove"))
            .build();

    RecordReader rr = new CollectionRecordReader(in);
    LocalTransformProcessRecordReader ltprr = new LocalTransformProcessRecordReader(rr, tp);

    List<List<Writable>> out = new ArrayList<>();
    while(ltprr.hasNext()){
        out.add(ltprr.next());
    }

    List<List<Writable>> exp = Arrays.asList(in.get(0), in.get(2));

    assertEquals(exp, out);

    //Check reset:
    ltprr.reset();
    out.clear();
    while(ltprr.hasNext()){
        out.add(ltprr.next());
    }
    assertEquals(exp, out);


    //Also test Record method:
    List<Record> rl = new ArrayList<>();
    rr.reset();
    while(rr.hasNext()){
        rl.add(rr.nextRecord());
    }
    List<Record> exp2 = Arrays.asList(rl.get(0), rl.get(2));

    List<Record> act = new ArrayList<>();
    ltprr.reset();
    while(ltprr.hasNext()){
        act.add(ltprr.nextRecord());
    }
}
 

@Test
    public void testAnalysisVsLocal() throws Exception {

        Schema s = new Schema.Builder()
                .addColumnsDouble("%d", 0, 3)
                .addColumnInteger("label")
                .build();

        RecordReader rr = new CSVRecordReader();
        rr.initialize(new FileSplit(new ClassPathResource("iris.txt").getFile()));

        List<List<Writable>> toParallelize = new ArrayList<>();
        while(rr.hasNext()){
            toParallelize.add(rr.next());
        }

        JavaRDD<List<Writable>> rdd = sc.parallelize(toParallelize).coalesce(1);


        rr.reset();
        DataAnalysis local = AnalyzeLocal.analyze(s, rr);
        DataAnalysis spark = AnalyzeSpark.analyze(s, rdd);

//        assertEquals(local.toJson(), spark.toJson());
        assertEquals(local, spark);


        //Also quality analysis:
        rr.reset();
        DataQualityAnalysis localQ = AnalyzeLocal.analyzeQuality(s, rr);
        DataQualityAnalysis sparkQ = AnalyzeSpark.analyzeQuality(s, rdd);

        assertEquals(localQ, sparkQ);


        //And, check unique etc:
        rr.reset();
        Map<String,Set<Writable>> mapLocal = AnalyzeLocal.getUnique(s.getColumnNames(), s, rr);
        Map<String,List<Writable>> mapSpark = AnalyzeSpark.getUnique(s.getColumnNames(), s, rdd);

        assertEquals(mapLocal.keySet(), mapSpark.keySet());
        for( String k : mapLocal.keySet()){
            assertEquals(mapLocal.get(k), new HashSet<Writable>(mapSpark.get(k)));
        }
    }
 

@Test
public void testLineReaderMetaData() throws Exception {
    File tmpdir = testDir.newFolder();

    File tmp1 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp1.txt"));
    File tmp2 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp2.txt"));
    File tmp3 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp3.txt"));

    FileUtils.writeLines(tmp1, Arrays.asList("1", "2", "3"));
    FileUtils.writeLines(tmp2, Arrays.asList("4", "5", "6"));
    FileUtils.writeLines(tmp3, Arrays.asList("7", "8", "9"));

    InputSplit split = new FileSplit(tmpdir);

    RecordReader reader = new LineRecordReader();
    reader.initialize(split);

    List<List<Writable>> list = new ArrayList<>();
    while (reader.hasNext()) {
        list.add(reader.next());
    }
    assertEquals(9, list.size());


    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    reader.reset();
    int count = 0;
    while (reader.hasNext()) {
        Record r = reader.nextRecord();
        out2.add(r.getRecord());
        out3.add(r);
        meta.add(r.getMetaData());
        int fileIdx = count / 3;
        URI uri = r.getMetaData().getURI();
        assertEquals(uri, split.locations()[fileIdx]);
        count++;
    }

    assertEquals(list, out2);

    List<Record> fromMeta = reader.loadFromMetaData(meta);
    assertEquals(out3, fromMeta);

    //try: second line of second and third files only...
    List<RecordMetaData> subsetMeta = new ArrayList<>();
    subsetMeta.add(meta.get(4));
    subsetMeta.add(meta.get(7));
    List<Record> subset = reader.loadFromMetaData(subsetMeta);
    assertEquals(2, subset.size());
    assertEquals(out3.get(4), subset.get(0));
    assertEquals(out3.get(7), subset.get(1));
}
 

@Override
public void reset() {
    for (RecordReader reader : readers)
        reader.reset();

}
 

@Override
public void reset() {
    for (RecordReader reader : readers)
        reader.reset();
}
 

@Test
public void testLineReaderMetaData() throws Exception {
    String tempDir = System.getProperty("java.io.tmpdir");
    File tmpdir = new File(tempDir, "tmpdir-testLineReader");
    if (tmpdir.exists())
        tmpdir.delete();
    tmpdir.mkdir();

    File tmp1 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp1.txt"));
    File tmp2 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp2.txt"));
    File tmp3 = new File(FilenameUtils.concat(tmpdir.getPath(), "tmp3.txt"));

    FileUtils.writeLines(tmp1, Arrays.asList("1", "2", "3"));
    FileUtils.writeLines(tmp2, Arrays.asList("4", "5", "6"));
    FileUtils.writeLines(tmp3, Arrays.asList("7", "8", "9"));

    InputSplit split = new FileSplit(tmpdir);

    RecordReader reader = new LineRecordReader();
    reader.initialize(split);

    List<List<Writable>> list = new ArrayList<>();
    while (reader.hasNext()) {
        list.add(reader.next());
    }
    assertEquals(9, list.size());


    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> out3 = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    reader.reset();
    int count = 0;
    while (reader.hasNext()) {
        Record r = reader.nextRecord();
        out2.add(r.getRecord());
        out3.add(r);
        meta.add(r.getMetaData());
        int fileIdx = count / 3;
        URI uri = r.getMetaData().getURI();
        assertEquals(uri, split.locations()[fileIdx]);
        count++;
    }

    assertEquals(list, out2);

    List<Record> fromMeta = reader.loadFromMetaData(meta);
    assertEquals(out3, fromMeta);

    //try: second line of second and third files only...
    List<RecordMetaData> subsetMeta = new ArrayList<>();
    subsetMeta.add(meta.get(4));
    subsetMeta.add(meta.get(7));
    List<Record> subset = reader.loadFromMetaData(subsetMeta);
    assertEquals(2, subset.size());
    assertEquals(out3.get(4), subset.get(0));
    assertEquals(out3.get(7), subset.get(1));


    try {
        FileUtils.deleteDirectory(tmpdir);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

@Override
public void reset() {
    for (RecordReader reader : readers)
        reader.reset();

}
 

@Override
public void reset() {
    for (RecordReader reader : readers)
        reader.reset();
}
 

@Test
public void testLocalFilter(){

    List<List<Writable>> in = new ArrayList<>();
    in.add(Arrays.asList(new Text("Keep"), new IntWritable(0)));
    in.add(Arrays.asList(new Text("Remove"), new IntWritable(1)));
    in.add(Arrays.asList(new Text("Keep"), new IntWritable(2)));
    in.add(Arrays.asList(new Text("Remove"), new IntWritable(3)));

    Schema s = new Schema.Builder()
            .addColumnCategorical("cat", "Keep", "Remove")
            .addColumnInteger("int")
            .build();

    TransformProcess tp = new TransformProcess.Builder(s)
            .filter(new CategoricalColumnCondition("cat", ConditionOp.Equal, "Remove"))
            .build();

    RecordReader rr = new CollectionRecordReader(in);
    LocalTransformProcessRecordReader ltprr = new LocalTransformProcessRecordReader(rr, tp);

    List<List<Writable>> out = new ArrayList<>();
    while(ltprr.hasNext()){
        out.add(ltprr.next());
    }

    List<List<Writable>> exp = Arrays.asList(in.get(0), in.get(2));

    assertEquals(exp, out);

    //Check reset:
    ltprr.reset();
    out.clear();
    while(ltprr.hasNext()){
        out.add(ltprr.next());
    }
    assertEquals(exp, out);


    //Also test Record method:
    List<Record> rl = new ArrayList<>();
    rr.reset();
    while(rr.hasNext()){
        rl.add(rr.nextRecord());
    }
    List<Record> exp2 = Arrays.asList(rl.get(0), rl.get(2));

    List<Record> act = new ArrayList<>();
    ltprr.reset();
    while(ltprr.hasNext()){
        act.add(ltprr.nextRecord());
    }
}