Java Code Examples for org.datavec.api.records.Record#getMetaData()

The following examples show how to use org.datavec.api.records.Record#getMetaData() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransformProcessRecordReader.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
/**
 * Whether there are anymore records
 *
 * @return
 */
@Override
public boolean hasNext() {
    if(next != null){
        return true;
    }
    if(!recordReader.hasNext()){
        return false;
    }

    //Prefetch, until we find one that isn't filtered out - or we run out of data
    while(next == null && recordReader.hasNext()){
        Record r = recordReader.nextRecord();
        List<Writable> temp = transformProcess.execute(r.getRecord());
        if(temp == null){
            continue;
        }
        next = new org.datavec.api.records.impl.Record(temp, r.getMetaData());
    }

    return next != null;
}
Example 2
Source File: TransformProcessRecordReader.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
/**
 * Whether there are anymore records
 *
 * @return
 */
@Override
public boolean hasNext() {
    if(next != null){
        return true;
    }
    if(!recordReader.hasNext()){
        return false;
    }

    //Prefetch, until we find one that isn't filtered out - or we run out of data
    while(next == null && recordReader.hasNext()){
        Record r = recordReader.nextRecord();
        List<Writable> temp = transformProcess.execute(r.getRecord());
        if(temp == null){
            continue;
        }
        next = new org.datavec.api.records.impl.Record(temp, r.getMetaData());
    }

    return next != null;
}
Example 3
Source File: JDBCRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNextRecord() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        assertNotNull(r);
        assertNotNull(fields);
        assertNotNull(meta);
        assertEquals(new Text("Bolivian Dark"), fields.get(0));
        assertEquals(new Text("14-001"), fields.get(1));
        assertEquals(new DoubleWritable(8.95), fields.get(2));
        assertEquals(RecordMetaDataJdbc.class, meta.getClass());
    }
}
Example 4
Source File: JDBCRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNextRecordAndRecover() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        Record recovered = reader.loadFromMetaData(meta);
        List<Writable> fieldsRecovered = recovered.getRecord();
        assertEquals(fields.size(), fieldsRecovered.size());
        for (int i = 0; i < fields.size(); i++) {
            assertEquals(fields.get(i), fieldsRecovered.get(i));
        }
    }
}
Example 5
Source File: JDBCRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNextRecord() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        assertNotNull(r);
        assertNotNull(fields);
        assertNotNull(meta);
        assertEquals(new Text("Bolivian Dark"), fields.get(0));
        assertEquals(new Text("14-001"), fields.get(1));
        assertEquals(new DoubleWritable(8.95), fields.get(2));
        assertEquals(RecordMetaDataJdbc.class, meta.getClass());
    }
}
Example 6
Source File: JDBCRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNextRecordAndRecover() throws Exception {
    try (JDBCRecordReader reader = getInitializedReader("SELECT * FROM Coffee")) {
        Record r = reader.nextRecord();
        List<Writable> fields = r.getRecord();
        RecordMetaData meta = r.getMetaData();
        Record recovered = reader.loadFromMetaData(meta);
        List<Writable> fieldsRecovered = recovered.getRecord();
        assertEquals(fields.size(), fieldsRecovered.size());
        for (int i = 0; i < fields.size(); i++) {
            assertEquals(fields.get(i), fieldsRecovered.get(i));
        }
    }
}
Example 7
Source File: CSVLineSequenceRecordReader.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
protected SequenceRecord convert(Record r){
    List<Writable> line = r.getRecord();
    List<List<Writable>> out = new ArrayList<>();
    for(Writable w : line){
        out.add(Collections.singletonList(w));
    }
    return new org.datavec.api.records.impl.SequenceRecord(out, r.getMetaData());
}
Example 8
Source File: CSVRecordReaderTest.java    From DataVec with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMeta() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("iris.dat").getFile()));

    int lineCount = 0;
    List<RecordMetaData> metaList = new ArrayList<>();
    List<List<Writable>> writables = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        assertEquals(5, r.getRecord().size());
        lineCount++;
        RecordMetaData meta = r.getMetaData();
        //            System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI());

        metaList.add(meta);
        writables.add(r.getRecord());
    }
    assertFalse(rr.hasNext());
    assertEquals(150, lineCount);
    rr.reset();


    System.out.println("\n\n\n--------------------------------");
    List<Record> contents = rr.loadFromMetaData(metaList);
    assertEquals(150, contents.size());
    //        for(Record r : contents ){
    //            System.out.println(r);
    //        }

    List<RecordMetaData> meta2 = new ArrayList<>();
    meta2.add(metaList.get(100));
    meta2.add(metaList.get(90));
    meta2.add(metaList.get(80));
    meta2.add(metaList.get(70));
    meta2.add(metaList.get(60));

    List<Record> contents2 = rr.loadFromMetaData(meta2);
    assertEquals(writables.get(100), contents2.get(0).getRecord());
    assertEquals(writables.get(90), contents2.get(1).getRecord());
    assertEquals(writables.get(80), contents2.get(2).getRecord());
    assertEquals(writables.get(70), contents2.get(3).getRecord());
    assertEquals(writables.get(60), contents2.get(4).getRecord());
}
Example 9
Source File: CSVRecordReaderTest.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMeta() throws Exception {
    CSVRecordReader rr = new CSVRecordReader(0, ',');
    rr.initialize(new FileSplit(new ClassPathResource("datavec-api/iris.dat").getFile()));

    int lineCount = 0;
    List<RecordMetaData> metaList = new ArrayList<>();
    List<List<Writable>> writables = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        assertEquals(5, r.getRecord().size());
        lineCount++;
        RecordMetaData meta = r.getMetaData();
        //            System.out.println(r.getRecord() + "\t" + meta.getLocation() + "\t" + meta.getURI());

        metaList.add(meta);
        writables.add(r.getRecord());
    }
    assertFalse(rr.hasNext());
    assertEquals(150, lineCount);
    rr.reset();


    System.out.println("\n\n\n--------------------------------");
    List<Record> contents = rr.loadFromMetaData(metaList);
    assertEquals(150, contents.size());
    //        for(Record r : contents ){
    //            System.out.println(r);
    //        }

    List<RecordMetaData> meta2 = new ArrayList<>();
    meta2.add(metaList.get(100));
    meta2.add(metaList.get(90));
    meta2.add(metaList.get(80));
    meta2.add(metaList.get(70));
    meta2.add(metaList.get(60));

    List<Record> contents2 = rr.loadFromMetaData(meta2);
    assertEquals(writables.get(100), contents2.get(0).getRecord());
    assertEquals(writables.get(90), contents2.get(1).getRecord());
    assertEquals(writables.get(80), contents2.get(2).getRecord());
    assertEquals(writables.get(70), contents2.get(3).getRecord());
    assertEquals(writables.get(60), contents2.get(4).getRecord());
}