org.datavec.arrow.recordreader.ArrowRecordReader Java Examples

The following examples show how to use org.datavec.arrow.recordreader.ArrowRecordReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ArrowConverterTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRecordReaderAndWriteFile() throws Exception {
    val recordsToWrite = recordToWrite();
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream);
    byte[] arr = byteArrayOutputStream.toByteArray();
    val read = ArrowConverter.readFromBytes(arr);
    assertEquals(recordsToWrite,read);

    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();

    recordReader.initialize(new FileSplit(tmp));

    List<Writable> record = recordReader.next();
    assertEquals(2,record.size());

}
Example #2
Source File: ArrowConverterTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRecordReaderAndWriteFile() throws Exception {
    val recordsToWrite = recordToWrite();
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream);
    byte[] arr = byteArrayOutputStream.toByteArray();
    val read = ArrowConverter.readFromBytes(arr);
    assertEquals(recordsToWrite,read);

    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();

    recordReader.initialize(new FileSplit(tmp));

    List<Writable> record = recordReader.next();
    assertEquals(2,record.size());

}
Example #3
Source File: ArrowBinaryInputAdapter.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
@Override
public ArrowWritableRecordBatch convert(Buffer input, ConverterArgs parameters, Map<String, Object> contextData) {
    ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
    arrowRecordReader.initialize(new InputStreamInputSplit(new ByteArrayInputStream(input.getBytes())));
    arrowRecordReader.next();
    return arrowRecordReader.getCurrentBatch();
}
Example #4
Source File: RecordMapperTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCopyFromCsvToArrow() throws Exception {
    val recordsPair = records();

    Path p = Files.createTempFile("csvwritetest", ".csv");
    FileUtils.write(p.toFile(),recordsPair.getFirst());
    p.toFile().deleteOnExit();


    CSVRecordReader recordReader = new CSVRecordReader();
    FileSplit fileSplit = new FileSplit(p.toFile());

    ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle());
    File outputFile = Files.createTempFile("outputarrow","arrow").toFile();
    FileSplit outputFileSplit = new FileSplit(outputFile);
    RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(fileSplit)
            .outputUrl(outputFileSplit).partitioner(new NumberOfRecordsPartitioner())
            .recordReader(recordReader).recordWriter(arrowRecordWriter)
            .build();
    mapper.copy();

    ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
    arrowRecordReader.initialize(outputFileSplit);
    List<List<Writable>> next = arrowRecordReader.next(10);
    System.out.println(next);
    assertEquals(10,next.size());

}
Example #5
Source File: ArrowConverterTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreateNDArray() throws Exception {
    val recordsToWrite = recordToWrite();
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream);

    File tmpFile = new File("tmp-arrow-file-" + UUID.randomUUID().toString() + ".arrorw");
    FileOutputStream outputStream = new FileOutputStream(tmpFile);
    tmpFile.deleteOnExit();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),outputStream);
    outputStream.flush();
    outputStream.close();

    Pair<Schema, ArrowWritableRecordBatch> schemaArrowWritableRecordBatchPair = ArrowConverter.readFromFile(tmpFile);
    assertEquals(recordsToWrite.getFirst(),schemaArrowWritableRecordBatchPair.getFirst());
    assertEquals(recordsToWrite.getRight(),schemaArrowWritableRecordBatchPair.getRight().toArrayList());

    byte[] arr = byteArrayOutputStream.toByteArray();
    val read = ArrowConverter.readFromBytes(arr);
    assertEquals(recordsToWrite,read);

    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    ArrowRecordReader recordReader = new ArrowRecordReader();

    recordReader.initialize(new FileSplit(tmp));

    recordReader.next();
    ArrowWritableRecordBatch currentBatch = recordReader.getCurrentBatch();
    INDArray arr2 = ArrowConverter.toArray(currentBatch);
    assertEquals(2,arr2.rows());
    assertEquals(2,arr2.columns());
}
Example #6
Source File: ArrowConverterTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRecordReaderMetaDataList() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(Arrays.<RecordMetaData>asList(recordMetaDataIndex));

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());

}
Example #7
Source File: ArrowConverterTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRecordReaderMetaData() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(recordMetaDataIndex);

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());
}
Example #8
Source File: RecordMapperTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCopyFromCsvToArrow() throws Exception {
    val recordsPair = records();

    Path p = Files.createTempFile("csvwritetest", ".csv");
    FileUtils.write(p.toFile(),recordsPair.getFirst());
    p.toFile().deleteOnExit();


    CSVRecordReader recordReader = new CSVRecordReader();
    FileSplit fileSplit = new FileSplit(p.toFile());

    ArrowRecordWriter arrowRecordWriter = new ArrowRecordWriter(recordsPair.getMiddle());
    File outputFile = Files.createTempFile("outputarrow","arrow").toFile();
    FileSplit outputFileSplit = new FileSplit(outputFile);
    RecordMapper mapper = RecordMapper.builder().batchSize(10).inputUrl(fileSplit)
            .outputUrl(outputFileSplit).partitioner(new NumberOfRecordsPartitioner())
            .recordReader(recordReader).recordWriter(arrowRecordWriter)
            .build();
    mapper.copy();

    ArrowRecordReader arrowRecordReader = new ArrowRecordReader();
    arrowRecordReader.initialize(outputFileSplit);
    List<List<Writable>> next = arrowRecordReader.next(10);
    System.out.println(next);
    assertEquals(10,next.size());

}
Example #9
Source File: ArrowConverterTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCreateNDArray() throws Exception {
    val recordsToWrite = recordToWrite();
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),byteArrayOutputStream);

    File f = testDir.newFolder();

    File tmpFile = new File(f, "tmp-arrow-file-" + UUID.randomUUID().toString() + ".arrorw");
    FileOutputStream outputStream = new FileOutputStream(tmpFile);
    tmpFile.deleteOnExit();
    ArrowConverter.writeRecordBatchTo(recordsToWrite.getRight(),recordsToWrite.getFirst(),outputStream);
    outputStream.flush();
    outputStream.close();

    Pair<Schema, ArrowWritableRecordBatch> schemaArrowWritableRecordBatchPair = ArrowConverter.readFromFile(tmpFile);
    assertEquals(recordsToWrite.getFirst(),schemaArrowWritableRecordBatchPair.getFirst());
    assertEquals(recordsToWrite.getRight(),schemaArrowWritableRecordBatchPair.getRight().toArrayList());

    byte[] arr = byteArrayOutputStream.toByteArray();
    val read = ArrowConverter.readFromBytes(arr);
    assertEquals(recordsToWrite,read);

    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    ArrowRecordReader recordReader = new ArrowRecordReader();

    recordReader.initialize(new FileSplit(tmp));

    recordReader.next();
    ArrowWritableRecordBatch currentBatch = recordReader.getCurrentBatch();
    INDArray arr2 = ArrowConverter.toArray(currentBatch);
    assertEquals(2,arr2.rows());
    assertEquals(2,arr2.columns());
}
Example #10
Source File: ArrowConverterTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRecordReaderMetaDataList() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(Arrays.<RecordMetaData>asList(recordMetaDataIndex));

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());

}
Example #11
Source File: ArrowConverterTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRecordReaderMetaData() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(recordMetaDataIndex);

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());
}