Java Code Examples for org.datavec.api.conf.Configuration#set()

The following examples show how to use org.datavec.api.conf.Configuration#set() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 2
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReader() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    //        System.out.println(record.size());

    Iterator<List<Writable>> it = record.iterator();
    List<Writable> first = it.next();
    //        System.out.println(first);

    //Expected size: 80x46x3
    assertEquals(1, first.size());
    assertEquals(80 * 46 * 3, ((ArrayWritable) first.iterator().next()).length());
}
Example 3
Source File: PartitionerTests.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testInputAddFile() throws Exception {
    Partitioner partitioner = new NumberOfRecordsPartitioner();
    File tmpDir = Files.createTempDir();
    FileSplit fileSplit = new FileSplit(tmpDir);
    assertTrue(fileSplit.needsBootstrapForWrite());
    fileSplit.bootStrapForWrite();
    Configuration configuration = new Configuration();
    configuration.set(NumberOfRecordsPartitioner.RECORDS_PER_FILE_CONFIG,String.valueOf(5));
    partitioner.init(configuration,fileSplit);
    partitioner.updatePartitionInfo(PartitionMetaData.builder().numRecordsUpdated(5).build());
    assertTrue(partitioner.needsNewPartition());
    OutputStream os = partitioner.openNewStream();
    os.close();
    assertNotNull(os);
    //run more than once to ensure output stream creation works properly
    partitioner.updatePartitionInfo(PartitionMetaData.builder().numRecordsUpdated(5).build());
    os = partitioner.openNewStream();
    os.close();
    assertNotNull(os);


}
Example 4
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReader() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    //        System.out.println(record.size());

    Iterator<List<Writable>> it = record.iterator();
    List<Writable> first = it.next();
    //        System.out.println(first);

    //Expected size: 80x46x3
    assertEquals(1, first.size());
    assertEquals(80 * 46 * 3, ((ArrayWritable) first.iterator().next()).length());
}
Example 5
Source File: DataVecConsumer.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
public DataVecConsumer(DataVecEndpoint endpoint, Processor processor) {
    super(endpoint, processor);
    this.endpoint = endpoint;

    try {
        inputFormatClazz = (Class<? extends InputFormat>) Class.forName(endpoint.getInputFormat());
        inputFormat = inputFormatClazz.newInstance();
        marshallerClazz = (Class<? extends DataVecMarshaller>) Class.forName(endpoint.getInputMarshaller());
        marshaller = marshallerClazz.newInstance();
        configuration = new Configuration();
        for (String prop : endpoint.getConsumerProperties().keySet())
            configuration.set(prop, endpoint.getConsumerProperties().get(prop).toString());

    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
Example 6
Source File: DataVecProducer.java    From DataVec with Apache License 2.0 6 votes vote down vote up 
public DataVecProducer(DataVecEndpoint endpoint) {
    super(endpoint);
    if (endpoint.getInputFormat() != null) {
        try {
            inputFormatClazz = (Class<? extends InputFormat>) Class.forName(endpoint.getInputFormat());
            inputFormat = inputFormatClazz.newInstance();
            marshallerClazz = (Class<? extends DataVecMarshaller>) Class.forName(endpoint.getInputMarshaller());
            Class<? extends WritableConverter> converterClazz =
                            (Class<? extends WritableConverter>) Class.forName(endpoint.getWritableConverter());
            writableConverter = converterClazz.newInstance();
            marshaller = marshallerClazz.newInstance();
            configuration = new Configuration();
            for (String prop : endpoint.getConsumerProperties().keySet())
                configuration.set(prop, endpoint.getConsumerProperties().get(prop).toString());

        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

}
Example 7
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith(file.getName()));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 8
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCodecReader() throws Exception {
    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    //        System.out.println(record.size());

    Iterator<List<Writable>> it = record.iterator();
    List<Writable> first = it.next();
    //        System.out.println(first);

    //Expected size: 80x46x3
    assertEquals(1, first.size());
    assertEquals(80 * 46 * 3, ((ArrayWritable) first.iterator().next()).length());
}
Example 9
Source File: JDBCRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigurationDataSourceInitialization() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true");
        conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName);
        reader.initialize(conf, null);
        assertTrue(reader.hasNext());
    }
}
Example 10
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeViaDataInputStream() throws Exception {

    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");

    Configuration conf2 = new Configuration(conf);

    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> expected = reader.sequenceRecord();


    SequenceRecordReader reader2 = new NativeCodecRecordReader();
    reader2.setConf(conf2);

    DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file));
    List<List<Writable>> actual = reader2.sequenceRecord(null, dataInputStream);

    assertEquals(expected, actual);
}
Example 11
Source File: TestPairSequenceRecordReaderBytesFunction.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
private static SequenceRecordReader getReader() {
    SequenceRecordReader seqRR = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "0");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "25");
    conf.set(CodecRecordReader.ROWS, "64");
    conf.set(CodecRecordReader.COLUMNS, "64");
    seqRR.setConf(conf);
    return seqRR;
}
Example 12
Source File: CodecReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testViaDataInputStream() throws Exception {

    File file = new ClassPathResource("datavec-data-codec/fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");

    Configuration conf2 = new Configuration(conf);

    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> expected = reader.sequenceRecord();


    SequenceRecordReader reader2 = new CodecRecordReader();
    reader2.setConf(conf2);

    DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file));
    List<List<Writable>> actual = reader2.sequenceRecord(null, dataInputStream);

    assertEquals(expected, actual);
}
Example 13
Source File: TestPairSequenceRecordReaderBytesFunction.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
private static SequenceRecordReader getReader() {
    SequenceRecordReader seqRR = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "0");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "25");
    conf.set(CodecRecordReader.ROWS, "64");
    conf.set(CodecRecordReader.COLUMNS, "64");
    seqRR.setConf(conf);
    return seqRR;
}
Example 14
Source File: MiniBatchTests.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test
public void testMiniBatches() throws Exception {
    log.info("Setting up Spark Context...");
    JavaRDD<String> lines = sc.textFile(new ClassPathResource("svmLight/iris_svmLight_0.txt")
                    .getTempFileFromArchive().toURI().toString()).cache();
    long count = lines.count();
    assertEquals(300, count);
    // gotta map this to a Matrix/INDArray
    RecordReader rr = new SVMLightRecordReader();
    Configuration c = new Configuration();
    c.set(SVMLightRecordReader.NUM_FEATURES, "5");
    rr.setConf(c);
    JavaRDD<DataSet> points = lines.map(new RecordReaderFunction(rr, 4, 3)).cache();
    count = points.count();
    assertEquals(300, count);

    List<DataSet> collect = points.collect();

    points = points.repartition(1);
    JavaRDD<DataSet> miniBatches = new RDDMiniBatches(10, points).miniBatchesJava();
    count = miniBatches.count();
    List<DataSet> list = miniBatches.collect();
    assertEquals(30, count);    //Expect exactly 30 from 1 partition... could be more for multiple input partitions

    lines.unpersist();
    points.unpersist();
    miniBatches.map(new DataSetAssertionFunction());
}
Example 15
Source File: JDBCRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalArgumentException.class)
public void testInitConfigurationMissingParametersShouldFail() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "should fail anyway");
        reader.initialize(conf, null);
    }
}
Example 16
Source File: JDBCRecordReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConfigurationDataSourceInitialization() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true");
        conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName);
        reader.initialize(conf, null);
        assertTrue(reader.hasNext());
    }
}
Example 17
Source File: JDBCRecordReaderTest.java    From deeplearning4j with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalArgumentException.class)
public void testInitConfigurationMissingParametersShouldFail() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "should fail anyway");
        reader.initialize(conf, null);
    }
}
Example 18
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Ignore
@Test
public void testNativeCodecReaderMeta() throws Exception {
    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new NativeCodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");
    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> record = reader.sequenceRecord();
    assertEquals(500, record.size()); //500 frames

    reader.reset();
    SequenceRecord seqR = reader.nextSequence();
    assertEquals(record, seqR.getSequenceRecord());
    RecordMetaData meta = seqR.getMetaData();
    //        System.out.println(meta);
    assertTrue(meta.getURI().toString().endsWith("fire_lowres.mp4"));

    SequenceRecord fromMeta = reader.loadSequenceFromMetaData(meta);
    assertEquals(seqR, fromMeta);
}
Example 19
Source File: CodecReaderTest.java    From DataVec with Apache License 2.0 5 votes vote down vote up 
@Test
public void testViaDataInputStream() throws Exception {

    File file = new ClassPathResource("fire_lowres.mp4").getFile();
    SequenceRecordReader reader = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "160");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "500");
    conf.set(CodecRecordReader.ROWS, "80");
    conf.set(CodecRecordReader.COLUMNS, "46");

    Configuration conf2 = new Configuration(conf);

    reader.initialize(new FileSplit(file));
    reader.setConf(conf);
    assertTrue(reader.hasNext());
    List<List<Writable>> expected = reader.sequenceRecord();


    SequenceRecordReader reader2 = new CodecRecordReader();
    reader2.setConf(conf2);

    DataInputStream dataInputStream = new DataInputStream(new FileInputStream(file));
    List<List<Writable>> actual = reader2.sequenceRecord(null, dataInputStream);

    assertEquals(expected, actual);
}
Example 20
Source File: TestSequenceRecordReaderBytesFunction.java    From deeplearning4j with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRecordReaderBytesFunction() throws Exception {

    //Local file path
    File f = testDir.newFolder();
    new ClassPathResource("datavec-spark/video/").copyDirectory(f);
    String path = f.getAbsolutePath() + "/*";

    //Load binary data from local file system, convert to a sequence file:
    //Load and convert
    JavaPairRDD<String, PortableDataStream> origData = sc.binaryFiles(path);
    JavaPairRDD<Text, BytesWritable> filesAsBytes = origData.mapToPair(new FilesAsBytesFunction());
    //Write the sequence file:
    Path p = Files.createTempDirectory("dl4j_rrbytesTest");
    p.toFile().deleteOnExit();
    String outPath = p.toString() + "/out";
    filesAsBytes.saveAsNewAPIHadoopFile(outPath, Text.class, BytesWritable.class, SequenceFileOutputFormat.class);

    //Load data from sequence file, parse via SequenceRecordReader:
    JavaPairRDD<Text, BytesWritable> fromSeqFile = sc.sequenceFile(outPath, Text.class, BytesWritable.class);
    SequenceRecordReader seqRR = new CodecRecordReader();
    Configuration conf = new Configuration();
    conf.set(CodecRecordReader.RAVEL, "true");
    conf.set(CodecRecordReader.START_FRAME, "0");
    conf.set(CodecRecordReader.TOTAL_FRAMES, "25");
    conf.set(CodecRecordReader.ROWS, "64");
    conf.set(CodecRecordReader.COLUMNS, "64");
    Configuration confCopy = new Configuration(conf);
    seqRR.setConf(conf);
    JavaRDD<List<List<Writable>>> dataVecData = fromSeqFile.map(new SequenceRecordReaderBytesFunction(seqRR));



    //Next: do the same thing locally, and compare the results
    InputSplit is = new FileSplit(f, new String[] {"mp4"}, true);
    SequenceRecordReader srr = new CodecRecordReader();
    srr.initialize(is);
    srr.setConf(confCopy);

    List<List<List<Writable>>> list = new ArrayList<>(4);
    while (srr.hasNext()) {
        list.add(srr.sequenceRecord());
    }
    assertEquals(4, list.size());

    List<List<List<Writable>>> fromSequenceFile = dataVecData.collect();

    assertEquals(4, list.size());
    assertEquals(4, fromSequenceFile.size());

    boolean[] found = new boolean[4];
    for (int i = 0; i < 4; i++) {
        int foundIndex = -1;
        List<List<Writable>> collection = fromSequenceFile.get(i);
        for (int j = 0; j < 4; j++) {
            if (collection.equals(list.get(j))) {
                if (foundIndex != -1)
                    fail(); //Already found this value -> suggests this spark value equals two or more of local version? (Shouldn't happen)
                foundIndex = j;
                if (found[foundIndex])
                    fail(); //One of the other spark values was equal to this one -> suggests duplicates in Spark list
                found[foundIndex] = true; //mark this one as seen before
            }
        }
    }
    int count = 0;
    for (boolean b : found)
        if (b)
            count++;
    assertEquals(4, count); //Expect all 4 and exactly 4 pairwise matches between spark and local versions
}