org.datavec.api.conf.Configuration Java Examples

@Test(expected = NumberFormatException.class)
public void nonIntegerMultilabel() throws Exception {
    List<Writable> record = Arrays.asList((Writable) new IntWritable(3),
                                            new IntWritable(2),
                                            new DoubleWritable(1.2));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 1);
        configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }
}
 

@Test
public void testReadRecordFromMetaData() throws Exception {
    TfidfVectorizer vectorizer = new TfidfVectorizer();
    Configuration conf = new Configuration();
    conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1);
    conf.setBoolean(RecordReader.APPEND_LABEL, true);
    vectorizer.initialize(conf);
    TfidfRecordReader reader = new TfidfRecordReader();
    reader.initialize(conf, new FileSplit(new ClassPathResource("labeled").getFile()));

    Record record = reader.nextRecord();

    Record reread = reader.loadFromMetaData(record.getMetaData());

    assertEquals(record.getRecord().size(), 2);
    assertEquals(reread.getRecord().size(), 2);
    assertEquals(record.getRecord().get(0), reread.getRecord().get(0));
    assertEquals(record.getRecord().get(1), reread.getRecord().get(1));
    assertEquals(record.getMetaData(), reread.getMetaData());
}
 

@Test(expected = NumberFormatException.class)
public void nonBinaryMultilabel() throws Exception {
    List<Writable> record = Arrays.asList((Writable) new IntWritable(0),
            new IntWritable(1),
            new IntWritable(2));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN,0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN,1);
        configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL,true);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }
}
 

@Test(expected = NumberFormatException.class)
public void nonIntegerMultilabel() throws Exception {
    List<Writable> record = Arrays.asList((Writable) new IntWritable(3),
                                            new IntWritable(2),
                                            new DoubleWritable(1.2));
    File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1);
        configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }
}
 

@Test
public void testZeroBasedIndexing() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true);
    configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 10);
    configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true);

    Configuration configReader = new Configuration();
    configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 11);
    configReader.setBoolean(LibSvmRecordReader.MULTILABEL, true);
    configReader.setInt(LibSvmRecordReader.NUM_LABELS, 5);

    File inputFile = new ClassPathResource("svmlight/multilabel.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}
 

@Test
public void testZeroBasedIndexing() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setBoolean(SVMLightRecordWriter.ZERO_BASED_INDEXING, true);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 10);
    configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true);

    Configuration configReader = new Configuration();
    configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 11);
    configReader.setBoolean(SVMLightRecordReader.MULTILABEL, true);
    configReader.setInt(SVMLightRecordReader.NUM_LABELS, 5);

    File inputFile = new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}
 

@Override
public void initialize(Configuration conf) {
    tokenizerFactory = createTokenizerFactory(conf);
    minWordFrequency = conf.getInt(MIN_WORD_FREQUENCY, 5);
    if(conf.get(STOP_WORDS) != null)
        stopWords = conf.getStringCollection(STOP_WORDS);
    if (stopWords == null)
        stopWords = StopWords.getStopWords();

    String clazz = conf.get(VOCAB_CACHE, DefaultVocabCache.class.getName());
    try {
        Class<? extends VocabCache> tokenizerFactoryClazz = (Class<? extends VocabCache>) Class.forName(clazz);
        cache = tokenizerFactoryClazz.newInstance();
        cache.initialize(conf);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public void setConf(Configuration conf) {
    super.setConf(conf);
    startFrame = conf.getInt(START_FRAME, 0);
    numFrames = conf.getInt(TOTAL_FRAMES, -1);
    rows = conf.getInt(ROWS, 28);
    cols = conf.getInt(COLUMNS, 28);
    framesPerSecond = conf.getFloat(TIME_SLICE, -1);
    videoLength = conf.getFloat(VIDEO_DURATION, -1);
    ravel = conf.getBoolean(RAVEL, false);
    totalFrames = conf.getInt(TOTAL_FRAMES, -1);
}
 

@Test(expected = UnsupportedOperationException.class)
public void failedToSetNumMultiabelsException() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/multilabel.txt").getFile()));
    while (rr.hasNext())
        rr.next();
}
 

@Override
public void initialize(InputSplit inputSplit, Partitioner partitioner) throws Exception {
    this.conf = new Configuration();
    this.partitioner = partitioner;
    partitioner.init(inputSplit);
    out = new DataOutputStream(partitioner.currentOutputStream());
    initPoi();


}
 

@Test
public void testBasicRecord() throws IOException, InterruptedException {
    Map<Integer, List<Writable>> correct = new HashMap<>();
    // 7 2:1 4:2 6:3 8:4 10:5
    correct.put(0, Arrays.asList(ZERO, ONE,
                                ZERO, new DoubleWritable(2),
                                ZERO, new DoubleWritable(3),
                                ZERO, new DoubleWritable(4),
                                ZERO, new DoubleWritable(5),
                                new IntWritable(7)));
    // 2 qid:42 1:0.1 2:2 6:6.6 8:80
    correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2),
                                ZERO, ZERO,
                                ZERO, new DoubleWritable(6.6),
                                ZERO, new DoubleWritable(80),
                                ZERO, ZERO,
                                new IntWritable(2)));
    // 33
    correct.put(2, Arrays.asList(ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                ZERO, ZERO,
                                new IntWritable(33)));

    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true);
    config.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile()));
    int i = 0;
    while (rr.hasNext()) {
        List<Writable> record = rr.next();
        assertEquals(correct.get(i), record);
        i++;
    }
    assertEquals(i, correct.size());
}
 

public static void executeTest(Configuration configWriter, Configuration configReader, File inputFile) throws Exception {
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
       FileSplit outputSplit = new FileSplit(tempFile);
       writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        LibSvmRecordReader rr = new LibSvmRecordReader();
        rr.initialize(configReader, new FileSplit(inputFile));
        while (rr.hasNext()) {
            List<Writable> record = rr.next();
            writer.write(record);
        }
    }

    Pattern p = Pattern.compile(String.format("%s:\\d+ ", LibSvmRecordReader.QID_PREFIX));
    List<String> linesOriginal = new ArrayList<>();
    for (String line : FileUtils.readLines(inputFile)) {
        if (!line.startsWith(LibSvmRecordReader.COMMENT_CHAR)) {
            String lineClean = line.split(LibSvmRecordReader.COMMENT_CHAR, 2)[0];
            if (lineClean.startsWith(" ")) {
                lineClean = " " + lineClean.trim();
            } else {
                lineClean = lineClean.trim();
            }
            Matcher m = p.matcher(lineClean);
            lineClean = m.replaceAll("");
            linesOriginal.add(lineClean);
        }
    }
    List<String> linesNew = FileUtils.readLines(tempFile);
    assertEquals(linesOriginal, linesNew);
}
 

/**
 * <p>
 * Serializations are found by reading the <code>io.serializations</code>
 * property from <code>conf</code>, which is a comma-delimited list of
 * classnames.
 * </p>
 */
public SerializationFactory(Configuration conf) {
    super(conf);
    for (String serializerName : conf.getStrings("io.serializations",
                    new String[] {"org.apache.hadoop.io.serializer.WritableSerialization"})) {
        add(conf, serializerName);
    }
}
 

@Test(expected = UnsupportedOperationException.class)
public void testInconsistentNumLabelsException() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/inconsistentNumLabels.txt").getFile()));
    while (rr.hasNext())
        rr.next();
}
 

@Test
public void testBasic() throws Exception {
    Configuration configWriter = new Configuration();

    Configuration configReader = new Configuration();
    configReader.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    configReader.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);

    File inputFile = new ClassPathResource("datavec-api/svmlight/basic.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}
 

@Test(expected = UnsupportedOperationException.class)
public void testInconsistentNumLabelsException() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/inconsistentNumLabels.txt").getFile()));
    while (rr.hasNext())
        rr.next();
}
 

@Test(expected = UnsupportedOperationException.class)
public void testInconsistentNumMultiabelsException() throws Exception {
    LibSvmRecordReader rr = new LibSvmRecordReader();
    Configuration config = new Configuration();
    config.setBoolean(LibSvmRecordReader.MULTILABEL, false);
    config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/multilabel.txt").getFile()));
    while (rr.hasNext())
        rr.next();
}
 

@Test
public void testMiniBatches() throws Exception {
    log.info("Setting up Spark Context...");
    JavaRDD<String> lines = sc.textFile(new ClassPathResource("svmLight/iris_svmLight_0.txt")
                    .getTempFileFromArchive().toURI().toString()).cache();
    long count = lines.count();
    assertEquals(300, count);
    // gotta map this to a Matrix/INDArray
    RecordReader rr = new SVMLightRecordReader();
    Configuration c = new Configuration();
    c.set(SVMLightRecordReader.NUM_FEATURES, "5");
    rr.setConf(c);
    JavaRDD<DataSet> points = lines.map(new RecordReaderFunction(rr, 4, 3)).cache();
    count = points.count();
    assertEquals(300, count);

    List<DataSet> collect = points.collect();

    points = points.repartition(1);
    JavaRDD<DataSet> miniBatches = new RDDMiniBatches(10, points).miniBatchesJava();
    count = miniBatches.count();
    List<DataSet> list = miniBatches.collect();
    assertEquals(30, count);    //Expect exactly 30 from 1 partition... could be more for multiple input partitions

    lines.unpersist();
    points.unpersist();
    miniBatches.map(new DataSetAssertionFunction());
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testZeroIndexLabelWithoutUsingZeroIndexing() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 10);
    config.setBoolean(SVMLightRecordReader.MULTILABEL, true);
    config.setInt(SVMLightRecordReader.NUM_LABELS, 2);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/zeroIndexLabel.txt").getFile()));
    rr.next();
}
 

/**
 * <p>
 * Serializations are found by reading the <code>io.serializations</code>
 * property from <code>conf</code>, which is a comma-delimited list of
 * classnames.
 * </p>
 */
public SerializationFactory(Configuration conf) {
    super(conf);
    for (String serializerName : conf.getStrings("io.serializations",
                    new String[] {"org.apache.hadoop.io.serializer.WritableSerialization"})) {
        add(conf, serializerName);
    }
}
 

@Test
public void testNDArrayWritablesMultilabel() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 0);
    arr3.putScalar(1, 1);
    arr3.putScalar(2, 0);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
            new NDArrayWritable(arr2),
            new IntWritable(2),
            new DoubleWritable(3),
            new NDArrayWritable(arr3),
            new DoubleWritable(1));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "2,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0";

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}
 

@Test
public void testMultioutputRecord() throws Exception {
    Configuration configWriter = new Configuration();
    configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
    configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9);

    Configuration configReader = new Configuration();
    configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10);
    configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false);

    File inputFile = new ClassPathResource("datavec-api/svmlight/multioutput.txt").getFile();
    executeTest(configWriter, configReader, inputFile);
}
 

@Test
public void testReader() throws Exception {
    TfidfVectorizer vectorizer = new TfidfVectorizer();
    Configuration conf = new Configuration();
    conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1);
    conf.setBoolean(RecordReader.APPEND_LABEL, true);
    vectorizer.initialize(conf);
    TfidfRecordReader reader = new TfidfRecordReader();
    File f = testDir.newFolder();
    new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f);
    List<URI> u = new ArrayList<>();
    for(File f2 : f.listFiles()){
        if(f2.isDirectory()){
            for(File f3 : f2.listFiles()){
                u.add(f3.toURI());
            }
        } else {
            u.add(f2.toURI());
        }
    }
    Collections.sort(u);
    CollectionInputSplit c = new CollectionInputSplit(u);
    reader.initialize(conf, c);
    int count = 0;
    int[] labelAssertions = new int[3];
    while (reader.hasNext()) {
        Collection<Writable> record = reader.next();
        Iterator<Writable> recordIter = record.iterator();
        NDArrayWritable writable = (NDArrayWritable) recordIter.next();
        labelAssertions[count] = recordIter.next().toInt();
        count++;
    }

    assertArrayEquals(new int[] {0, 1, 2}, labelAssertions);
    assertEquals(3, reader.getLabels().size());
    assertEquals(3, count);
}
 

@Test(expected = NoSuchElementException.class)
public void testNoSuchElementException() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 11);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile()));
    while (rr.hasNext())
        rr.next();
    rr.next();
}
 

@Test
public void testConfigurationDataSourceInitialization() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true");
        conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName);
        reader.initialize(conf, null);
        assertTrue(reader.hasNext());
    }
}
 

@Test
public void testNDArrayWritables() throws Exception {
    INDArray arr2 = Nd4j.zeros(2);
    arr2.putScalar(0, 11);
    arr2.putScalar(1, 12);
    INDArray arr3 = Nd4j.zeros(3);
    arr3.putScalar(0, 13);
    arr3.putScalar(1, 14);
    arr3.putScalar(2, 15);
    List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1),
                                        new NDArrayWritable(arr2),
                                        new IntWritable(2),
                                        new DoubleWritable(3),
                                        new NDArrayWritable(arr3),
                                        new IntWritable(4));
    File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt");
    tempFile.setWritable(true);
    tempFile.deleteOnExit();
    if (tempFile.exists())
        tempFile.delete();

    String lineOriginal = "13.0,14.0,15.0,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0";

    try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) {
        Configuration configWriter = new Configuration();
        configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0);
        configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3);
        FileSplit outputSplit = new FileSplit(tempFile);
        writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner());
        writer.write(record);
    }

    String lineNew = FileUtils.readFileToString(tempFile).trim();
    assertEquals(lineOriginal, lineNew);
}
 

/**
 * Set DataVec configuration
 *
 * @param conf
 */
@Override
public void setConf(Configuration conf) {
    super.setConf(conf);
    featureFirstColumn = conf.getInt(FEATURE_FIRST_COLUMN, 0);
    hasLabel = conf.getBoolean(HAS_LABELS, true);
    multilabel = conf.getBoolean(MULTILABEL, false);
    labelFirstColumn = conf.getInt(LABEL_FIRST_COLUMN, -1);
    labelLastColumn = conf.getInt(LABEL_LAST_COLUMN, -1);
    featureLastColumn = conf.getInt(FEATURE_LAST_COLUMN, labelFirstColumn > 0 ? labelFirstColumn-1 : -1);
    zeroBasedIndexing = conf.getBoolean(ZERO_BASED_INDEXING, false);
    zeroBasedLabelIndexing = conf.getBoolean(ZERO_BASED_LABEL_INDEXING, false);
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testFeatureIndexExceedsNumFeatures() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 9);
    rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile()));
    rr.next();
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testFeatureIndexExceedsNumFeatures() throws Exception {
    SVMLightRecordReader rr = new SVMLightRecordReader();
    Configuration config = new Configuration();
    config.setInt(SVMLightRecordReader.NUM_FEATURES, 9);
    rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile()));
    rr.next();
}
 

@Test
public void testConfigurationDataSourceInitialization() throws Exception {
    try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee")) {
        Configuration conf = new Configuration();
        conf.set(JDBCRecordReader.JDBC_URL, "jdbc:derby:" + dbName + ";create=true");
        conf.set(JDBCRecordReader.JDBC_DRIVER_CLASS_NAME, driverClassName);
        reader.initialize(conf, null);
        assertTrue(reader.hasNext());
    }
}