Java Code Examples for org.datavec.api.conf.Configuration#setInt()
The following examples show how to use
org.datavec.api.conf.Configuration#setInt() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LibSvmRecordWriterTest.java From DataVec with Apache License 2.0 | 6 votes |
@Test public void testMultilabelRecord() throws Exception { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 9); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); Configuration configReader = new Configuration(); configReader.setInt(LibSvmRecordReader.NUM_FEATURES, 10); configReader.setBoolean(LibSvmRecordReader.MULTILABEL, true); configReader.setInt(LibSvmRecordReader.NUM_LABELS, 4); configReader.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); File inputFile = new ClassPathResource("svmlight/multilabel.txt").getFile(); executeTest(configWriter, configReader, inputFile); }
Example 2
Source File: TfidfRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Test public void testRecordMetaData() throws Exception { TfidfVectorizer vectorizer = new TfidfVectorizer(); Configuration conf = new Configuration(); conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); conf.setBoolean(RecordReader.APPEND_LABEL, true); vectorizer.initialize(conf); TfidfRecordReader reader = new TfidfRecordReader(); File f = testDir.newFolder(); new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f); reader.initialize(conf, new FileSplit(f)); while (reader.hasNext()) { Record record = reader.nextRecord(); assertNotNull(record.getMetaData().getURI()); assertEquals(record.getMetaData().getReaderClass(), TfidfRecordReader.class); } }
Example 3
Source File: TfidfRecordReaderTest.java From DataVec with Apache License 2.0 | 6 votes |
@Test public void testReadRecordFromMetaData() throws Exception { TfidfVectorizer vectorizer = new TfidfVectorizer(); Configuration conf = new Configuration(); conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); conf.setBoolean(RecordReader.APPEND_LABEL, true); vectorizer.initialize(conf); TfidfRecordReader reader = new TfidfRecordReader(); reader.initialize(conf, new FileSplit(new ClassPathResource("labeled").getFile())); Record record = reader.nextRecord(); Record reread = reader.loadFromMetaData(record.getMetaData()); assertEquals(record.getRecord().size(), 2); assertEquals(reread.getRecord().size(), 2); assertEquals(record.getRecord().get(0), reread.getRecord().get(0)); assertEquals(record.getRecord().get(1), reread.getRecord().get(1)); assertEquals(record.getMetaData(), reread.getMetaData()); }
Example 4
Source File: TfidfRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 6 votes |
@Test public void testReadRecordFromMetaData() throws Exception { TfidfVectorizer vectorizer = new TfidfVectorizer(); Configuration conf = new Configuration(); conf.setInt(TfidfVectorizer.MIN_WORD_FREQUENCY, 1); conf.setBoolean(RecordReader.APPEND_LABEL, true); vectorizer.initialize(conf); TfidfRecordReader reader = new TfidfRecordReader(); File f = testDir.newFolder(); new ClassPathResource("datavec-data-nlp/labeled/").copyDirectory(f); reader.initialize(conf, new FileSplit(f)); Record record = reader.nextRecord(); Record reread = reader.loadFromMetaData(record.getMetaData()); assertEquals(record.getRecord().size(), 2); assertEquals(reread.getRecord().size(), 2); assertEquals(record.getRecord().get(0), reread.getRecord().get(0)); assertEquals(record.getRecord().get(1), reread.getRecord().get(1)); assertEquals(record.getMetaData(), reread.getMetaData()); }
Example 5
Source File: LibSvmRecordWriterTest.java From DataVec with Apache License 2.0 | 6 votes |
@Test(expected = NumberFormatException.class) public void nonBinaryMultilabel() throws Exception { List<Writable> record = Arrays.asList((Writable) new IntWritable(0), new IntWritable(1), new IntWritable(2)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN,0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN,1); configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL,true); FileSplit outputSplit = new FileSplit(tempFile); writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } }
Example 6
Source File: SVMLightRecordWriterTest.java From DataVec with Apache License 2.0 | 6 votes |
@Test public void testNonIntegerButValidMultilabel() throws Exception { List<Writable> record = Arrays.asList((Writable) new IntWritable(3), new IntWritable(2), new DoubleWritable(1.0)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 1); configWriter.setBoolean(SVMLightRecordWriter.MULTILABEL, true); FileSplit outputSplit = new FileSplit(tempFile); writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } }
Example 7
Source File: SVMLightRecordWriterTest.java From DataVec with Apache License 2.0 | 5 votes |
@Test public void testNDArrayWritables() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); INDArray arr3 = Nd4j.zeros(3); arr3.putScalar(0, 13); arr3.putScalar(1, 14); arr3.putScalar(2, 15); List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new IntWritable(4)); File tempFile = File.createTempFile("SVMLightRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); String lineOriginal = "13.0,14.0,15.0,4 1:1.0 2:11.0 3:12.0 4:2.0 5:3.0"; try (SVMLightRecordWriter writer = new SVMLightRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setInt(SVMLightRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(SVMLightRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); }
Example 8
Source File: SVMLightRecordReaderTest.java From DataVec with Apache License 2.0 | 5 votes |
@Test(expected = IndexOutOfBoundsException.class) public void testLabelIndexExceedsNumLabels() throws Exception { SVMLightRecordReader rr = new SVMLightRecordReader(); Configuration config = new Configuration(); config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); config.setInt(SVMLightRecordReader.NUM_LABELS, 6); rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile())); rr.next(); }
Example 9
Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test(expected = IndexOutOfBoundsException.class) public void testLabelIndexExceedsNumLabels() throws Exception { LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); config.setInt(LibSvmRecordReader.NUM_LABELS, 6); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); rr.next(); }
Example 10
Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test(expected = IndexOutOfBoundsException.class) public void testFeatureIndexExceedsNumFeatures() throws Exception { LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setInt(LibSvmRecordReader.NUM_FEATURES, 9); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); rr.next(); }
Example 11
Source File: LibSvmRecordWriterTest.java From DataVec with Apache License 2.0 | 5 votes |
@Test public void testNDArrayWritablesZeroIndex() throws Exception { INDArray arr2 = Nd4j.zeros(2); arr2.putScalar(0, 11); arr2.putScalar(1, 12); INDArray arr3 = Nd4j.zeros(3); arr3.putScalar(0, 0); arr3.putScalar(1, 1); arr3.putScalar(2, 0); List<Writable> record = Arrays.asList((Writable) new DoubleWritable(1), new NDArrayWritable(arr2), new IntWritable(2), new DoubleWritable(3), new NDArrayWritable(arr3), new DoubleWritable(1)); File tempFile = File.createTempFile("LibSvmRecordWriter", ".txt"); tempFile.setWritable(true); tempFile.deleteOnExit(); if (tempFile.exists()) tempFile.delete(); String lineOriginal = "1,3 0:1.0 1:11.0 2:12.0 3:2.0 4:3.0"; try (LibSvmRecordWriter writer = new LibSvmRecordWriter()) { Configuration configWriter = new Configuration(); configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_INDEXING, true); // NOT STANDARD! configWriter.setBoolean(LibSvmRecordWriter.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! configWriter.setBoolean(LibSvmRecordWriter.MULTILABEL, true); configWriter.setInt(LibSvmRecordWriter.FEATURE_FIRST_COLUMN, 0); configWriter.setInt(LibSvmRecordWriter.FEATURE_LAST_COLUMN, 3); FileSplit outputSplit = new FileSplit(tempFile); writer.initialize(configWriter,outputSplit,new NumberOfRecordsPartitioner()); writer.write(record); } String lineNew = FileUtils.readFileToString(tempFile).trim(); assertEquals(lineOriginal, lineNew); }
Example 12
Source File: LibSvmRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testBasicRecord() throws IOException, InterruptedException { Map<Integer, List<Writable>> correct = new HashMap<>(); // 7 2:1 4:2 6:3 8:4 10:5 correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7))); // 2 qid:42 1:0.1 2:2 6:6.6 8:80 correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2))); // 33 correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33))); LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); int i = 0; while (rr.hasNext()) { List<Writable> record = rr.next(); assertEquals(correct.get(i), record); i++; } assertEquals(i, correct.size()); }
Example 13
Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0 | 5 votes |
@Test(expected = IndexOutOfBoundsException.class) public void testFeatureIndexExceedsNumFeatures() throws Exception { LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setInt(LibSvmRecordReader.NUM_FEATURES, 9); rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile())); rr.next(); }
Example 14
Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test(expected = IndexOutOfBoundsException.class) public void testLabelIndexExceedsNumLabels() throws Exception { SVMLightRecordReader rr = new SVMLightRecordReader(); Configuration config = new Configuration(); config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); config.setInt(SVMLightRecordReader.NUM_LABELS, 6); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); rr.next(); }
Example 15
Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test public void testBasicRecord() throws IOException, InterruptedException { Map<Integer, List<Writable>> correct = new HashMap<>(); // 7 2:1 4:2 6:3 8:4 10:5 correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), new IntWritable(7))); // 2 qid:42 1:0.1 2:2 6:6.6 8:80 correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, new IntWritable(2))); // 33 correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, new IntWritable(33))); SVMLightRecordReader rr = new SVMLightRecordReader(); Configuration config = new Configuration(); config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/basic.txt").getFile())); int i = 0; while (rr.hasNext()) { List<Writable> record = rr.next(); assertEquals(correct.get(i), record); i++; } assertEquals(i, correct.size()); }
Example 16
Source File: JDBCRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 5 votes |
@Test(expected = RuntimeException.class) public void testResetForwardOnlyShouldFail() throws Exception { try (JDBCRecordReader reader = new JDBCRecordReader("SELECT * FROM Coffee", dataSource)) { Configuration conf = new Configuration(); conf.setInt(JDBCRecordReader.JDBC_RESULTSET_TYPE, ResultSet.TYPE_FORWARD_ONLY); reader.initialize(conf, null); reader.next(); reader.reset(); } }
Example 17
Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0 | 5 votes |
@Test(expected = NoSuchElementException.class) public void testNoSuchElementException() throws Exception { LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setInt(LibSvmRecordReader.NUM_FEATURES, 11); rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/basic.txt").getFile())); while (rr.hasNext()) rr.next(); rr.next(); }
Example 18
Source File: SVMLightRecordReaderTest.java From deeplearning4j with Apache License 2.0 | 4 votes |
@Test public void testZeroBasedIndexing() throws IOException, InterruptedException { Map<Integer, List<Writable>> correct = new HashMap<>(); // 1,3 2:1 4:2 6:3 8:4 10:5 correct.put(0, Arrays.asList(ZERO, ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); // 2 qid:42 1:0.1 2:2 6:6.6 8:80 correct.put(1, Arrays.asList(ZERO, new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); // 1,2,4 correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); // 1:1.0 correct.put(3, Arrays.asList(ZERO, new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); // correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); SVMLightRecordReader rr = new SVMLightRecordReader(); Configuration config = new Configuration(); // Zero-based indexing is default config.setBoolean(SVMLightRecordReader.ZERO_BASED_LABEL_INDEXING, true); // NOT STANDARD! config.setInt(SVMLightRecordReader.NUM_FEATURES, 11); config.setBoolean(SVMLightRecordReader.MULTILABEL, true); config.setInt(SVMLightRecordReader.NUM_LABELS, 5); rr.initialize(config, new FileSplit(new ClassPathResource("datavec-api/svmlight/multilabel.txt").getFile())); int i = 0; while (rr.hasNext()) { List<Writable> record = rr.next(); assertEquals(correct.get(i), record); i++; } assertEquals(i, correct.size()); }
Example 19
Source File: SVMLightRecordReaderTest.java From DataVec with Apache License 2.0 | 4 votes |
@Test public void testNoLabel() throws IOException, InterruptedException { Map<Integer, List<Writable>> correct = new HashMap<>(); // 2:1 4:2 6:3 8:4 10:5 correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5))); // qid:42 1:0.1 2:2 6:6.6 8:80 correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO)); // 1:1.0 correct.put(2, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); // correct.put(3, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO)); SVMLightRecordReader rr = new SVMLightRecordReader(); Configuration config = new Configuration(); config.setBoolean(SVMLightRecordReader.ZERO_BASED_INDEXING, false); config.setInt(SVMLightRecordReader.NUM_FEATURES, 10); config.setBoolean(SVMLightRecordReader.APPEND_LABEL, true); rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/noLabels.txt").getFile())); int i = 0; while (rr.hasNext()) { List<Writable> record = rr.next(); assertEquals(correct.get(i), record); i++; } assertEquals(i, correct.size()); }
Example 20
Source File: LibSvmRecordReaderTest.java From DataVec with Apache License 2.0 | 4 votes |
@Test public void testMultilabelRecord() throws IOException, InterruptedException { Map<Integer, List<Writable>> correct = new HashMap<>(); // 1,3 2:1 4:2 6:3 8:4 10:5 correct.put(0, Arrays.asList(ZERO, ONE, ZERO, new DoubleWritable(2), ZERO, new DoubleWritable(3), ZERO, new DoubleWritable(4), ZERO, new DoubleWritable(5), LABEL_ONE, LABEL_ZERO, LABEL_ONE, LABEL_ZERO)); // 2 qid:42 1:0.1 2:2 6:6.6 8:80 correct.put(1, Arrays.asList(new DoubleWritable(0.1), new DoubleWritable(2), ZERO, ZERO, ZERO, new DoubleWritable(6.6), ZERO, new DoubleWritable(80), ZERO, ZERO, LABEL_ZERO, LABEL_ONE, LABEL_ZERO, LABEL_ZERO)); // 1,2,4 correct.put(2, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ONE, LABEL_ONE, LABEL_ZERO, LABEL_ONE)); // 1:1.0 correct.put(3, Arrays.asList(new DoubleWritable(1.0), ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); // correct.put(4, Arrays.asList(ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO, LABEL_ZERO)); LibSvmRecordReader rr = new LibSvmRecordReader(); Configuration config = new Configuration(); config.setBoolean(LibSvmRecordReader.ZERO_BASED_INDEXING, false); config.setBoolean(LibSvmRecordReader.APPEND_LABEL, true); config.setInt(LibSvmRecordReader.NUM_FEATURES, 10); config.setBoolean(LibSvmRecordReader.MULTILABEL, true); config.setInt(LibSvmRecordReader.NUM_LABELS, 4); rr.initialize(config, new FileSplit(new ClassPathResource("svmlight/multilabel.txt").getFile())); int i = 0; while (rr.hasNext()) { List<Writable> record = rr.next(); assertEquals(correct.get(i), record); i++; } assertEquals(i, correct.size()); }