org.nd4j.shade.jackson.core.JsonFactory Java Examples

private String toJacksonString(JsonFactory factory) {
    ObjectMapper om = new ObjectMapper(factory);
    om.registerModule(new JodaModule());
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    om.enable(SerializationFeature.INDENT_OUTPUT);
    om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    String str;
    try {
        str = om.writeValueAsString(this);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return str;
}
 

@Test
public void testReadingJson() throws Exception {
    //Load 3 values from 3 JSON files
    //stricture: a:value, b:value, c:x:value, c:y:value
    //And we want to load only a:value, b:value and c:x:value
    //For first JSON file: all values are present
    //For second JSON file: b:value is missing
    //For third JSON file: c:x:value is missing

    ClassPathResource cpr = new ClassPathResource("datavec-api/json/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "json_test_%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
    rr.initialize(is);

    testJacksonRecordReader(rr);
}
 

/**
 *  Writes out all the parameters and their properties (final and resource) to
 *  the given {@link Writer}
 *  The format of the output would be
 *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
 *  key2.isFinal,key2.resource}... ] }
 *  It does not output the parameters of the configuration object which is
 *  loaded from an input stream.
 * @param out the Writer to write to
 * @throws IOException
 */
public static void dumpConfiguration(Configuration conf, Writer out) throws IOException {
    Configuration config = new Configuration(conf, true);
    config.reloadConfiguration();
    JsonFactory dumpFactory = new JsonFactory();
    JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
    dumpGenerator.writeStartObject();
    dumpGenerator.writeFieldName("properties");
    dumpGenerator.writeStartArray();
    dumpGenerator.flush();
    for (Map.Entry<Object, Object> item : config.getProps().entrySet()) {
        dumpGenerator.writeStartObject();
        dumpGenerator.writeStringField("key", (String) item.getKey());
        dumpGenerator.writeStringField("value", config.get((String) item.getKey()));
        dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey()));
        dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey()));
        dumpGenerator.writeEndObject();
    }
    dumpGenerator.writeEndArray();
    dumpGenerator.writeEndObject();
    dumpGenerator.flush();
}
 

private String toJacksonString(JsonFactory factory) {
    ObjectMapper om = new ObjectMapper(factory);
    om.registerModule(new JodaModule());
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    om.enable(SerializationFeature.INDENT_OUTPUT);
    om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    String str;
    try {
        str = om.writeValueAsString(this);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    return str;
}
 

@Test
public void testReadingJson() throws Exception {
    //Load 3 values from 3 JSON files
    //stricture: a:value, b:value, c:x:value, c:y:value
    //And we want to load only a:value, b:value and c:x:value
    //For first JSON file: all values are present
    //For second JSON file: b:value is missing
    //For third JSON file: c:x:value is missing

    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
    rr.initialize(is);

    testJacksonRecordReader(rr);
}
 

/**
 *  Writes out all the parameters and their properties (final and resource) to
 *  the given {@link Writer}
 *  The format of the output would be
 *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
 *  key2.isFinal,key2.resource}... ] }
 *  It does not output the parameters of the configuration object which is
 *  loaded from an input stream.
 * @param out the Writer to write to
 * @throws IOException
 */
public static void dumpConfiguration(Configuration conf, Writer out) throws IOException {
    Configuration config = new Configuration(conf, true);
    config.reloadConfiguration();
    JsonFactory dumpFactory = new JsonFactory();
    JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
    dumpGenerator.writeStartObject();
    dumpGenerator.writeFieldName("properties");
    dumpGenerator.writeStartArray();
    dumpGenerator.flush();
    for (Map.Entry<Object, Object> item : config.getProps().entrySet()) {
        dumpGenerator.writeStartObject();
        dumpGenerator.writeStringField("key", (String) item.getKey());
        dumpGenerator.writeStringField("value", config.get((String) item.getKey()));
        dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey()));
        dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey()));
        dumpGenerator.writeEndObject();
    }
    dumpGenerator.writeEndArray();
    dumpGenerator.writeEndObject();
    dumpGenerator.flush();
}
 

@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

@Test
   public void testReadJSON() throws Exception {
      
       RecordReader rr = new JacksonLineRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
       rr.initialize(new FileSplit(new ClassPathResource("json/json_test_3.txt").getFile()));
       
       testJacksonRecordReader(rr);
}
 

@Test
public void testRR() throws Exception {

    List<RecordReader> rrs = new ArrayList<>();

    rrs.add(new CSVNLinesSequenceRecordReader(10));
    rrs.add(new CSVRecordReader(10, ','));
    rrs.add(new CSVSequenceRecordReader(1, ","));
    rrs.add(new CSVVariableSlidingWindowRecordReader(5));
    rrs.add(new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"}));
    rrs.add(new JacksonRecordReader(new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b")
            .addField(new Text("MISSING_CX"), "c", "x").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new JacksonLineRecordReader(new FieldSelection.Builder().addField("value1")
    		.addField("value2").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new LibSvmRecordReader());
    rrs.add(new SVMLightRecordReader());
    rrs.add(new RegexLineRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new RegexSequenceRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new TransformProcessRecordReader(new CSVRecordReader(), getTp()));
    rrs.add(new TransformProcessSequenceRecordReader(new CSVSequenceRecordReader(), getTp()));
    rrs.add(new LineRecordReader());

    for(RecordReader r : rrs){
        System.out.println(r.getClass().getName());
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ObjectOutputStream os = new ObjectOutputStream(baos);
        os.writeObject(r);
        byte[] bytes = baos.toByteArray();

        ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));

        RecordReader r2 = (RecordReader) ois.readObject();
    }
}
 

protected static ObjectMapper getObjectMapper(JsonFactory factory) {
    ObjectMapper om = new ObjectMapper(factory);
    om.registerModule(new JodaModule());
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    om.enable(SerializationFeature.INDENT_OUTPUT);
    om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    om.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY);
    return om;
}
 

private static Schema fromJacksonString(String str, JsonFactory factory) {
    ObjectMapper om = new ObjectMapper(factory);
    om.registerModule(new JodaModule());
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    om.enable(SerializationFeature.INDENT_OUTPUT);
    om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    try {
        return om.readValue(str, Schema.class);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

protected static ObjectMapper getNewMapper(JsonFactory jsonFactory) {
    ObjectMapper om = new ObjectMapper(jsonFactory);
    om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    om.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    om.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
    om.enable(SerializationFeature.INDENT_OUTPUT);
    om.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    om.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    return om;
}
 

@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("datavec-api/json/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "json_test_%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

@Test
   public void testReadJSON() throws Exception {
      
       RecordReader rr = new JacksonLineRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
       rr.initialize(new FileSplit(new ClassPathResource("datavec-api/json/json_test_3.txt").getFile()));
       
       testJacksonRecordReader(rr);
}
 

@Test
public void testJacksonLineSequenceRecordReader() throws Exception {
	File dir = testDir.newFolder();
	new ClassPathResource("datavec-api/JacksonLineSequenceRecordReaderTest/").copyDirectory(dir);

	FieldSelection f = new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b")
			.addField(new Text("MISSING_CX"), "c", "x").build();

	JacksonLineSequenceRecordReader rr = new JacksonLineSequenceRecordReader(f, new ObjectMapper(new JsonFactory()));
	File[] files = dir.listFiles();
	Arrays.sort(files);
	URI[] u = new URI[files.length];
	for( int i=0; i<files.length; i++ ){
		u[i] = files[i].toURI();
	}
	rr.initialize(new CollectionInputSplit(u));

	List<List<Writable>> expSeq0 = new ArrayList<>();
	expSeq0.add(Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0")));
	expSeq0.add(Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1")));
	expSeq0.add(Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX")));

	List<List<Writable>> expSeq1 = new ArrayList<>();
	expSeq1.add(Arrays.asList((Writable) new Text("aValue3"), new Text("bValue3"), new Text("cxValue3")));


	int count = 0;
	while(rr.hasNext()){
		List<List<Writable>> next = rr.sequenceRecord();
		if(count++ == 0){
			assertEquals(expSeq0, next);
		} else {
			assertEquals(expSeq1, next);
		}
	}

	assertEquals(2, count);
}
 

@Test
public void testRR() throws Exception {

    List<RecordReader> rrs = new ArrayList<>();

    rrs.add(new CSVNLinesSequenceRecordReader(10));
    rrs.add(new CSVRecordReader(10, ','));
    rrs.add(new CSVSequenceRecordReader(1, ","));
    rrs.add(new CSVVariableSlidingWindowRecordReader(5));
    rrs.add(new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"}));
    rrs.add(new JacksonRecordReader(new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b")
            .addField(new Text("MISSING_CX"), "c", "x").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new JacksonLineRecordReader(new FieldSelection.Builder().addField("value1")
    		.addField("value2").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new LibSvmRecordReader());
    rrs.add(new SVMLightRecordReader());
    rrs.add(new RegexLineRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new RegexSequenceRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new TransformProcessRecordReader(new CSVRecordReader(), getTp()));
    rrs.add(new TransformProcessSequenceRecordReader(new CSVSequenceRecordReader(), getTp()));
    rrs.add(new LineRecordReader());

    for(RecordReader r : rrs){
        System.out.println(r.getClass().getName());
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ObjectOutputStream os = new ObjectOutputStream(baos);
        os.writeObject(r);
        byte[] bytes = baos.toByteArray();

        ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));

        RecordReader r2 = (RecordReader) ois.readObject();
    }
}
 

@Test
public void testAppendingLabels() throws Exception {
    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<Writable> exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"),
                    new IntWritable(0));
    assertEquals(exp0, rr.next());

    List<Writable> exp1 = Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"),
                    new IntWritable(1));
    assertEquals(exp1, rr.next());

    List<Writable> exp2 = Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"),
                    new IntWritable(2));
    assertEquals(exp2, rr.next());

    //Insert at position 0:
    rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen(), 0);
    rr.initialize(is);

    exp0 = Arrays.asList((Writable) new IntWritable(0), new Text("aValue0"), new Text("bValue0"),
                    new Text("cxValue0"));
    assertEquals(exp0, rr.next());

    exp1 = Arrays.asList((Writable) new IntWritable(1), new Text("aValue1"), new Text("MISSING_B"),
                    new Text("cxValue1"));
    assertEquals(exp1, rr.next());

    exp2 = Arrays.asList((Writable) new IntWritable(2), new Text("aValue2"), new Text("bValue2"),
                    new Text("MISSING_CX"));
    assertEquals(exp2, rr.next());
}
 

protected static synchronized ObjectMapper getJsonMapper() {
    if (jsonMapper == null) {
        jsonMapper = getNewMapper(new JsonFactory());
    }
    return jsonMapper;
}
 

@Test
public void testAppendingLabels() throws Exception {

    ClassPathResource cpr = new ClassPathResource("datavec-api/json/");
    File f = testDir.newFolder();
    cpr.copyDirectory(f);
    String path = new File(f, "json_test_%d.txt").getAbsolutePath();

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<Writable> exp0 = Arrays.asList((Writable) new Text("aValue0"), new Text("bValue0"), new Text("cxValue0"),
                    new IntWritable(0));
    assertEquals(exp0, rr.next());

    List<Writable> exp1 = Arrays.asList((Writable) new Text("aValue1"), new Text("MISSING_B"), new Text("cxValue1"),
                    new IntWritable(1));
    assertEquals(exp1, rr.next());

    List<Writable> exp2 = Arrays.asList((Writable) new Text("aValue2"), new Text("bValue2"), new Text("MISSING_CX"),
                    new IntWritable(2));
    assertEquals(exp2, rr.next());

    //Insert at position 0:
    rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen(), 0);
    rr.initialize(is);

    exp0 = Arrays.asList((Writable) new IntWritable(0), new Text("aValue0"), new Text("bValue0"),
                    new Text("cxValue0"));
    assertEquals(exp0, rr.next());

    exp1 = Arrays.asList((Writable) new IntWritable(1), new Text("aValue1"), new Text("MISSING_B"),
                    new Text("cxValue1"));
    assertEquals(exp1, rr.next());

    exp2 = Arrays.asList((Writable) new IntWritable(2), new Text("aValue2"), new Text("bValue2"),
                    new Text("MISSING_CX"));
    assertEquals(exp2, rr.next());
}
 

/**
 * Serialize this schema to json
 * @return a json representation of this schema
 */
public String toJson() {
    return toJacksonString(new JsonFactory());
}
 

/**
 * Serialize this schema to json
 * @return a json representation of this schema
 */
public String toJson() {
    return toJacksonString(new JsonFactory());
}