org.nd4j.shade.jackson.databind.ObjectMapper Java Examples

@Test
public void testReadingJson() throws Exception {
    //Load 3 values from 3 JSON files
    //stricture: a:value, b:value, c:x:value, c:y:value
    //And we want to load only a:value, b:value and c:x:value
    //For first JSON file: all values are present
    //For second JSON file: b:value is missing
    //For third JSON file: c:x:value is missing

    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
    rr.initialize(is);

    testJacksonRecordReader(rr);
}
 

@Test
public void testJsonSerialization() throws Exception {

    INDArray w = Nd4j.create(new double[] {1.0, 2.0, 3.0});

    ILossFunction[] lossFns = new ILossFunction[] {new LossBinaryXENT(), new LossBinaryXENT(w),
                    new LossCosineProximity(), new LossHinge(), new LossKLD(), new LossL1(), new LossL1(w),
                    new LossL2(), new LossL2(w), new LossMAE(), new LossMAE(w), new LossMAPE(), new LossMAPE(w),
                    new LossMCXENT(), new LossMCXENT(w), new LossMSE(), new LossMSE(w), new LossMSLE(),
                    new LossMSLE(w), new LossNegativeLogLikelihood(), new LossNegativeLogLikelihood(w),
                    new LossPoisson(), new LossSquaredHinge(), new LossMultiLabel()};

    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    mapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
    mapper.enable(SerializationFeature.INDENT_OUTPUT);

    for (ILossFunction lf : lossFns) {
        String asJson = mapper.writeValueAsString(lf);
        //            System.out.println(asJson);

        ILossFunction fromJson = mapper.readValue(asJson, ILossFunction.class);
        assertEquals(lf, fromJson);
    }
}
 

private static ObjectMapper configureMapper(ObjectMapper ret) {
    ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false);
    ret.enable(SerializationFeature.INDENT_OUTPUT);
    SimpleModule atomicModule = new SimpleModule();
    atomicModule.addSerializer(AtomicDouble.class, new JsonSerializerAtomicDouble());
    atomicModule.addSerializer(AtomicBoolean.class, new JsonSerializerAtomicBoolean());
    atomicModule.addDeserializer(AtomicDouble.class, new JsonDeserializerAtomicDouble());
    atomicModule.addDeserializer(AtomicBoolean.class, new JsonDeserializerAtomicBoolean());
    ret.registerModule(atomicModule);
    //Serialize fields only, not using getters
    ret.setVisibilityChecker(ret.getSerializationConfig().getDefaultVisibilityChecker()
            .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
            .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withCreatorVisibility(JsonAutoDetect.Visibility.ANY)
    );
    return ret;
}
 

private static ObjectMapper configureMapper(ObjectMapper ret) {
    ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, false);
    ret.enable(SerializationFeature.INDENT_OUTPUT);
    SimpleModule atomicModule = new SimpleModule();
    atomicModule.addSerializer(AtomicDouble.class,new JsonSerializerAtomicDouble());
    atomicModule.addSerializer(AtomicBoolean.class,new JsonSerializerAtomicBoolean());
    atomicModule.addDeserializer(AtomicDouble.class,new JsonDeserializerAtomicDouble());
    atomicModule.addDeserializer(AtomicBoolean.class,new JsonDeserializerAtomicBoolean());
    ret.registerModule(atomicModule);
    //Serialize fields only, not using getters
    ret.setVisibilityChecker(ret.getSerializationConfig().getDefaultVisibilityChecker()
            .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
            .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withCreatorVisibility(JsonAutoDetect.Visibility.NONE));
    return ret;
}
 

@Test
public void testDistributionDeserializer() throws Exception {
    //Test current format:
    Distribution[] distributions =
                    new Distribution[] {new NormalDistribution(3, 0.5), new UniformDistribution(-2, 1),
                                    new GaussianDistribution(2, 1.0), new BinomialDistribution(10, 0.3)};

    ObjectMapper om = NeuralNetConfiguration.mapper();

    for (Distribution d : distributions) {
        String json = om.writeValueAsString(d);
        Distribution fromJson = om.readValue(json, Distribution.class);

        assertEquals(d, fromJson);
    }
}
 

protected static ValidationResult isValidJson(String content, File f) {
    try {
        ObjectMapper om = new ObjectMapper();
        JavaType javaType = om.getTypeFactory().constructMapType(Map.class, String.class, Object.class);
        om.readValue(content, javaType);    //Don't care about result, just that it can be parsed successfully
    } catch (Throwable t) {
        //Jackson should tell us specifically where error occurred also
        return ValidationResult.builder()
                .valid(false)
                .formatType("JSON")
                .path(getPath(f))
                .issues(Collections.singletonList("File does not appear to be valid JSON"))
                .exception(t)
                .build();
    }


    return ValidationResult.builder()
            .valid(true)
            .formatType("JSON")
            .path(getPath(f))
            .build();
}
 

/**
 * Return this configuration as json
 *
 * @return this configuration represented as json
 */
public String toJson() {
    ObjectMapper mapper = mapper();

    try {
        return mapper.writeValueAsString(this);
    } catch (org.nd4j.shade.jackson.core.JsonProcessingException e) {
        throw new RuntimeException(e);
    }
}
 

@Test
   public void testReadJSON() throws Exception {
      
       RecordReader rr = new JacksonLineRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()));
       rr.initialize(new FileSplit(new ClassPathResource("datavec-api/json/json_test_3.txt").getFile()));
       
       testJacksonRecordReader(rr);
}
 

private static void configureMapper(ObjectMapper ret) {
    ret.registerModule(new JodaModule());
    ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
    ret.enable(SerializationFeature.INDENT_OUTPUT);
    ret.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
    ret.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
    ret.setVisibility(PropertyAccessor.CREATOR, JsonAutoDetect.Visibility.ANY);     //Need this otherwise JsonProperty annotations on constructors won't be seen
}
 

private <T> T load(String str, Class<T> clazz) {
    ObjectMapper om = getObjectMapper();
    try {
        return om.readValue(str, clazz);
    } catch (Exception e) {
        //TODO better exception
        throw new RuntimeException(e);
    }
}
 

public static ObjectMapper mapper() {
    /*
          DO NOT ENABLE INDENT_OUTPUT FEATURE
          we need THIS json to be single-line
      */
    ObjectMapper ret = new ObjectMapper();
    ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
    return ret;
}
 

/**
 * Serialize a list of DataActions
 */
public String serializeDataActionList(List<DataAction> list) {
    ObjectMapper om = getObjectMapper();
    try {
        return om.writeValueAsString(new ListWrappers.DataActionList(list));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * Create a neural net configuration from json
 *
 * @param json the neural net configuration from json
 * @return
 */
public static NeuralNetConfiguration fromYaml(String json) {
    ObjectMapper mapper = mapperYaml();
    try {
        NeuralNetConfiguration ret = mapper.readValue(json, NeuralNetConfiguration.class);
        return ret;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

public static final ObjectMapper newMapper() {
    ObjectMapper ret = new ObjectMapper();
    ret.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
    ret.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
    ret.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
    ret.enable(SerializationFeature.INDENT_OUTPUT);
    return ret;
}
 

@Override
public String toYaml() {
    ObjectMapper om = getYamlMapper();

    try {
        return om.writeValueAsString(this);
    } catch (JsonProcessingException e) {
        throw new RuntimeException("Error producing YAML representation for ParameterAveragingTrainingMaster", e);
    }
}
 

/**
 * Serialize a list of Filters
 */
public String serializeFilterList(List<Filter> list) {
    ObjectMapper om = getObjectMapper();
    try {
        return om.writeValueAsString(new ListWrappers.FilterList(list));
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private <T> T load(String str, TypeReference<T> typeReference) {
    ObjectMapper om = getObjectMapper();
    try {
        return om.readValue(str, typeReference);
    } catch (Exception e) {
        //TODO better exception
        throw new RuntimeException(e);
    }
}
 

private String toJson(Map<String, List<Map<String, Object>>> jsonRepresentation) {
    ObjectMapper om = new JsonSerializer().getObjectMapper();
    try {
        return om.writeValueAsString(jsonRepresentation);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private static ObjectMapper getMapper() {
    ObjectMapper om = new ObjectMapper();
    //Serialize fields only, not using getters
    //Not all getters are supported - for example, UserEntity
    om.setVisibilityChecker(om.getSerializationConfig()
            .getDefaultVisibilityChecker()
            .withFieldVisibility(JsonAutoDetect.Visibility.ANY)
            .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withSetterVisibility(JsonAutoDetect.Visibility.NONE)
            .withCreatorVisibility(JsonAutoDetect.Visibility.NONE));
    om.setSerializationInclusion(JsonInclude.Include.NON_NULL);
    return om;
}
 

/**
 * Deserialize a YAML DataAnalysis String that was previously serialized with {@link #toYaml()}
 */
public static DataAnalysis fromYaml(String yaml) {
    try{
        return new YamlSerializer().getObjectMapper().readValue(yaml, DataAnalysis.class);
    } catch (Exception e){
        //Legacy format
        ObjectMapper om = new YamlSerializer().getObjectMapper();
        return fromMapper(om, yaml);
    }
}
 

/**
 * Get the onnx op descriptors by name
 * @return the onnx op descriptors by name
 * @throws Exception
 */
public static Map<String,OpDescriptor> onnxOpDescriptors() throws Exception {
    try(InputStream is = new ClassPathResource("onnxops.json").getInputStream()) {
        ObjectMapper objectMapper = new ObjectMapper();
        OnnxDescriptor opDescriptor = objectMapper.readValue(is,OnnxDescriptor.class);
        Map<String,OpDescriptor> descriptorMap = new HashMap<>();
        for(OpDescriptor descriptor : opDescriptor.getDescriptors()) {
            descriptorMap.put(descriptor.getName(),descriptor);
        }



        return descriptorMap;
    }
}
 

private static ObjectMapper objectMapper() {
    ObjectMapper mapper = new ObjectMapper();
    SimpleModule nd4j = new SimpleModule("nd4j");
    nd4j.addDeserializer(INDArray.class, new VectorDeSerializer());
    nd4j.addSerializer(INDArray.class, new VectorSerializer());
    mapper.registerModule(nd4j);
    return mapper;

}
 

/**
 * Get the onnx op descriptors by name
 * @return the onnx op descriptors by name
 * @throws Exception
 */
public static Map<String,OpDescriptor> onnxOpDescriptors() throws Exception {
    try(InputStream is = new ClassPathResource("onnxops.json").getInputStream()) {
        ObjectMapper objectMapper = new ObjectMapper();
        OnnxDescriptor opDescriptor = objectMapper.readValue(is,OnnxDescriptor.class);
        Map<String,OpDescriptor> descriptorMap = new HashMap<>();
        for(OpDescriptor descriptor : opDescriptor.getDescriptors()) {
            descriptorMap.put(descriptor.getName(),descriptor);
        }



        return descriptorMap;
    }
}
 

@Test
public void testRR() throws Exception {

    List<RecordReader> rrs = new ArrayList<>();

    rrs.add(new CSVNLinesSequenceRecordReader(10));
    rrs.add(new CSVRecordReader(10, ','));
    rrs.add(new CSVSequenceRecordReader(1, ","));
    rrs.add(new CSVVariableSlidingWindowRecordReader(5));
    rrs.add(new CSVRegexRecordReader(0, ",", null, new String[] {null, "(.+) (.+) (.+)"}));
    rrs.add(new JacksonRecordReader(new FieldSelection.Builder().addField("a").addField(new Text("MISSING_B"), "b")
            .addField(new Text("MISSING_CX"), "c", "x").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new JacksonLineRecordReader(new FieldSelection.Builder().addField("value1")
    		.addField("value2").build(), new ObjectMapper(new JsonFactory())));
    rrs.add(new LibSvmRecordReader());
    rrs.add(new SVMLightRecordReader());
    rrs.add(new RegexLineRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new RegexSequenceRecordReader("(.+) (.+) (.+)", 0));
    rrs.add(new TransformProcessRecordReader(new CSVRecordReader(), getTp()));
    rrs.add(new TransformProcessSequenceRecordReader(new CSVSequenceRecordReader(), getTp()));
    rrs.add(new LineRecordReader());

    for(RecordReader r : rrs){
        System.out.println(r.getClass().getName());
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ObjectOutputStream os = new ObjectOutputStream(baos);
        os.writeObject(r);
        byte[] bytes = baos.toByteArray();

        ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));

        RecordReader r2 = (RecordReader) ois.readObject();
    }
}
 

/**
 * Create a SharedTrainingMaster instance by deserializing a JSON string that has been serialized with
 * {@link #toJson()}
 *
 * @param jsonStr SharedTrainingMaster configuration serialized as JSON
 */
public static SharedTrainingMaster fromJson(String jsonStr) {
    ObjectMapper om = getJsonMapper();
    try {
        return om.readValue(jsonStr, SharedTrainingMaster.class);
    } catch (IOException e) {
        throw new RuntimeException("Could not parse JSON", e);
    }
}
 

/**
 * This utility method serializes ElementPair into JSON + packs it into Base64-encoded string
 *
 * @return
 */
protected String toEncodedJson() {
    ObjectMapper mapper = SequenceElement.mapper();
    Base64 base64 = new Base64(Integer.MAX_VALUE);
    try {
        String json = mapper.writeValueAsString(this);
        String output = base64.encodeAsString(json.getBytes("UTF-8"));
        return output;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testAppendingLabelsMetaData() throws Exception {
    ClassPathResource cpr = new ClassPathResource("json/json_test_0.txt");
    String path = cpr.getFile().getAbsolutePath().replace("0", "%d");

    InputSplit is = new NumberedFileInputSplit(path, 0, 2);

    //Insert at the end:
    RecordReader rr = new JacksonRecordReader(getFieldSelection(), new ObjectMapper(new JsonFactory()), false, -1,
                    new LabelGen());
    rr.initialize(is);

    List<List<Writable>> out = new ArrayList<>();
    while (rr.hasNext()) {
        out.add(rr.next());
    }
    assertEquals(3, out.size());

    rr.reset();

    List<List<Writable>> out2 = new ArrayList<>();
    List<Record> outRecord = new ArrayList<>();
    List<RecordMetaData> meta = new ArrayList<>();
    while (rr.hasNext()) {
        Record r = rr.nextRecord();
        out2.add(r.getRecord());
        outRecord.add(r);
        meta.add(r.getMetaData());
    }

    assertEquals(out, out2);

    List<Record> fromMeta = rr.loadFromMetaData(meta);
    assertEquals(outRecord, fromMeta);
}
 

public static TraceEvent[] readEvents(File file) throws IOException {
    String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8);
    content = StringUtils.trimTrailingWhitespace(content);
    if (content.endsWith(","))
        content = content.substring(0, content.length()-1) + "]";
    if (StringUtils.isEmpty(content))
        return new TraceEvent[0];
    TraceEvent[] events = new ObjectMapper().readValue(content, TraceEvent[].class);
    return events;
}
 

/**
 * Read JSON-formatted string attribute.
 *
 * @param attribute HDF5 attribute to read as JSON formatted string.
 * @return JSON formatted string from HDF5 attribute
 * @throws UnsupportedKerasConfigurationException Unsupported Keras config
 */
private String readAttributeAsJson(Attribute attribute) throws UnsupportedKerasConfigurationException {
    synchronized (Hdf5Archive.LOCK_OBJECT) {
        VarLenType vl = attribute.getVarLenType();
        int currBufferLength = 2048;
        String s;
        /* TODO: find a less hacky way to do this.
         * Reading variable length strings (from attributes) is a giant
         * pain. There does not appear to be any way to determine the
         * length of the string in advance, so we use a hack: choose a
         * buffer size and read the config. If Jackson fails to parse
         * it, then we must not have read the entire config. Increase
         * buffer and repeat.
         */
        while (true) {
            byte[] attrBuffer = new byte[currBufferLength];
            BytePointer attrPointer = new BytePointer(currBufferLength);
            attribute.read(vl, attrPointer);
            attrPointer.get(attrBuffer);
            s = new String(attrBuffer);
            ObjectMapper mapper = new ObjectMapper();
            mapper.enable(DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY);
            try {
                mapper.readTree(s);
                break;
            } catch (IOException e) {
                //OK - we don't know how long the buffer needs to be, so we'll try again with larger buffer
            }

            if(currBufferLength == MAX_BUFFER_SIZE_BYTES){
                throw new UnsupportedKerasConfigurationException("Could not read abnormally long HDF5 attribute: size exceeds " + currBufferLength + " bytes");
            } else {
                currBufferLength = (int)Math.min(MAX_BUFFER_SIZE_BYTES, currBufferLength * 4L);
            }
        }
        vl.deallocate();
        return s;
    }
}
 

private String toYaml(Map<String, List<Map<String, Object>>> jsonRepresentation) {
    ObjectMapper om = new YamlSerializer().getObjectMapper();
    try {
        return om.writeValueAsString(jsonRepresentation);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}