org.nd4j.shade.jackson.core.JsonGenerator Java Examples

@Override
public void serialize(FixedValue fixedValue, JsonGenerator j, SerializerProvider serializerProvider) throws IOException {
    Object o = fixedValue.getValue();

    j.writeStringField("@valueclass", o.getClass().getName());
    if(o instanceof Number || o instanceof String || o instanceof Enum){
        j.writeObjectField("value", o);
    } else {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ObjectOutputStream oos = new ObjectOutputStream(baos);
        oos.writeObject(o);
        baos.close();
        byte[] b = baos.toByteArray();
        String base64 = new Base64().encodeToString(b);
        j.writeStringField("data", base64);
    }
}
 

@Override
public void serialize(ConfusionMatrix<Integer> cm, JsonGenerator gen, SerializerProvider provider)
                throws IOException, JsonProcessingException {
    List<Integer> classes = cm.getClasses();
    Map<Integer, Multiset<Integer>> matrix = cm.getMatrix();

    Map<Integer, int[][]> m2 = new LinkedHashMap<>();
    for (Integer i : matrix.keySet()) { //i = Actual class
        Multiset<Integer> ms = matrix.get(i);
        int[][] arr = new int[2][ms.size()];
        int used = 0;
        for (Integer j : ms.elementSet()) {
            int count = ms.count(j);
            arr[0][used] = j; //j = Predicted class
            arr[1][used] = count; //prediction count
            used++;
        }
        m2.put(i, arr);
    }

    gen.writeStartObject();
    gen.writeObjectField("classes", classes);
    gen.writeObjectField("matrix", m2);
    gen.writeEndObject();
}
 

private void writePoint(JsonGenerator jg, Point p) throws IOException {
    jg.writeStartObject();
    jg.writeFieldName(Data.RESERVED_KEY_POINT_COORDS);
    jg.writeStartArray(p.dimensions());
    for (int i = 0; i < p.dimensions(); i++) {
        writeDouble(jg, p.get(i));
    }
    jg.writeEndArray();

    if(p.label() != null){
        jg.writeFieldName("label");
        jg.writeString(p.label());
    }
    if(p.probability() != null){
        jg.writeFieldName("probability");
        jg.writeNumber(p.probability());
    }

    jg.writeEndObject();
}
 

/**
 *  Writes out all the parameters and their properties (final and resource) to
 *  the given {@link Writer}
 *  The format of the output would be
 *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
 *  key2.isFinal,key2.resource}... ] }
 *  It does not output the parameters of the configuration object which is
 *  loaded from an input stream.
 * @param out the Writer to write to
 * @throws IOException
 */
public static void dumpConfiguration(Configuration conf, Writer out) throws IOException {
    Configuration config = new Configuration(conf, true);
    config.reloadConfiguration();
    JsonFactory dumpFactory = new JsonFactory();
    JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
    dumpGenerator.writeStartObject();
    dumpGenerator.writeFieldName("properties");
    dumpGenerator.writeStartArray();
    dumpGenerator.flush();
    for (Map.Entry<Object, Object> item : config.getProps().entrySet()) {
        dumpGenerator.writeStartObject();
        dumpGenerator.writeStringField("key", (String) item.getKey());
        dumpGenerator.writeStringField("value", config.get((String) item.getKey()));
        dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey()));
        dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey()));
        dumpGenerator.writeEndObject();
    }
    dumpGenerator.writeEndArray();
    dumpGenerator.writeEndObject();
    dumpGenerator.flush();
}
 

/**
 *  Writes out all the parameters and their properties (final and resource) to
 *  the given {@link Writer}
 *  The format of the output would be
 *  { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
 *  key2.isFinal,key2.resource}... ] }
 *  It does not output the parameters of the configuration object which is
 *  loaded from an input stream.
 * @param out the Writer to write to
 * @throws IOException
 */
public static void dumpConfiguration(Configuration conf, Writer out) throws IOException {
    Configuration config = new Configuration(conf, true);
    config.reloadConfiguration();
    JsonFactory dumpFactory = new JsonFactory();
    JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
    dumpGenerator.writeStartObject();
    dumpGenerator.writeFieldName("properties");
    dumpGenerator.writeStartArray();
    dumpGenerator.flush();
    for (Map.Entry<Object, Object> item : config.getProps().entrySet()) {
        dumpGenerator.writeStartObject();
        dumpGenerator.writeStringField("key", (String) item.getKey());
        dumpGenerator.writeStringField("value", config.get((String) item.getKey()));
        dumpGenerator.writeBooleanField("isFinal", config.finalParameters.contains(item.getKey()));
        dumpGenerator.writeStringField("resource", config.updatingResource.get(item.getKey()));
        dumpGenerator.writeEndObject();
    }
    dumpGenerator.writeEndArray();
    dumpGenerator.writeEndObject();
    dumpGenerator.flush();
}
 

@Override
public void serialize(INDArray indArray, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    if (indArray.isView())
        indArray = indArray.dup(indArray.ordering());
    jsonGenerator.writeStartObject();
    DataBuffer view = indArray.data();
    jsonGenerator.writeArrayFieldStart("dataBuffer");
    for (int i = 0; i < view.length(); i++) {
        jsonGenerator.writeNumber(view.getDouble(i));
    }

    jsonGenerator.writeEndArray();

    jsonGenerator.writeArrayFieldStart("shapeField");
    for (int i = 0; i < indArray.rank(); i++) {
        jsonGenerator.writeNumber(indArray.size(i));
    }
    jsonGenerator.writeEndArray();

    jsonGenerator.writeArrayFieldStart("strideField");
    for (int i = 0; i < indArray.rank(); i++)
        jsonGenerator.writeNumber(indArray.stride(i));
    jsonGenerator.writeEndArray();

    jsonGenerator.writeNumberField("offsetField", indArray.offset());
    jsonGenerator.writeStringField("typeField", indArray instanceof IComplexNDArray ? "complex" : "real");
    jsonGenerator.writeNumberField("rankField", indArray.rank());
    jsonGenerator.writeNumberField("numElements", view.length());
    jsonGenerator.writeStringField("orderingField", String.valueOf(indArray.ordering()));
    jsonGenerator.writeEndObject();
}
 

@Override
public void serializeWithType(FixedValue value, JsonGenerator gen, SerializerProvider serializers, TypeSerializer typeSer) throws IOException {
    WritableTypeId typeId = typeSer.typeId(value, START_OBJECT);
    typeSer.writeTypePrefix(gen, typeId);
    serialize(value, gen, serializers);
    typeSer.writeTypeSuffix(gen, typeId);
}
 

@Override
public void serialize(StorageLevel storageLevel, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException, JsonProcessingException {
    //This is a little ugly, but Spark doesn't provide many options here...
    String s = null;
    if (storageLevel != null) {
        s = map.get(storageLevel);
    }
    jsonGenerator.writeString(s);
}
 

@Override
public void serialize(INDArray indArray, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    String toBase64 = Nd4jBase64.base64String(indArray);
    jsonGenerator.writeStartObject();
    jsonGenerator.writeStringField("array", toBase64);
    jsonGenerator.writeEndObject();
}
 

@Override
public void serialize(INDArray array, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    if (array.isView()) {
        array = array.dup();
    }
    double[] dArr = array.data().asDouble();
    jsonGenerator.writeObject(dArr);
}
 

@Override
public void serialize(ROC[] rocs, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException, JsonProcessingException {
    jsonGenerator.writeStartArray();
    for (ROC r : rocs) {
        jsonGenerator.writeStartObject();
        jsonGenerator.writeStringField("@class", ROC.class.getName());
        serializer.serialize(r, jsonGenerator, serializerProvider);
        jsonGenerator.writeEndObject();
    }
    jsonGenerator.writeEndArray();
}
 

@Override
public void serializeWithType(ROC value, JsonGenerator gen, SerializerProvider serializers, TypeSerializer typeSer)
                throws IOException {
    typeSer.writeTypePrefixForObject(value, gen);
    serialize(value, gen, serializers);
    typeSer.writeTypeSuffixForObject(value, gen);
}
 

@Override
public void serialize(ROC roc, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    boolean empty = roc.getExampleCount() == 0;

    if (roc.isExact() && !empty) {
        //For exact ROC implementation: force AUC and AUPRC calculation, so result can be stored in JSON, such
        //that we have them once deserialized.
        //Due to potentially huge size, exact mode doesn't store the original predictions in JSON
        roc.calculateAUC();
        roc.calculateAUCPR();
    }
    jsonGenerator.writeNumberField("thresholdSteps", roc.getThresholdSteps());
    jsonGenerator.writeNumberField("countActualPositive", roc.getCountActualPositive());
    jsonGenerator.writeNumberField("countActualNegative", roc.getCountActualNegative());
    jsonGenerator.writeObjectField("counts", roc.getCounts());
    if(!empty) {
        jsonGenerator.writeNumberField("auc", roc.calculateAUC());
        jsonGenerator.writeNumberField("auprc", roc.calculateAUCPR());
    }
    if (roc.isExact() && !empty) {
        //Store ROC and PR curves only for exact mode... they are redundant + can be calculated again for thresholded mode
        jsonGenerator.writeObjectField("rocCurve", roc.getRocCurve());
        jsonGenerator.writeObjectField("prCurve", roc.getPrecisionRecallCurve());
    }
    jsonGenerator.writeBooleanField("isExact", roc.isExact());
    jsonGenerator.writeNumberField("exampleCount", roc.getExampleCount());
    jsonGenerator.writeBooleanField("rocRemoveRedundantPts", roc.isRocRemoveRedundantPts());
}
 

@Override
public void serialize(DateTimeFieldType dateTimeFieldType, JsonGenerator jsonGenerator,
                SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
    jsonGenerator.writeStartObject();
    jsonGenerator.writeStringField("fieldType", dateTimeFieldType.getName());
    jsonGenerator.writeEndObject();
}
 

@Override
public void serialize(TDigest td, JsonGenerator j, SerializerProvider sp) throws IOException, JsonProcessingException {
    try(ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos)){
        oos.writeObject(td);
        oos.close();
        byte[] bytes = baos.toByteArray();
        Base64 b = new Base64();
        String str = b.encodeAsString(bytes);
        j.writeStartObject();
        j.writeStringField("digest", str);
        j.writeEndObject();
    }
}
 

@Override
public void serialize(INDArray indArray, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    String toBase64 = Nd4jBase64.base64String(indArray);
    jsonGenerator.writeStartObject();
    jsonGenerator.writeStringField("array", toBase64);
    jsonGenerator.writeEndObject();
}
 

@Override
public void serialize(INDArray array, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
                throws IOException {
    if (array.isView()) {
        array = array.dup();
    }
    double[] dArr = array.data().asDouble();
    jsonGenerator.writeObject(dArr);
}
 

@Override
public void serialize(DateTimeFieldType dateTimeFieldType, JsonGenerator jsonGenerator,
                SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
    jsonGenerator.writeStartObject();
    jsonGenerator.writeStringField("fieldType", dateTimeFieldType.getName());
    jsonGenerator.writeEndObject();
}
 

@Override
public void serialize(TDigest td, JsonGenerator j, SerializerProvider sp) throws IOException, JsonProcessingException {
    try(ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos)){
        oos.writeObject(td);
        oos.close();
        byte[] bytes = baos.toByteArray();
        Base64 b = new Base64();
        String str = b.encodeAsString(bytes);
        j.writeStartObject();
        j.writeStringField("digest", str);
        j.writeEndObject();
    }
}
 

private void writeBytes(JsonGenerator jg, byte[] bytes) throws IOException {
    //TODO add option to do raw bytes array - [0, 1, 2, ...] style
    jg.writeStartObject();
    jg.writeFieldName(Data.RESERVED_KEY_BYTES_BASE64);
    String base64 = Base64.getEncoder().encodeToString(bytes);
    jg.writeString(base64);
    jg.writeEndObject();
}
 

public static void writeBB(JsonGenerator jg, BoundingBox bb) throws IOException {
    //We'll keep it in the original format, if possible - but encode it as a X/Y format otherwise
    jg.writeStartObject();
    if(bb instanceof BBoxCHW){
        BBoxCHW b = (BBoxCHW)bb;
        jg.writeFieldName(Data.RESERVED_KEY_BB_CX);
        jg.writeNumber(b.cx());
        jg.writeFieldName(Data.RESERVED_KEY_BB_CY);
        jg.writeNumber(b.cy());
        jg.writeFieldName(Data.RESERVED_KEY_BB_H);
        jg.writeNumber(b.h());
        jg.writeFieldName(Data.RESERVED_KEY_BB_W);
        jg.writeNumber(b.w());
    } else {
        jg.writeFieldName(Data.RESERVED_KEY_BB_X1);
        jg.writeNumber(bb.x1());
        jg.writeFieldName(Data.RESERVED_KEY_BB_X2);
        jg.writeNumber(bb.x2());
        jg.writeFieldName(Data.RESERVED_KEY_BB_Y1);
        jg.writeNumber(bb.y1());
        jg.writeFieldName(Data.RESERVED_KEY_BB_Y2);
        jg.writeNumber(bb.y2());
    }
    if(bb.label() != null){
        jg.writeFieldName("label");
        jg.writeString(bb.label());
    }
    if(bb.probability() != null){
        jg.writeFieldName("probability");
        jg.writeNumber(bb.probability());
    }

    jg.writeEndObject();
}
 

private void writeNDArray(JsonGenerator jg, NDArray n) throws IOException {
    jg.writeStartObject();

    SerializedNDArray sn = n.getAs(SerializedNDArray.class);
    NDArrayType type = sn.getType();
    long[] shape = sn.getShape();
    jg.writeFieldName(Data.RESERVED_KEY_NDARRAY_TYPE);
    jg.writeString(type.toString());
    jg.writeFieldName(Data.RESERVED_KEY_NDARRAY_SHAPE);
    jg.writeArray(shape, 0, shape.length);

    ByteBuffer bb = sn.getBuffer();
    bb.rewind();
    byte[] array;
    if (bb.hasArray()) {
        array = bb.array();
    } else {
        int size = bb.remaining();
        array = new byte[size];
        for (int i = 0; i < size; i++) {
            array[i] = bb.get(i);
        }
    }

    String base64 = Base64.getEncoder().encodeToString(array);
    jg.writeFieldName(Data.RESERVED_KEY_NDARRAY_DATA_ARRAY);
    jg.writeString(base64);
    jg.writeEndObject();
}
 

private void writeImage(JsonGenerator jg, Image i) throws IOException {
    Png png = i.getAs(Png.class);
    byte[] imgData = png.getBytes();
    jg.writeStartObject();
    jg.writeFieldName(Data.RESERVED_KEY_IMAGE_FORMAT);
    jg.writeString("PNG");      //TODO No magic constant
    jg.writeFieldName(Data.RESERVED_KEY_IMAGE_DATA);
    String base64 = Base64.getEncoder().encodeToString(imgData);
    jg.writeString(base64);
    jg.writeEndObject();
}
 

@Override
public void serialize(AsyncPipeline ap, JsonGenerator jg, SerializerProvider sp) throws IOException {
    Pipeline p = ap.underlying();
    AsyncPipelineSerializationHelper h = new AsyncPipelineSerializationHelper(ap.trigger(), p);
    jg.writeObject(h);
}
 

@Override
public void serialize(Data data, JsonGenerator jg, SerializerProvider sp) throws IOException {
    //TODO do we serialize in any particular order?

    jg.writeStartObject();

    List<String> l = data.keys();

    for (String s : l) {
        ValueType vt = data.type(s);
        jg.writeFieldName(s);

        switch (vt) {
            case NDARRAY:
                NDArray n = data.getNDArray(s);
                writeNDArray(jg, n);
                break;
            case STRING:
                String str = data.getString(s);
                jg.writeString(str);
                break;
            case BYTES:
                writeBytes(jg, data.getBytes(s));
                break;
            case IMAGE:
                writeImage(jg, data.getImage(s));
                break;
            case DOUBLE:
                writeDouble(jg, data.getDouble(s));
                break;
            case INT64:
                writeLong(jg, data.getLong(s));
                break;
            case BOOLEAN:
                boolean b = data.getBoolean(s);
                jg.writeBoolean(b);
                break;
            case DATA:
                Data d = data.getData(s);
                writeNestedData(jg, d);
                break;
            case LIST:
                /*
                Format:
                "myList" : ["x", "y", "z"]
                 */
                ValueType listVt = data.listType(s);
                List<?> list = data.getList(s, listVt);
                writeList(jg, list, listVt);
                break;
            case BOUNDING_BOX:
                BoundingBox bb = data.getBoundingBox(s);
                writeBB(jg, bb);
                break;
            case POINT:
                Point p = data.getPoint(s);
                writePoint(jg, p);
                break;
            default:
                throw new IllegalStateException("Value type not yet supported/implemented: " + vt);
        }
    }

    if (data.getMetaData() != null) {
        Data md = data.getMetaData();
        jg.writeFieldName(Data.RESERVED_KEY_METADATA);
        writeNestedData(jg, md);
    }

    jg.writeEndObject();
}
 

private void writeNestedData(JsonGenerator jg, Data data) throws IOException {
    ObjectMapper om = (ObjectMapper) jg.getCodec();
    String dataStr = om.writeValueAsString(data);
    jg.writeRawValue(dataStr);
}
 

@Override
public void serialize(DataFormat dataFormat, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
    jsonGenerator.writeString(dataFormat.toString());
}
 

private void writeDouble(JsonGenerator jg, double d) throws IOException {
    jg.writeNumber(d);
}
 

private void writeLong(JsonGenerator jg, long l) throws IOException {
    jg.writeNumber(l);
}
 

@Override
public void serialize(AtomicDouble atomicDouble, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
    jsonGenerator.writeNumber(atomicDouble.doubleValue());
}