Java Code Examples for org.nd4j.common.primitives.Pair#of()

protected Pair<Map<Integer, Integer>, List<? extends Map<FieldName, ?>>> doTransformProcessConvertPmmlWithErrors(Schema schema, JsonArray jsonArray, TransformProcess transformProcess, DataPipelineErrorHandler dataPipelineErrorHandler) {
    Schema outputSchema = transformProcess.getFinalSchema();

    if (!transformProcess.getInitialSchema().equals(schema)) {
        throw new IllegalArgumentException("Transform process specified, but does not match target input inputSchema");
    }


    List<Map<FieldName, Object>> ret = new ArrayList<>(jsonArray.size());
    List<FieldName> fieldNames = getNameRepresentationFor(outputSchema);

    Pair<Map<Integer, Integer>, ArrowWritableRecordBatch> convertWithErrors = convertWithErrors(schema, jsonArray, transformProcess, dataPipelineErrorHandler);
    ArrowWritableRecordBatch conversion = convertWithErrors.getRight();
    for (int i = 0; i < conversion.size(); i++) {
        List<Writable> recordToMap = conversion.get(i);
        Map<FieldName, Object> record = new LinkedHashMap();
        for (int j = 0; j < outputSchema.numColumns(); j++) {
            record.put(fieldNames.get(j), WritableValueRetriever.getUnderlyingValue(recordToMap.get(j)));

        }

        ret.add(record);
    }

    return Pair.of(convertWithErrors.getKey(), ret);
}
 

private static GraphRunner getRunner(TensorDataType from,TensorDataType to) {
    Pair<TensorDataType,TensorDataType> key = Pair.of(from,to);
    if(!recastGraphDefs.containsKey(key)) {
        byte[] graphForDataType = graphForDataType(from,to);
        GraphRunner graphRunner = GraphRunner.builder()
                .graphBytes(graphForDataType)
                .inputNames(Arrays.asList("input"))
                .outputNames(Arrays.asList("cast_output"))
                .build();

        recastGraphDefs.put(key,graphRunner);
        return graphRunner;
    }

    return recastGraphDefs.get(key);
}
 

public static Pair<Schema, ArrowWritableRecordBatch> readFromFile(FileInputStream input) throws IOException {
    BufferAllocator allocator = new RootAllocator(9223372036854775807L);
    Schema retSchema = null;
    ArrowWritableRecordBatch ret = null;
    SeekableReadChannel channel = new SeekableReadChannel(input.getChannel());
    ArrowFileReader reader = new ArrowFileReader(channel, allocator);
    reader.loadNextBatch();
    retSchema = toDatavecSchema(reader.getVectorSchemaRoot().getSchema());
    VectorUnloader unloader = new VectorUnloader(reader.getVectorSchemaRoot());
    VectorLoader vectorLoader = new VectorLoader(reader.getVectorSchemaRoot());
    ArrowRecordBatch recordBatch = unloader.getRecordBatch();
    vectorLoader.load(recordBatch);
    ret = asDataVecBatch(recordBatch, retSchema, reader.getVectorSchemaRoot());
    ret.setUnloader(unloader);
    return Pair.of(retSchema, ret);
}
 

public static Pair<Schema, ArrowWritableRecordBatch> readFromBytes(byte[] input) throws IOException {
    BufferAllocator allocator = new RootAllocator(9223372036854775807L);
    Schema retSchema = null;
    ArrowWritableRecordBatch ret = null;
    SeekableReadChannel channel = new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(input));
    ArrowFileReader reader = new ArrowFileReader(channel, allocator);
    reader.loadNextBatch();
    retSchema = toDatavecSchema(reader.getVectorSchemaRoot().getSchema());
    VectorUnloader unloader = new VectorUnloader(reader.getVectorSchemaRoot());
    VectorLoader vectorLoader = new VectorLoader(reader.getVectorSchemaRoot());
    ArrowRecordBatch recordBatch = unloader.getRecordBatch();
    vectorLoader.load(recordBatch);
    ret = asDataVecBatch(recordBatch, retSchema, reader.getVectorSchemaRoot());
    ret.setUnloader(unloader);
    return Pair.of(retSchema, ret);
}
 

/**
 * Read a datavec schema and record set
 * from the given arrow file.
 * @param input the input to read
 * @return the associated datavec schema and record
 */
public static Pair<Schema,ArrowWritableRecordBatch> readFromFile(FileInputStream input) throws IOException {
    BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
    Schema retSchema = null;
    ArrowWritableRecordBatch ret = null;
    SeekableReadChannel channel = new SeekableReadChannel(input.getChannel());
    ArrowFileReader reader = new ArrowFileReader(channel, allocator);
    reader.loadNextBatch();
    retSchema = toDatavecSchema(reader.getVectorSchemaRoot().getSchema());
    //load the batch
    VectorUnloader unloader = new VectorUnloader(reader.getVectorSchemaRoot());
    VectorLoader vectorLoader = new VectorLoader(reader.getVectorSchemaRoot());
    ArrowRecordBatch recordBatch = unloader.getRecordBatch();

    vectorLoader.load(recordBatch);
    ret = asDataVecBatch(recordBatch,retSchema,reader.getVectorSchemaRoot());
    ret.setUnloader(unloader);

    return Pair.of(retSchema,ret);

}
 

@Override
public Pair<String, List<Writable>> apply(List<Writable> writables) {
    List<String> keyColumns = reducer.getKeyColumns();

    if(keyColumns == null){
        //Global reduction
        return Pair.of(GLOBAL_KEY, writables);
    } else {
        Schema schema = reducer.getInputSchema();
        String key;
        if (keyColumns.size() == 1)
            key = writables.get(schema.getIndexOfColumn(keyColumns.get(0))).toString();
        else {
            StringBuilder sb = new StringBuilder();
            for (int i = 0; i < keyColumns.size(); i++) {
                if (i > 0)
                    sb.append("_");
                sb.append(writables.get(schema.getIndexOfColumn(keyColumns.get(i))).toString());
            }
            key = sb.toString();
        }

        return Pair.of(key, writables);
    }
}
 

private Pair<KDNode, Integer> max(KDNode node, int disc, int _disc) {
    int discNext = (_disc + 1) % dims;
    if (_disc == disc) {
        KDNode child = node.getLeft();
        if (child != null) {
            return max(child, disc, discNext);
        }
    } else if (node.getLeft() != null || node.getRight() != null) {
        Pair<KDNode, Integer> left = null, right = null;
        if (node.getLeft() != null)
            left = max(node.getLeft(), disc, discNext);
        if (node.getRight() != null)
            right = max(node.getRight(), disc, discNext);
        if (left != null && right != null) {
            double pointLeft = left.getKey().getPoint().getDouble(disc);
            double pointRight = right.getKey().getPoint().getDouble(disc);
            if (pointLeft > pointRight)
                return left;
            else
                return right;
        } else if (left != null)
            return left;
        else
            return right;
    }

    return Pair.of(node, _disc);
}
 

@Override
public Pair<INDArray, MaskState> feedForwardMaskArrays(INDArray[] maskArrays, MaskState currentMaskState, int minibatchSize) {
    if(maskArrays != null) {
        if(maskArrays[0] == null) {
            // Queries are unmasked, we don't need to pass on any mask
            return null;
        }else{
            // Queries are masked, keep the masking going
            return Pair.of(maskArrays[0], currentMaskState);
        }
    }else {
        return Pair.of(null, currentMaskState);
    }
}
 

@Override
public Pair<Text, BytesWritable> apply(Pair<String, InputStream> in) {
    try {
        return Pair.of(new Text(in.getFirst()), new BytesWritable(IOUtils.toByteArray(in.getSecond())));
    } catch (IOException e) {
        throw new IllegalStateException(e);

    }

}
 

@Override
public Pair<List<Writable>, List<Writable>> apply(List<Writable> writables) {
    List<Writable> keyOut = new ArrayList<>(keyColumnIdxs.length);
    for (int keyColumnIdx : keyColumnIdxs) {
        keyOut.add(writables.get(keyColumnIdx));
    }
    return Pair.of(keyOut, writables);
}
 

@Override
public Pair<List<Writable>, List<Writable>> apply(List<Writable> writables) {

    List<Writable> keyValues;
    if (columnIndexes.length == 1) {
        keyValues = Collections.singletonList(writables.get(columnIndexes[0]));
    } else {
        keyValues = new ArrayList<>(columnIndexes.length);
        for (int i : columnIndexes) {
            keyValues.add(writables.get(i));
        }
    }

    return Pair.of(keyValues, writables);
}
 

private Pair<Schema,List<List<Writable>>> recordToWrite() {
    List<List<Writable>> records = new ArrayList<>();
    records.add(Arrays.<Writable>asList(new DoubleWritable(0.0),new DoubleWritable(0.0)));
    records.add(Arrays.<Writable>asList(new DoubleWritable(0.0),new DoubleWritable(0.0)));
    Schema.Builder schemaBuilder = new Schema.Builder();
    for(int i = 0; i < 2; i++) {
        schemaBuilder.addColumnFloat("col-" + i);
    }

    return Pair.of(schemaBuilder.build(),records);
}
 

private Pair<KDNode, Integer> min(KDNode node, int disc, int _disc) {
    int discNext = (_disc + 1) % dims;
    if (_disc == disc) {
        KDNode child = node.getLeft();
        if (child != null) {
            return min(child, disc, discNext);
        }
    } else if (node.getLeft() != null || node.getRight() != null) {
        Pair<KDNode, Integer> left = null, right = null;
        if (node.getLeft() != null)
            left = min(node.getLeft(), disc, discNext);
        if (node.getRight() != null)
            right = min(node.getRight(), disc, discNext);
        if (left != null && right != null) {
            double pointLeft = left.getKey().getPoint().getDouble(disc);
            double pointRight = right.getKey().getPoint().getDouble(disc);
            if (pointLeft < pointRight)
                return left;
            else
                return right;
        } else if (left != null)
            return left;
        else
            return right;
    }

    return Pair.of(node, _disc);
}
 

/**
 * Sets up promethues and returns the
 * registry
 * @return
 */
public static Pair<MicrometerMetricsOptions,MeterRegistry> setupPrometheus() {
    PrometheusMeterRegistry registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);

    MicrometerMetricsOptions micrometerMetricsOptions = new MicrometerMetricsOptions()
            .setMicrometerRegistry(registry)
            .setPrometheusOptions(new VertxPrometheusOptions()
                    .setEnabled(true));
    BackendRegistries.setupBackend(micrometerMetricsOptions);

    return Pair.of(micrometerMetricsOptions,registry);

}
 

/**
 * {@inheritDoc}
 */
@Override
public Pair<Map<Integer, Integer>, List<? extends Map<FieldName, ?>>> convertPmmlWithErrors(Schema schema, JsonArray jsonArray, TransformProcess transformProcess, DataPipelineErrorHandler dataPipelineErrorHandler) {
    if (transformProcess != null) {
        return doTransformProcessConvertPmmlWithErrors(schema, jsonArray, transformProcess, dataPipelineErrorHandler);
    }


    List<FieldName> fieldNames = getNameRepresentationFor(schema);


    List<Map<FieldName, Object>> ret = new ArrayList<>(jsonArray.size());
    Map<Integer, Integer> mapping = new LinkedHashMap<>();
    int numSucceeded = 0;
    for (int i = 0; i < jsonArray.size(); i++) {
        try {
            JsonObject jsonObject = jsonArray.getJsonObject(i);
            if (jsonObject.size() != schema.numColumns()) {
                throw new IllegalArgumentException("Found illegal item at row " + i);
            }
            Map<FieldName, Object> record = new LinkedHashMap();
            for (int j = 0; j < schema.numColumns(); j++) {
                record.put(fieldNames.get(j), jsonObject.getValue(schema.getName(j)));
            }

            mapping.put(numSucceeded, i);
            numSucceeded++;
            ret.add(record);
        } catch (Exception e) {
            dataPipelineErrorHandler.onError(e, jsonArray.getJsonObject(i), i);
        }
    }


    return Pair.of(mapping, ret);
}
 

private Pair<String, Integer> partNameAndIndex(String name) {
    //inputName[partIndex]
    //1 part only
    if (name.indexOf('[') < 0) {
        return Pair.of(name, 0);
    }

    String outputName = name.substring(0, name.indexOf('['));
    int partIndex = Integer.parseInt(name.substring(name.indexOf('[') + 1, name.lastIndexOf(']')));
    return Pair.of(outputName, partIndex);
}
 

@Override
public Pair<Writable, Long> apply(List<Writable> list) {
    return Pair.of(list.get(columnIndex), 1L);
}
 

@Override
public Pair<Writable, List<Writable>> apply(List<Writable> writables) {
    return Pair.of(writables.get(columnIdx), writables);
}
 

@Override
public Pair<Writable, List<Writable>> apply(List<Writable> writables) {
    return Pair.of(writables.get(keyColumnIdx), writables);
}
 

protected Pair<List<Image>, List<BoundingBox>> cropGrid(Mat m, List<Point> pxPoints, double gx, double gy) {
    Point tl = pxPoints.get(0);
    Point tr = pxPoints.get(1);
    Point bl = pxPoints.get(2);
    Point br = pxPoints.get(3);


    List<Image> out = new ArrayList<>();
    List<BoundingBox> bbox = (step != null ? step.boundingBoxName() != null : fStep.boundingBoxName() != null) ? new ArrayList<>() : null;
    //Note we are iterating (adding to output) in order: (0,0), (0, 1), ..., (0, C-1), ..., (R-1, C-1) - i.e., per row
    for (int j = 0; j < gy; j++) {
        for (int i = 0; i < gx; i++) {
            //Work out the corners of the current crop box
            Point boxTL = topLeft(j, i, (int) gy, (int) gx, tl, tr, bl, br);
            Point boxTR = topRight(j, i, (int) gy, (int) gx, tl, tr, bl, br);
            Point boxBL = bottomLeft(j, i, (int) gy, (int) gx, tl, tr, bl, br);
            Point boxBR = bottomRight(j, i, (int) gy, (int) gx, tl, tr, bl, br);

            double minX = min(boxTL.x(), boxTR.x(), boxBL.x(), boxBR.x());
            double maxX = max(boxTL.x(), boxTR.x(), boxBL.x(), boxBR.x());
            double minY = min(boxTL.y(), boxTR.y(), boxBL.y(), boxBR.y());
            double maxY = max(boxTL.y(), boxTR.y(), boxBL.y(), boxBR.y());

            int w = (int)(maxX - minX);
            int h = (int)(maxY - minY);

            if ((step != null && step.aspectRatio() != null) || (fStep != null && fStep.aspectRatio() != null)) {
                double currAr = w / (double) h;
                double ar = step != null ? step.aspectRatio() : fStep.aspectRatio();
                if (ar < currAr) {
                    //Need to increase height dimension to give desired AR
                    int newH = (int) (w / ar);
                    minY -= (newH - h) / 2.0;
                    h = newH;
                } else if (ar > currAr) {
                    //Need ot increase width dimension to give desired AR
                    int newW = (int) (h * ar);
                    minX -= (newW - w) / 2.0;
                    w = newW;
                }
            }

            //Make sure bounds are inside image. TODO handle this differently for aspect ratio preserving?
            if (minX < 0) {
                w += minX;
                minX = 0;
            }
            if (minX + w > m.cols()) {
                w = m.cols() - (int)minX;
            }
            if (minY < 0) {
                h += minY;
                minY = 0;
            }
            if (minY + h > m.rows()) {
                h = m.rows() - (int)minY;
            }


            Rect r = new Rect((int)minX, (int)minY, w, h);
            Mat crop = m.apply(r).clone();
            out.add(Image.create(crop));

            if (bbox != null) {
                bbox.add(BoundingBox.createXY(minX / (double) m.cols(), (minX + w) / (double) m.cols(), minY / (double) m.rows(), (minY + h) / (double) m.rows()));
            }
        }
    }

    return Pair.of(out, bbox);
}