Java Code Examples for org.datavec.arrow.recordreader.ArrowWritableRecordBatch#size()

The following examples show how to use org.datavec.arrow.recordreader.ArrowWritableRecordBatch#size() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BaseJsonArrayConverter.java    From konduit-serving with Apache License 2.0 6 votes vote down vote up 
protected Pair<Map<Integer, Integer>, List<? extends Map<FieldName, ?>>> doTransformProcessConvertPmmlWithErrors(Schema schema, JsonArray jsonArray, TransformProcess transformProcess, DataPipelineErrorHandler dataPipelineErrorHandler) {
    Schema outputSchema = transformProcess.getFinalSchema();

    if (!transformProcess.getInitialSchema().equals(schema)) {
        throw new IllegalArgumentException("Transform process specified, but does not match target input inputSchema");
    }


    List<Map<FieldName, Object>> ret = new ArrayList<>(jsonArray.size());
    List<FieldName> fieldNames = getNameRepresentationFor(outputSchema);

    Pair<Map<Integer, Integer>, ArrowWritableRecordBatch> convertWithErrors = convertWithErrors(schema, jsonArray, transformProcess, dataPipelineErrorHandler);
    ArrowWritableRecordBatch conversion = convertWithErrors.getRight();
    for (int i = 0; i < conversion.size(); i++) {
        List<Writable> recordToMap = conversion.get(i);
        Map<FieldName, Object> record = new LinkedHashMap();
        for (int j = 0; j < outputSchema.numColumns(); j++) {
            record.put(fieldNames.get(j), WritableValueRetriever.getUnderlyingValue(recordToMap.get(j)));

        }

        ret.add(record);
    }

    return Pair.of(convertWithErrors.getKey(), ret);
}
Example 2
Source File: PipelineExecutioner.java    From konduit-serving with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates input for use in the {@link PipelineExecutioner}
 * @param input the input object
 * @param transformProcess the {@link TransformProcess} to use
 * @param conversionSchema The {@link Schema} to use
 * @return the DataVec type input records.
 */
public static Record[] createInput(Object input,TransformProcess transformProcess,Schema conversionSchema) {
    Preconditions.checkNotNull(input, "Input data was null!");

    if(input instanceof String) {
        String inputJson = (String) input;
        if (inputJson.charAt(0) == '{') {
            //json object
            log.info("Auto converting json object to json array");
            inputJson = "[" + input + "]";
        }

        JsonArray jsonArray = new JsonArray(inputJson);
        ArrowWritableRecordBatch convert;
        try {
            convert = mapConverter.convert(conversionSchema, jsonArray, transformProcess);
        } catch (Exception e) {
            log.error("Error performing conversion", e);
            throw e;
        }

        Preconditions.checkNotNull(convert, "Conversion was null!");
        Record[] pipelineInput = new Record[convert.size()];
        for (int i = 0; i < pipelineInput.length; i++) {
            pipelineInput[i] = new ArrowRecord(convert, i, null);
        }

        return pipelineInput;
    }

    else {
        //ndarrays already
        return (Record[]) input;
    }

}
Example 3
Source File: BaseJsonArrayConverter.java    From konduit-serving with Apache License 2.0 3 votes vote down vote up 
protected List<Map<FieldName, Object>> doTransformProcessConvertPmml(Schema schema, JsonArray jsonArray, TransformProcess transformProcess) {
    Schema outputSchema = transformProcess.getFinalSchema();

    if (!transformProcess.getInitialSchema().equals(schema)) {
        throw new IllegalArgumentException("Transform process specified, but does not match target input inputSchema");
    }


    List<Map<FieldName, Object>> ret = new ArrayList<>(jsonArray.size());
    List<FieldName> fieldNames = getNameRepresentationFor(outputSchema);

    ArrowWritableRecordBatch conversion = convert(schema, jsonArray, transformProcess);
    for (int i = 0; i < conversion.size(); i++) {
        List<Writable> recordToMap = conversion.get(i);
        Map<FieldName, Object> record = new LinkedHashMap();
        for (int j = 0; j < outputSchema.numColumns(); j++) {
            record.put(fieldNames.get(j), WritableValueRetriever.getUnderlyingValue(recordToMap.get(j)));

        }

        ret.add(record);
    }

    return ret;


}