Java Code Examples for org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector#getListLength()
The following examples show how to use
org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector#getListLength() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
/** * @return the number of true bits */ @Nonnull public static int setBits(@Nullable final Object argObj, @Nonnull final ListObjectInspector listOI, @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final BitSet bitset) throws UDFArgumentException { if (argObj == null) { return 0; } int count = 0; final int length = listOI.getListLength(argObj); for (int i = 0; i < length; i++) { final Object o = listOI.getListElement(argObj, i); if (o == null) { continue; } final int index = PrimitiveObjectInspectorUtils.getInt(o, elemOI); if (index < 0) { throw new UDFArgumentException("Negative index is not allowed: " + index); } bitset.set(index); count++; } return count; }
Example 2
Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull public static void toDoubleArray(@Nullable final Object argObj, @Nonnull final ListObjectInspector listOI, @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out, final double nullValue) throws UDFArgumentException { if (argObj == null) { return; } final int length = listOI.getListLength(argObj); if (out.length != length) { throw new UDFArgumentException( "Dimension mismatched. Expected: " + out.length + ", Actual: " + length); } for (int i = 0; i < length; i++) { Object o = listOI.getListElement(argObj, i); if (o == null) { out[i] = nullValue; continue; } out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); } return; }
Example 3
Source File: BitcoinTransactionHashUDF.java From hadoopcryptoledger with Apache License 2.0 | 6 votes |
/** * Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet) * * @param loi ObjectInspector for processing the Object containing a list * @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction * * @return a list of BitcoinTransactionOutputs * */ private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) { int listLength=loi.getListLength(listOfOutputsObject); List<BitcoinTransactionOutput> result=new ArrayList<>(listLength); StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector(); for (int i=0;i<listLength;i++) { Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i); StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value"); StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength"); StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript"); if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) { LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i); return new ArrayList<>(); } HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF)); byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF)); byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF)); BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript); result.add(currentBitcoinTransactionOutput); } return result; }
Example 4
Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull public static void toDoubleArray(@Nullable final Object argObj, @Nonnull final ListObjectInspector listOI, @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out, final boolean avoidNull) throws UDFArgumentException { if (argObj == null) { return; } final int length = listOI.getListLength(argObj); if (out.length != length) { throw new UDFArgumentException( "Dimension mismatched. Expected: " + out.length + ", Actual: " + length); } for (int i = 0; i < length; i++) { Object o = listOI.getListElement(argObj, i); if (o == null) { if (avoidNull) { continue; } throw new UDFArgumentException("Found null at index " + i); } out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); } return; }
Example 5
Source File: BitcoinTransactionHashSegwitUDF.java From hadoopcryptoledger with Apache License 2.0 | 6 votes |
/** * Read list of Bitcoin transaction outputs from a table in Hive in any format (e.g. ORC, Parquet) * * @param loi ObjectInspector for processing the Object containing a list * @param listOfOutputsObject object containing the list of outputs to a Bitcoin Transaction * * @return a list of BitcoinTransactionOutputs * */ private List<BitcoinTransactionOutput> readListOfOutputsFromTable(ListObjectInspector loi, Object listOfOutputsObject) { int listLength=loi.getListLength(listOfOutputsObject); List<BitcoinTransactionOutput> result=new ArrayList<>(listLength); StructObjectInspector listOfOutputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector(); for (int i=0;i<listLength;i++) { Object currentListOfOutputsObject = loi.getListElement(listOfOutputsObject,i); StructField valueSF = listOfOutputsElementObjectInspector.getStructFieldRef("value"); StructField txoutscriptlengthSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscriptlength"); StructField txoutscriptSF = listOfOutputsElementObjectInspector.getStructFieldRef("txoutscript"); if ((valueSF==null) || (txoutscriptlengthSF==null) || (txoutscriptSF==null)) { LOG.warn("Invalid BitcoinTransactionOutput detected at position "+i); return new ArrayList<>(); } HiveDecimal currentValue=hdoi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,valueSF)); byte[] currentTxOutScriptLength=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptlengthSF)); byte[] currentTxOutScript=wboi.getPrimitiveJavaObject(listOfOutputsElementObjectInspector.getStructFieldData(currentListOfOutputsObject,txoutscriptSF)); BitcoinTransactionOutput currentBitcoinTransactionOutput = new BitcoinTransactionOutput(currentValue.bigDecimalValue().toBigIntegerExact(),currentTxOutScriptLength,currentTxOutScript); result.add(currentBitcoinTransactionOutput); } return result; }
Example 6
Source File: ArrayAvgGenericUDAF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
void doIterate(@Nonnull final Object tuple, @Nonnull ListObjectInspector listOI, @Nonnull PrimitiveObjectInspector elemOI) throws HiveException { final int size = listOI.getListLength(tuple); if (_size == -1) { init(size); } if (size != _size) {// a corner case throw new HiveException( "Mismatch in the number of elements at tuple: " + tuple.toString()); } final double[] sum = _sum; final long[] count = _count; for (int i = 0, len = size; i < len; i++) { Object o = listOI.getListElement(tuple, i); if (o != null) { double v = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); sum[i] += v; count[i] += 1L; } } }
Example 7
Source File: XGBoostBatchPredictUDTF.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nonnull private static LabeledPointWithRowId parseDenseFeatures(@Nonnull final Writable rowId, @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI, @Nonnull final PrimitiveObjectInspector featureElemOI) throws UDFArgumentException { final int size = featureListOI.getListLength(argObj); final float[] values = new float[size]; for (int i = 0; i < size; i++) { final Object o = featureListOI.getListElement(argObj, i); if (o == null) { values[i] = Float.NaN; } else { float v = PrimitiveObjectInspectorUtils.getFloat(o, featureElemOI); values[i] = v; } } return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, null, values); }
Example 8
Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nullable public static double[] asDoubleArray(@Nullable final Object argObj, @Nonnull final ListObjectInspector listOI, @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull) throws UDFArgumentException { if (argObj == null) { return null; } final int length = listOI.getListLength(argObj); final double[] ary = new double[length]; for (int i = 0; i < length; i++) { Object o = listOI.getListElement(argObj, i); if (o == null) { if (avoidNull) { continue; } throw new UDFArgumentException("Found null at index " + i); } ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); } return ary; }
Example 9
Source File: HiveUtils.java From incubator-hivemall with Apache License 2.0 | 6 votes |
@Nullable public static ArrayList<Object> copyListObject(@Nonnull final DeferredObject argument, @Nonnull final ListObjectInspector loi, @Nonnull final ObjectInspectorCopyOption objectInspectorOption) throws HiveException { final Object o = argument.get(); if (o == null) { return null; } final int length = loi.getListLength(o); final ArrayList<Object> list = new ArrayList<Object>(length); for (int i = 0; i < length; i++) { Object e = ObjectInspectorUtils.copyToStandardObject(loi.getListElement(o, i), loi.getListElementObjectInspector(), objectInspectorOption); list.add(e); } return list; }
Example 10
Source File: Feature.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Nullable public static Feature[] parseFeatures(@Nonnull final Object arg, @Nonnull final ListObjectInspector listOI, @Nullable final Feature[] probes, final boolean asIntFeature) throws HiveException { if (arg == null) { return null; } final int length = listOI.getListLength(arg); final Feature[] ary; if (probes != null && probes.length == length) { ary = probes; } else { ary = new Feature[length]; } int j = 0; for (int i = 0; i < length; i++) { Object o = listOI.getListElement(arg, i); if (o == null) { continue; } String s = o.toString(); Feature f = ary[j]; if (f == null) { f = parseFeature(s, asIntFeature); } else { parseFeature(s, f, asIntFeature); } ary[j] = f; j++; } if (j == length) { return ary; } else { Feature[] dst = new Feature[j]; System.arraycopy(ary, 0, dst, 0, j); return dst; } }
Example 11
Source File: WriterImpl.java From hive-dwrf with Apache License 2.0 | 5 votes |
@Override void write(Object obj) throws IOException { long rawDataSize = 0; if (obj != null) { ListObjectInspector insp = (ListObjectInspector) inspector; int len = insp.getListLength(obj); lengths.write(len); for(int i=0; i < len; ++i) { childrenWriters[0].write(insp.getListElement(obj, i)); rawDataSize += childrenWriters[0].getRowRawDataSize(); } } super.write(obj, rawDataSize); }
Example 12
Source File: ArrayFlattenUDF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Override public List<Object> evaluate(DeferredObject[] args) throws HiveException { result.clear(); Object arg0 = args[0].get(); if (arg0 == null) { return null; } final int listLength = listOI.getListLength(arg0); for (int i = 0; i < listLength; i++) { final Object subarray = listOI.getListElement(arg0, i); if (subarray == null) { continue; } final ListObjectInspector subarrayOI = HiveUtils.asListOI(listOI.getListElementObjectInspector()); final ObjectInspector elemOI = subarrayOI.getListElementObjectInspector(); final int subarrayLength = subarrayOI.getListLength(subarray); for (int j = 0; j < subarrayLength; j++) { Object rawElem = subarrayOI.getListElement(subarray, j); if (rawElem == null) { continue; } Object elem = ObjectInspectorUtils.copyToStandardObject(rawElem, elemOI, ObjectInspectorCopyOption.WRITABLE); result.add(elem); } } return result; }
Example 13
Source File: OrcFlowFileWriter.java From nifi with Apache License 2.0 | 5 votes |
@Override void write(Object obj) throws IOException { super.write(obj); if (obj != null) { ListObjectInspector insp = (ListObjectInspector) inspector; int len = insp.getListLength(obj); lengths.write(len); if (createBloomFilter) { bloomFilter.addLong(len); } for (int i = 0; i < len; ++i) { childrenWriters[0].write(insp.getListElement(obj, i)); } } }
Example 14
Source File: Feature.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Nullable public static Feature[] parseFFMFeatures(@Nonnull final Object arg, @Nonnull final ListObjectInspector listOI, @Nullable final Feature[] probes, final int numFeatures, final int numFields) throws HiveException { if (arg == null) { return null; } final int length = listOI.getListLength(arg); final Feature[] ary; if (probes != null && probes.length == length) { ary = probes; } else { ary = new Feature[length]; } int j = 0; for (int i = 0; i < length; i++) { Object o = listOI.getListElement(arg, i); if (o == null) { continue; } String s = o.toString(); Feature f = ary[j]; if (f == null) { f = parseFFMFeature(s, numFeatures, numFields); } else { parseFFMFeature(s, f, numFeatures, numFields); } ary[j] = f; j++; } if (j == length) { return ary; } else { Feature[] dst = new Feature[j]; System.arraycopy(ary, 0, dst, 0, j); return dst; } }
Example 15
Source File: OrcFlowFileWriter.java From localization_nifi with Apache License 2.0 | 5 votes |
@Override void write(Object obj) throws IOException { super.write(obj); if (obj != null) { ListObjectInspector insp = (ListObjectInspector) inspector; int len = insp.getListLength(obj); lengths.write(len); if (createBloomFilter) { bloomFilter.addLong(len); } for (int i = 0; i < len; ++i) { childrenWriters[0].write(insp.getListElement(obj, i)); } } }
Example 16
Source File: FMPredictGenericUDAF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
void iterate(final double Wj, final double Xj, @Nonnull final Object Vif, @Nonnull final ListObjectInspector vOI, @Nonnull final PrimitiveObjectInspector vElemOI) throws HiveException { this.ret += (Wj * Xj); final int factors = vOI.getListLength(Vif); if (factors < 1) { throw new HiveException("# of Factor should be more than 0: " + factors); } if (sumVjXj == null) { this.sumVjXj = new double[factors]; this.sumV2X2 = new double[factors]; } else if (sumVjXj.length != factors) { throw new HiveException("Mismatch in the number of factors"); } for (int f = 0; f < factors; f++) { Object o = vOI.getListElement(Vif, f); if (o == null) { throw new HiveException("Vj" + f + " should not be null"); } double v = PrimitiveObjectInspectorUtils.getDouble(o, vElemOI); double vx = v * Xj; sumVjXj[f] += vx; sumV2X2[f] += (vx * vx); } }
Example 17
Source File: XGBoostBatchPredictUDTF.java From incubator-hivemall with Apache License 2.0 | 5 votes |
@Nonnull private static LabeledPointWithRowId parseSparseFeatures(@Nonnull final Writable rowId, @Nonnull final Object argObj, @Nonnull final ListObjectInspector featureListOI) throws UDFArgumentException { final int size = featureListOI.getListLength(argObj); final IntArrayList indices = new IntArrayList(size); final FloatArrayList values = new FloatArrayList(size); for (int i = 0; i < size; i++) { Object f = featureListOI.getListElement(argObj, i); if (f == null) { continue; } final String str = f.toString(); final int pos = str.indexOf(':'); if (pos < 1) { throw new UDFArgumentException("Invalid feature format: " + str); } final int index; final float value; try { index = Integer.parseInt(str.substring(0, pos)); value = Float.parseFloat(str.substring(pos + 1)); } catch (NumberFormatException e) { throw new UDFArgumentException("Failed to parse a feature value: " + str); } indices.add(index); values.add(value); } return new LabeledPointWithRowId(rowId, /* dummy label */ 0.f, indices.toArray(), values.toArray()); }
Example 18
Source File: BitcoinTransactionHashSegwitUDF.java From hadoopcryptoledger with Apache License 2.0 | 5 votes |
/** * Read list of Bitcoin transaction inputs from a table in Hive in any format (e.g. ORC, Parquet) * * @param loi ObjectInspector for processing the Object containing a list * @param listOfInputsObject object containing the list of inputs to a Bitcoin Transaction * * @return a list of BitcoinTransactionInputs * */ private List<BitcoinTransactionInput> readListOfInputsFromTable(ListObjectInspector loi, Object listOfInputsObject) { int listLength=loi.getListLength(listOfInputsObject); List<BitcoinTransactionInput> result = new ArrayList<>(listLength); StructObjectInspector listOfInputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector(); for (int i=0;i<listLength;i++) { Object currentlistofinputsObject = loi.getListElement(listOfInputsObject,i); StructField prevtransactionhashSF = listOfInputsElementObjectInspector.getStructFieldRef("prevtransactionhash"); StructField previoustxoutindexSF = listOfInputsElementObjectInspector.getStructFieldRef("previoustxoutindex"); StructField txinscriptlengthSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscriptlength"); StructField txinscriptSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscript"); StructField seqnoSF = listOfInputsElementObjectInspector.getStructFieldRef("seqno"); boolean prevFieldsNull = (prevtransactionhashSF==null) || (previoustxoutindexSF==null); boolean inFieldsNull = (txinscriptlengthSF==null) || (txinscriptSF==null); boolean otherAttribNull = seqnoSF==null; if (prevFieldsNull || inFieldsNull || otherAttribNull) { LOG.warn("Invalid BitcoinTransactionInput detected at position "+i); return new ArrayList<>(); } byte[] currentPrevTransactionHash = wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,prevtransactionhashSF)); long currentPreviousTxOutIndex = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,previoustxoutindexSF)); byte[] currentTxInScriptLength= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptlengthSF)); byte[] currentTxInScript= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptSF)); long currentSeqNo = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,seqnoSF)); BitcoinTransactionInput currentBitcoinTransactionInput = new BitcoinTransactionInput(currentPrevTransactionHash,currentPreviousTxOutIndex,currentTxInScriptLength,currentTxInScript,currentSeqNo); result.add(currentBitcoinTransactionInput); } return result; }
Example 19
Source File: BitcoinTransactionHashUDF.java From hadoopcryptoledger with Apache License 2.0 | 5 votes |
/** * Read list of Bitcoin transaction inputs from a table in Hive in any format (e.g. ORC, Parquet) * * @param loi ObjectInspector for processing the Object containing a list * @param listOfInputsObject object containing the list of inputs to a Bitcoin Transaction * * @return a list of BitcoinTransactionInputs * */ private List<BitcoinTransactionInput> readListOfInputsFromTable(ListObjectInspector loi, Object listOfInputsObject) { int listLength=loi.getListLength(listOfInputsObject); List<BitcoinTransactionInput> result = new ArrayList<>(listLength); StructObjectInspector listOfInputsElementObjectInspector = (StructObjectInspector)loi.getListElementObjectInspector(); for (int i=0;i<listLength;i++) { Object currentlistofinputsObject = loi.getListElement(listOfInputsObject,i); StructField prevtransactionhashSF = listOfInputsElementObjectInspector.getStructFieldRef("prevtransactionhash"); StructField previoustxoutindexSF = listOfInputsElementObjectInspector.getStructFieldRef("previoustxoutindex"); StructField txinscriptlengthSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscriptlength"); StructField txinscriptSF = listOfInputsElementObjectInspector.getStructFieldRef("txinscript"); StructField seqnoSF = listOfInputsElementObjectInspector.getStructFieldRef("seqno"); boolean prevFieldsNull = (prevtransactionhashSF==null) || (previoustxoutindexSF==null); boolean inFieldsNull = (txinscriptlengthSF==null) || (txinscriptSF==null); boolean otherAttribNull = seqnoSF==null; if (prevFieldsNull || inFieldsNull || otherAttribNull) { LOG.warn("Invalid BitcoinTransactionInput detected at position "+i); return new ArrayList<>(); } byte[] currentPrevTransactionHash = wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,prevtransactionhashSF)); long currentPreviousTxOutIndex = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,previoustxoutindexSF)); byte[] currentTxInScriptLength= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptlengthSF)); byte[] currentTxInScript= wboi.getPrimitiveJavaObject(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,txinscriptSF)); long currentSeqNo = wloi.get(listOfInputsElementObjectInspector.getStructFieldData(currentlistofinputsObject,seqnoSF)); BitcoinTransactionInput currentBitcoinTransactionInput = new BitcoinTransactionInput(currentPrevTransactionHash,currentPreviousTxOutIndex,currentTxInScriptLength,currentTxInScript,currentSeqNo); result.add(currentBitcoinTransactionInput); } return result; }
Example 20
Source File: ArrayUtils.java From hive-third-functions with Apache License 2.0 | 5 votes |
public static boolean arrayEquals(Object left, Object right, ListObjectInspector arrayOI) { if (left == null || right == null) { if (left == null && right == null) { return true; } return false; } int leftArrayLength = arrayOI.getListLength(left); int rightArrayLength = arrayOI.getListLength(right); if (leftArrayLength != rightArrayLength) { return false; } ObjectInspector arrayElementOI = arrayOI.getListElementObjectInspector(); for (int i = 0; i < leftArrayLength; i++) { Object leftArrayElement = arrayOI.getListElement(left, i); Object rightArrayElement = arrayOI.getListElement(right, i); int compareValue = ObjectInspectorUtils.compare(leftArrayElement, arrayElementOI, rightArrayElement, arrayElementOI); if (compareValue != 0) { return false; } } return true; }