Java Code Examples for org.apache.hive.hcatalog.data.HCatRecord#get()

The following examples show how to use org.apache.hive.hcatalog.data.HCatRecord#get() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ColumnCardinalityMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void map(T key, HCatRecord value, Context context) throws IOException, InterruptedException {

    HCatFieldSchema field;
    Object fieldValue;
    for (int m = 0; m < columnSize; m++) {
        field = schema.get(m);
        fieldValue = value.get(field.getName(), schema);
        if (fieldValue == null)
            fieldValue = "NULL";
        
        if (counter < 5 && m < 10) {
            System.out.println("Get row " + counter + " column '" + field.getName() + "'  value: " + fieldValue);
        }

        if (fieldValue != null)
            getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
    }

    counter++;
}
Example 2
Source File: FactDistinctColumnsMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {

    try {

        int[] flatTableIndexes = intermediateTableDesc.getRowKeyColumnIndexes();
        HCatFieldSchema fieldSchema = null;
        for (int i : factDictCols) {
            outputKey.set((short) i);
            fieldSchema = schema.get(flatTableIndexes[i]);
            Object fieldValue = record.get(fieldSchema.getName(), schema);
            if (fieldValue == null)
                continue;
            byte[] bytes = Bytes.toBytes(fieldValue.toString());
            outputValue.set(bytes, 0, bytes.length);
            context.write(outputKey, outputValue);
        }
    } catch (Exception ex) {
        handleErrorRecord(record, ex);
    }

}
Example 3
Source File: IIDistinctColumnsMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {

    HCatFieldSchema fieldSchema = null;
    for (short i = 0; i < columnSize; i++) {
        outputKey.set(i);
        fieldSchema = schema.get(i);
        Object fieldValue = record.get(fieldSchema.getName(), schema);
        if (fieldValue == null)
            continue;
        byte[] bytes = Bytes.toBytes(fieldValue.toString());
        outputValue.set(bytes, 0, bytes.length);
        context.write(outputKey, outputValue);
    }

}
Example 4
Source File: HiveTableReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static String[] getRowAsStringArray(HCatRecord record) {
    String[] arr = new String[record.size()];
    for (int i = 0; i < arr.length; i++) {
        Object o = record.get(i);
        arr[i] = (o == null || "\\N".equals(o)) ? null : o.toString();
    }
    return arr;
}
Example 5
Source File: SqoopHCatExportHelper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up 
public SqoopRecord convertToSqoopRecord(HCatRecord hcr)
  throws IOException {
  Text key = new Text();
  for (Map.Entry<String, Object> e : sqoopRecord.getFieldMap().entrySet()) {
    String colName = e.getKey();
    String hfn = colName.toLowerCase();
    key.set(hfn);
    Object hCatVal = hcr.get(hfn, hCatFullTableSchema);
    if (!isOdps) {
      String javaColType = colTypesJava.get(key).toString();
      int sqlType = ((IntWritable) colTypesSql.get(key)).get();
      HCatFieldSchema field = hCatFullTableSchema.get(hfn);
      HCatFieldSchema.Type fieldType = field.getType();
      String hCatTypeString = field.getTypeString();
      Object sqlVal = convertToSqoop(hCatVal, fieldType, javaColType, hCatTypeString);
      if (debugHCatExportMapper) {
        LOG.debug("hCatVal " + hCatVal + " of type "
            + (hCatVal == null ? null : hCatVal.getClass().getName()) + ",sqlVal " + sqlVal
            + " of type " + (sqlVal == null ? null : sqlVal.getClass().getName()) + ",java type "
            + javaColType + ", sql type = " + SqoopHCatUtilities.sqlTypeString(sqlType));
      }
      sqoopRecord.setField(colName, sqlVal);
    } else {
      sqoopRecord.setField(colName, hCatVal == null ? null : hCatVal.toString());
    }
  }
  return sqoopRecord;
}
Example 6
Source File: HiveTableReader.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static String[] getRowAsStringArray(HCatRecord record) {
    String[] arr = new String[record.size()];
    for (int i = 0; i < arr.length; i++) {
        Object o = record.get(i);
        arr[i] = (o == null || "\\N".equals(o)) ? null : o.toString();
    }
    return arr;
}
Example 7
Source File: InvertedIndexMapper.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException {

    rec.reset();
    for (int i = 0; i < fields.size(); i++) {
        Object fieldValue = record.get(i);
        rec.setValueString(i, fieldValue == null? null : fieldValue.toString());
    }

    outputKey.set(rec.getTimestamp());
    // outputValue's backing bytes array is the same as rec

    context.write(outputKey, outputValue);
}
Example 8
Source File: HCatalogTestUtils.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify that on a given row, a column has a given value.
 *
 * @param id
 *          the id column specifying the row to test.
 */
public static void assertHCatColValForRowId(List<HCatRecord> recs,
  HCatSchema schema, int id, String fieldName,
  Object expectedVal) throws IOException {
  LOG.info("Verifying field " + fieldName + " has value " + expectedVal);

  Object actualVal = null;
  for (HCatRecord rec : recs) {
    if (rec.getInteger("id", schema).equals(id)) {
      actualVal = rec.get(fieldName, schema);
      break;
    }
  }
  if (actualVal == null) {
    throw new IOException("No record found with id = " + id);
  }
  if (expectedVal != null && expectedVal instanceof byte[]) {
    Assert
      .assertArrayEquals((byte[]) expectedVal, (byte[]) actualVal);
  } else {
    if (expectedVal instanceof Float) {
      if (actualVal instanceof Double) {
        Assert.assertEquals(((Float) expectedVal).floatValue(),
          ((Double) actualVal).doubleValue(), DELTAVAL);
      } else {
        Assert
          .assertEquals("Got unexpected column value", expectedVal,
            actualVal);
      }
    } else if (expectedVal instanceof Double) {
      if (actualVal instanceof Float) {
        Assert.assertEquals(((Double) expectedVal).doubleValue(),
          ((Float) actualVal).doubleValue(), DELTAVAL);
      } else {
        Assert
          .assertEquals("Got unexpected column value", expectedVal,
            actualVal);
      }
    } else {
      Assert
        .assertEquals("Got unexpected column value", expectedVal,
          actualVal);
    }
  }
}