Java Code Examples for org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils#getTypeInfoFromTypeString()

@Test
public void testTopLevelPrimitive() throws Exception {
    Double expected1 = Double.valueOf(3.3);
    Text json1 = new Text("3.3");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("double");

    Object deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    Assert.assertEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);

    Boolean expected2 = Boolean.FALSE;
    Text json2 = new Text("false");

    Boolean deserialized2 = JsonSerdeUtils.deserialize(json2);
    Assert.assertEquals(expected2, deserialized2);
    Text serialized2 = JsonSerdeUtils.serialize(deserialized2,
        PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    Assert.assertEquals(json2, serialized2);
}
 

@Test
public void testTopLevelArray() throws Exception {
    List<String> expected1 = Arrays.asList("Taro", "Tanaka");
    Text json1 = new Text("[\"Taro\",\"Tanaka\"]");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("array<string>");

    List<Object> deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    assertRecordEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);

    List<Double> expected2 = Arrays.asList(1.1d, 2.2d, 3.3d);
    Text json2 = new Text("[1.1,2.2,3.3]");
    TypeInfo type2 = TypeInfoUtils.getTypeInfoFromTypeString("array<double>");

    List<Object> deserialized2 = JsonSerdeUtils.deserialize(json2, type2);
    assertRecordEquals(expected2, deserialized2);
    Text serialized2 = JsonSerdeUtils.serialize(deserialized2,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type2));
    Assert.assertEquals(json2, serialized2);
}
 

@Override
public Type toMetacatType(final String type) {
    // Hack to fix presto "varchar" type coming in with no length which is required by Hive.
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
        "varchar".equals(type.toLowerCase()) ? serdeConstants.STRING_TYPE_NAME : type);
    ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    // The standard struct object inspector forces field names to lower case, however in Metacat we need to preserve
    // the original case of the struct fields so we wrap it with our wrapper to force the fieldNames to keep
    // their original case
    if (typeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        final StandardStructObjectInspector objectInspector = (StandardStructObjectInspector) oi;
        oi = new HiveTypeConverter.SameCaseStandardStructObjectInspector(
            structTypeInfo.getAllStructFieldNames(), objectInspector);
    }
    return getCanonicalType(oi);
}
 

@Nonnull
public static ObjectInspector getObjectInspector(@Nonnull final String typeString,
        final boolean preferWritable) {
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeString);
    if (preferWritable) {
        return TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo);
    } else {
        return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    }
}
 

private void verifyORCUsers(final Path orcUsers, final int numExpectedUsers, BiFunction<List<Object>, Integer, Void> assertFunction) throws IOException {
    Reader reader = OrcFile.createReader(orcUsers, OrcFile.readerOptions(testConf));
    RecordReader recordReader = reader.rows();

    TypeInfo typeInfo =
            TypeInfoUtils.getTypeInfoFromTypeString("struct<name:string,favorite_number:int,favorite_color:string,scale:double>");
    StructObjectInspector inspector = (StructObjectInspector)
            OrcStruct.createObjectInspector(typeInfo);

    int currUser = 0;
    Object nextRecord = null;
    while ((nextRecord = recordReader.next(nextRecord)) != null) {
        Assert.assertNotNull(nextRecord);
        Assert.assertTrue("Not an OrcStruct", nextRecord instanceof OrcStruct);
        List<Object> x = inspector.getStructFieldsDataAsList(nextRecord);

        if (assertFunction == null) {
            assertEquals("name" + currUser, x.get(0).toString());
            assertEquals(currUser, ((IntWritable) x.get(1)).get());
            assertEquals("blue" + currUser, x.get(2).toString());
            assertEquals(10.0 * currUser, ((DoubleWritable) x.get(3)).get(), Double.MIN_VALUE);
        } else {
            assertFunction.apply(x, currUser);
        }
        currUser++;
    }

    assertEquals(numExpectedUsers, currUser);
}
 

public CobolStringField(String debugInfo, int levelNo, String name,
		String picClause) {
	super();
	super.debugInfo = debugInfo;
	super.levelNo = levelNo;
	super.type = CobolFieldType.STRING;
	super.name = name;
	String fieldType = "string";
	if (picClause.contains("(")) {
		String[] s = picClause.split("\\(|\\)|\\.");
		if (s.length == 2) {
			try {
				super.length = Integer.parseInt(s[1]);
			} catch (NumberFormatException e) {
				throw e;
			}
		} else {
			throw new RuntimeException(
					"Alphanumeric Picture clause has more brackets:"
							+ this.debugInfo);
		}
	} else {
		if (picClause.trim().toLowerCase().matches("[x|a]+\\."))
			super.length = picClause.length() -1;
		else if (picClause.trim().toLowerCase().matches("[x|a]+"))
			super.length = picClause.length();
		else {
			throw new RuntimeException(
					"Alphanumeric Picture clause incorrect '"
							+ this.debugInfo);

		}
	}
	if (super.length < 65355) {
		fieldType = "varchar(" + this.length + ")";
	}
	super.typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fieldType);
	this.oi = TypeInfoUtils
			.getStandardJavaObjectInspectorFromTypeInfo(this.typeInfo);
}
 

@Test
public void testTopLevelMap() throws Exception {
    Map<String, Integer> expected1 = ImmutableMap.of("one", 1, "two", 2);
    Text json1 = new Text("{\"one\":1,\"two\":2}");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("map<string,int>");

    Map<String, Integer> deserialized1 = JsonSerdeUtils.deserialize(json1, type1);
    Assert.assertEquals(expected1, deserialized1);
    Text serialized1 = JsonSerdeUtils.serialize(deserialized1,
        HCatRecordObjectInspectorFactory.getStandardObjectInspectorFromTypeInfo(type1));
    Assert.assertEquals(json1, serialized1);
}
 

@Test
public void testStructWithoutColumnNames() throws Exception {
    Text json1 = new Text("{ \"person\" : { \"name\" : \"makoto\" , \"age\" : 37 } }");
    TypeInfo type1 = TypeInfoUtils.getTypeInfoFromTypeString("struct<name:string,age:int>");
    List<Object> expected1 = Arrays.<Object>asList("makoto", 37);

    List<Object> deserialized1 =
            JsonSerdeUtils.deserialize(json1, Arrays.asList("person"), Arrays.asList(type1));

    assertRecordEquals(expected1, deserialized1);
}
 

private static List<ColumnInfo> buildColumnInfo(final Table table, final InputFormat<?, ?> format, boolean
  includeComplexParquetCols) {
  final List<ColumnInfo> columnInfos = new ArrayList<>();
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    Field f = HiveSchemaConverter.getArrowFieldFromHiveType(hiveField.getName(), typeInfo, format, includeComplexParquetCols);
    if (f != null) {
      columnInfos.add(getColumnInfo(typeInfo));
    }
  }
  return columnInfos;
}
 

private static boolean isFieldTypeVarchar(FieldSchema hiveField) {
  final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
  if (typeInfo.getCategory() == Category.PRIMITIVE) {
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    if (pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR ||
      pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR) {
      return true;
    }
  }
  return false;
}
 

private static List<ColumnInfo> buildColumnInfo(final Table table, final InputFormat<?, ?> format, final boolean includeComplexParquetCols) {
  final List<ColumnInfo> columnInfos = new ArrayList<>();
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    Field f = HiveSchemaConverter.getArrowFieldFromHiveType(hiveField.getName(), typeInfo, format, includeComplexParquetCols);
    if (f != null) {
      columnInfos.add(getColumnInfo(typeInfo));
    }
  }
  return columnInfos;
}
 

private static boolean isFieldTypeVarchar(FieldSchema hiveField) {
  final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
  if (typeInfo.getCategory() == Category.PRIMITIVE) {
    PrimitiveTypeInfo pTypeInfo = (PrimitiveTypeInfo) typeInfo;
    if (pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.VARCHAR ||
      pTypeInfo.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.CHAR) {
      return true;
    }
  }
  return false;
}
 

/**
 * iterates over all fields of a table and checks if any field exceeds
 * maximum allowed nested level
 * @param table
 * @param maxNestedLevels
 */
public static void checkFieldNestedLevels(final Table table, int maxNestedLevels) {
  for (FieldSchema hiveField : table.getSd().getCols()) {
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(hiveField.getType());
    int depth = findFieldDepth(typeInfo);
    if (depth > maxNestedLevels) {
      throw new ColumnNestedTooDeepException(hiveField.getName(), maxNestedLevels);
    }
  }
}
 

public static ObjectInspector createJavaObjectInspectorFromFieldSchema(String columnTypeString) {
  TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(columnTypeString);
  return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
}
 

public static TypeInfo buildNestedComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myMapOfArray:map<string,array<double>>,myArrayOfMap:array<map<string,string>>>");
}
 

public static TypeInfo buildComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myInt:int,myMap:map<string,double>,myEnum:string,myLongOrFloat:uniontype<int>,myIntList:array<int>>");
}
 

public static TypeInfo buildComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myInt:int,myMap:map<string,double>,myEnum:string,myLongOrFloat:uniontype<int>,myIntList:array<int>>");
}
 

private void setFieldTypeInfo() {
	this.fieldTypeInfo = TypeInfoUtils
			.getTypeInfoFromTypeString(this.fieldType);
}
 

public static TypeInfo buildNestedComplexOrcSchema() {
    return TypeInfoUtils.getTypeInfoFromTypeString("struct<myMapOfArray:map<string,array<double>>,myArrayOfMap:array<map<string,string>>>");
}
 

@Test
public void testOrcCompressFile() {
    // TODO: add test for blob and uuid
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
            "struct<" +
            "name:string," +
            "p_boolean:boolean," +
            "p_byte:tinyint," +
            "p_int:int," +
            "p_long:bigint," +
            "p_float:float," +
            "p_double:double," +
            "p_string:string," +
            "p_date:date" +
            ">");

    Date now = Date.valueOf("2019-12-09");
    ioUtil.writeOrc("vertex_person.orc", typeInfo,
                    "marko", true, (byte) 1, 2, 3L,
                    4.0F, 5.0D, "marko", now);
    String[] args = new String[]{
            "-f", structPath("orc_compress_file/struct.json"),
            "-s", configPath("orc_compress_file/schema.groovy"),
            "-g", GRAPH,
            "-h", SERVER,
            "--batch-insert-threads", "2",
            "--test-mode", "true"
    };
    HugeGraphLoader.main(args);

    List<Vertex> vertices = CLIENT.graph().listVertices();
    Assert.assertEquals(1, vertices.size());

    Vertex vertex = vertices.get(0);
    Assert.assertEquals(true, vertex.property("p_boolean"));
    Assert.assertEquals(1, vertex.property("p_byte"));
    Assert.assertEquals(2, vertex.property("p_int"));
    Assert.assertEquals(3, vertex.property("p_long"));
    Assert.assertEquals(4.0D, vertex.property("p_float"));
    Assert.assertEquals(5.0D, vertex.property("p_double"));
    Assert.assertEquals("marko", vertex.property("p_string"));
    Assert.assertEquals(now.toEpochMilli(), vertex.property("p_date"));
}