org.codehaus.jackson.node.NullNode Java Examples

/**
 * It takes a record structure in <String, HiveTypeInfo> format.
 * Generate a schema and return in String.
 * @param record : record structure
 * @return String representation of Avro schema.
 * @throws StageException: If record contains unsupported type
 */
@Override
public String inferSchema(Map<String, HiveTypeInfo> record)
    throws StageException
{
  Map<String, Schema> fields = new LinkedHashMap<>();
  for(Map.Entry<String, HiveTypeInfo> pair:  record.entrySet()) {
    if(!HiveMetastoreUtil.validateObjectName(pair.getKey())) {
      throw new HiveStageCheckedException(Errors.HIVE_30, pair.getKey());
    }
    Schema columnSchema = Schema.createUnion(ImmutableList.of(Schema.create(Schema.Type.NULL), traverse(pair)));
    // We always set default value to null
    columnSchema.addProp("default", NullNode.getInstance());
    fields.put(pair.getKey(), columnSchema);
  }
  Schema schema =  buildSchema(fields);
  return schema.toString();
}
 

private Schema convertFields(String name, List<Type> parquetFields) {
  List<Schema.Field> fields = new ArrayList<Schema.Field>();
  for (Type parquetType : parquetFields) {
    Schema fieldSchema = convertField(parquetType);
    if (parquetType.isRepetition(REPEATED)) {
      throw new UnsupportedOperationException("REPEATED not supported outside LIST or MAP. Type: " + parquetType);
    } else if (parquetType.isRepetition(Type.Repetition.OPTIONAL)) {
      fields.add(new Schema.Field(
          parquetType.getName(), optional(fieldSchema), null, NullNode.getInstance()));
    } else { // REQUIRED
      fields.add(new Schema.Field(parquetType.getName(), fieldSchema, null, null));
    }
  }
  Schema schema = Schema.createRecord(name, null, null, false);
  schema.setFields(fields);
  return schema;
}
 

@Override
public Schema getEvolvedSchema(Schema original) {
  List<Schema.Field> fields = Lists.newArrayList();
  fields.add(new Schema.Field("new",
    Schema.createUnion(ImmutableList.of(
        Schema.create(Schema.Type.NULL),
        Schema.create(Schema.Type.STRING))),
    "New field", NullNode.getInstance()));

  for (Schema.Field field : original.getFields()) {
    fields.add(new Schema.Field(field.name(), field.schema(), field.doc(),
      field.defaultValue()));
  }

  Schema evolved = Schema.createRecord(original.getName(), original.getDoc(),
    original.getNamespace(), false);
  evolved.setFields(fields);

  return evolved;
}
 

public static Schema.Field createUnionField(String name, Schema... schemas) {
  List<Schema> typeList = new ArrayList<>();
  typeList.add(Schema.create(Schema.Type.NULL));
  typeList.addAll(Arrays.asList(schemas));

  Schema unionSchema = Schema.createUnion(typeList);
  return new Schema.Field(name, unionSchema, null, NullNode.getInstance(), Schema.Field.Order.ASCENDING);
}
 

public static Schema.Field createPrimitiveUnionFieldSchema(String name, Schema.Type... types) {
  List<Schema> typeList = new ArrayList<>();
  typeList.add(Schema.create(Schema.Type.NULL));
  typeList.addAll(Arrays.asList(types).stream().map(Schema::create).collect(Collectors.toList()));

  Schema unionSchema = Schema.createUnion(typeList);
  return new Schema.Field(name, unionSchema, null, NullNode.getInstance(), Schema.Field.Order.ASCENDING);
}
 

public Schema generate(String schemaNameOverride) throws IOException {
  ClassWriter classWriter = new ClassWriter(options, connManager,
      tableName, null);
  Map<String, Integer> columnTypes = classWriter.getColumnTypes();
  String[] columnNames = classWriter.getColumnNames(columnTypes);

  List<Field> fields = new ArrayList<Field>();
  for (String columnName : columnNames) {
    String cleanedCol = AvroUtil.toAvroIdentifier(ClassWriter.toJavaIdentifier(columnName));
    int sqlType = columnTypes.get(columnName);
    Schema avroSchema = toAvroSchema(sqlType, columnName);
    Field field = new Field(cleanedCol, avroSchema, null,  NullNode.getInstance());
    field.addProp("columnName", columnName);
    field.addProp("sqlType", Integer.toString(sqlType));
    fields.add(field);
  }

  TableClassName tableClassName = new TableClassName(options);
  String shortClassName = tableClassName.getShortClassForTable(tableName);
  String avroTableName = (tableName == null ? TableClassName.QUERY_RESULT : tableName);
  String avroName = schemaNameOverride != null ? schemaNameOverride :
      (shortClassName == null ? avroTableName : shortClassName);
  String avroNamespace = tableClassName.getPackageForTable();

  String doc = "Sqoop import of " + avroTableName;
  Schema schema = Schema.createRecord(avroName, doc, avroNamespace, false);
  schema.setFields(fields);
  schema.addProp("tableName", avroTableName);
  return schema;
}
 

private static Schema initRecordKeySchema() {
  Schema.Field recordKeyField =
      new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
  Schema recordKeySchema = Schema.createRecord("HoodieRecordKey", "", "", false);
  recordKeySchema.setFields(Collections.singletonList(recordKeyField));
  return recordKeySchema;
}
 

/**
 * Fetch schema for record key and partition path.
 */
public static Schema getRecordKeyPartitionPathSchema() {
  List<Schema.Field> toBeAddedFields = new ArrayList<>();
  Schema recordSchema = Schema.createRecord("HoodieRecordKey", "", "", false);

  Schema.Field recordKeyField =
      new Schema.Field(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
  Schema.Field partitionPathField =
      new Schema.Field(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", NullNode.getInstance());

  toBeAddedFields.add(recordKeyField);
  toBeAddedFields.add(partitionPathField);
  recordSchema.setFields(toBeAddedFields);
  return recordSchema;
}
 

/**
 * Add null fields to passed in schema. Caller is responsible for ensuring there is no duplicates. As different query
 * engines have varying constraints regarding treating the case-sensitivity of fields, its best to let caller
 * determine that.
 *
 * @param schema Passed in schema
 * @param newFieldNames Null Field names to be added
 */
public static Schema appendNullSchemaFields(Schema schema, List<String> newFieldNames) {
  List<Field> newFields = schema.getFields().stream()
      .map(field -> new Field(field.name(), field.schema(), field.doc(), field.defaultValue())).collect(Collectors.toList());
  for (String newField : newFieldNames) {
    newFields.add(new Schema.Field(newField, METADATA_FIELD_SCHEMA, "", NullNode.getInstance()));
  }
  Schema newSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
  newSchema.setFields(newFields);
  return newSchema;
}
 

@Test
public void testJsonNodeNullWithDefaultValues() {
  List<Schema.Field> fields = new ArrayList<>();
  Schema initialSchema = Schema.createRecord("test_record", "test record", "org.test.namespace", false);
  Schema.Field field1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field field3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  fields.add(field1);
  fields.add(field2);
  fields.add(field3);
  initialSchema.setFields(fields);
  GenericRecord rec = new GenericData.Record(initialSchema);
  rec.put("key", "val");
  rec.put("key1", "val1");
  rec.put("key2", "val2");

  List<Schema.Field> evolvedFields = new ArrayList<>();
  Schema evolvedSchema = Schema.createRecord("evolved_record", "evolved record", "org.evolved.namespace", false);
  Schema.Field evolvedField1 = new Schema.Field("key", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField2 = new Schema.Field("key1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField3 = new Schema.Field("key2", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  Schema.Field evolvedField4 = new Schema.Field("evolved_field", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
  Schema.Field evolvedField5 = new Schema.Field("evolved_field1", HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
  evolvedFields.add(evolvedField1);
  evolvedFields.add(evolvedField2);
  evolvedFields.add(evolvedField3);
  evolvedFields.add(evolvedField4);
  evolvedFields.add(evolvedField5);
  evolvedSchema.setFields(evolvedFields);

  GenericRecord rec1 = HoodieAvroUtils.rewriteRecord(rec, evolvedSchema);
  //evolvedField4.defaultVal() returns a JsonProperties.Null instance.
  assertNull(rec1.get("evolved_field"));
  //evolvedField5.defaultVal() returns null.
  assertNull(rec1.get("evolved_field1"));
}
 

/**
 * Returns default value for given field or null if no default value should be used.
 */
private JsonNode getDefaultValue(Schema schema) {
  if(getConfig().avroNullableFields && getConfig().avroDefaultNullable) {
    return NullNode.getInstance();
  }

  if(!getConfig().avroNullableFields && defaultValuesForTypes.containsKey(schema.getType())) {
    return defaultValuesForTypes.get(schema.getType());
  }

  return null;
}
 

public static Schema getSchema(SeekableInput input) throws IOException
{
    DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
    DataFileReader<GenericRecord> dataFileReader =
            new DataFileReader<GenericRecord>(input, datumReader);
    Schema schema = dataFileReader.getSchema();

    if (PadDefaultNullsToSchema)
    {
        // a list of "cloned" fields, with optional default value set to null
        ArrayList<Field> paddedFields = new ArrayList<Field>();

        for (Field field: schema.getFields())
        {
            // should this field be padded?
            boolean needsNullPadding = (field.schema() != null) // the field has nested schema
                && (field.schema().getType().equals(Type.UNION)) // the nested schema is UNION
                && (field.schema().getTypes().get(0).getType().equals(Type.NULL)); // the first element of union is NULL type

            JsonNode defValue = needsNullPadding ? NullNode.getInstance() : field.defaultValue();

            Field f = new Field(field.name(), field.schema(), field.doc(), defValue);
            paddedFields.add(f);
        }

        schema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), schema.isError());
        schema.setFields(paddedFields);
    }

    return schema;
}
 

private Object processElement(final String key, final JsonNode element) {
    Object rval = null;
    if (element instanceof ObjectNode) {
        Map<String, Object> r2 = processObject((ObjectNode) element);
        
        // convert TreeMap entries into Link instances.
        if (key.equals(Entity.LINKS_KEY)) {
            
            String refName = (String) r2.get(Link.LINK_RESOURCE_KEY);
            String hrefString = (String) r2.get(Link.LINK_HREF_KEY);
            
            try {
                rval = new BasicLink(refName, new URL(hrefString));
                
            } catch (MalformedURLException e) {
                rval = r2;
            }
        } else {
            rval = r2;
        }
    } else if (element instanceof NullNode) {
        rval = null;
    } else if (element instanceof ArrayNode) {
        rval = processArray(key, (ArrayNode) element);
    } else {
        rval = processPrimitive(element);
    }
    
    return rval;
}
 

@Test
public void withJsonNodeNull() throws IOException {

    JsonNode node = NullNode.getInstance();
    ObjectMapper mapper = new ObjectMapper();

    NilUnionTypeJ1 b = mapper.treeToValue(node, NilUnionTypeJ1.class);

    assertTrue(b.isNil());
}