Java Code Examples for org.apache.avro.Schema.Field#getProp()

/**
 *
 */
private static ByteBuffer encodeLogicalType(final Field field, final String fieldValue) {
    String logicalType = field.getProp("logicalType");
    if (!"decimal".contentEquals(logicalType)) {
        throw new IllegalArgumentException("The field '" + field.name() + "' has a logical type of '" + logicalType
                + "' that is currently not supported.");
    }

    JsonNode rawPrecision = field.getJsonProp("precision");
    if (null == rawPrecision) {
        throw new IllegalArgumentException("The field '" + field.name() + "' is missing the required precision property");
    }
    int precision = rawPrecision.asInt();
    JsonNode rawScale = field.getJsonProp("scale");
    int scale = null == rawScale ? 0 : rawScale.asInt();

    NumberFormat numberFormat = DecimalFormat.getInstance();
    numberFormat.setGroupingUsed(false);
    normalizeNumberFormat(numberFormat, scale, precision);
    BigDecimal decimal = null == fieldValue ? new BigDecimal(retrieveDefaultFieldValue(field).asText()) : new BigDecimal(fieldValue);
    return ByteBuffer.wrap(numberFormat.format(decimal).getBytes(StandardCharsets.UTF_8));
}
 

/**
 * Initialize converters per each schema field
 * 
 * @param schema
 *            design schema
 */
private void initConverters(Schema schema) {
	converters = new StringConverter[size];
	List<Field> fields = schema.getFields();
	for (int i = 0; i < size; i++) {
		Field field = fields.get(i);
		Schema fieldSchema = field.schema();
		fieldSchema = AvroUtils.unwrapIfNullable(fieldSchema);
		if (LogicalTypeUtils.isLogicalTimestampMillis(fieldSchema)) {
			String datePattern = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
			converters[i] = new StringTimestampConverter(datePattern);
		} else {
			Type type = fieldSchema.getType();
			converters[i] = converterRegistry.get(type);
		}
	}
}
 

private Schema initDefaultSchema(Schema designSchema) {
    AvroRegistry avroReg = new AvroRegistry();
    FieldAssembler<Schema> record = SchemaBuilder.record("Main").fields();
    for (SplunkJSONEventField metadataField : SplunkJSONEventField.getMetadataFields()) {
        Schema base = avroReg.getConverter(metadataField.getDataType()).getSchema();
        FieldBuilder<Schema> fieldBuilder = record.name(metadataField.getName());
        if (metadataField.getName().equals(SplunkJSONEventField.TIME.getName())) {
            String datePattern;
            Field designField = designSchema.getField(metadataField.getName());
            if (designField != null) {
                datePattern = designField.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            } else {
                datePattern = designSchema.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            }
            if (datePattern == null || datePattern.isEmpty()) {
                datePattern = "dd-MM-yyyy";
            }
            fieldBuilder.prop(SchemaConstants.TALEND_COLUMN_PATTERN, datePattern);
        }
        fieldBuilder.type(AvroUtils.wrapAsNullable(base)).noDefault();
    }
    Schema defaultSchema = record.endRecord();
    return defaultSchema;
}
 

public List<String> getMarketoColumns(Schema schema) {
    List<String> result = new ArrayList<>();
    Map<String, String> mappings = getInputedNameMappingsForMarketo();
    String marketoCol = null;
    String schemaCol = null;
    for (Field f : schema.getFields()) {
        marketoCol = mappings.get(f.name());
        if (StringUtils.isEmpty(marketoCol)) {
            schemaCol = f.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
            if (!StringUtils.isEmpty(schemaCol)) {
                marketoCol = schemaCol;
            } else {
                marketoCol = f.name();
            }
        }
        result.add(marketoCol);
    }
    return result;
}
 

@Override
public void onVisit(Field field, Collection<String> breadcrumb) {
  if (field.getProp(LogicalType.LOGICAL_TYPE_PROP) != null) {
    String path = breadcrumb.stream().collect(joining("/", "/", ""));
    throw new IllegalArgumentException("Invalid logical type declared on field at " + path);
  }
}
 

private static StringSchemaInfo getStringSchemaInfo(TSnowflakeOutputProperties outputProperties, Schema mainSchema,
        List<Field> columns) {
    boolean isUpperCase = false;
    boolean upsert = false;
    if (outputProperties != null) {
        isUpperCase = outputProperties.convertColumnsAndTableToUppercase.getValue();
        upsert = UPSERT.equals(outputProperties.outputAction.getValue());
    }

    List<String> keyStr = new ArrayList<>();
    List<String> columnsStr = new ArrayList<>();

    int i = 0;
    for (Field overlapField : columns) {
        Field f = overlapField == null ? mainSchema.getFields().get(i) : overlapField;
        i++;
        if (Boolean.valueOf(f.getProp(SnowflakeAvroRegistry.TALEND_FIELD_AUTOINCREMENTED))) {
            continue;
        }
        String dbColumnName = f.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
        if (dbColumnName == null) {
            dbColumnName = f.name();
        }

        String fName = isUpperCase ? dbColumnName.toUpperCase() : dbColumnName;
        columnsStr.add(fName);
        if (null != f.getProp(SchemaConstants.TALEND_COLUMN_IS_KEY)) {
            keyStr.add(fName);
        }
    }

    if (upsert) {
        keyStr.clear();
        String upserKeyColumn = outputProperties.upsertKeyColumn.getValue();
        keyStr.add(isUpperCase ? upserKeyColumn.toUpperCase() : upserKeyColumn);
    }

    return new StringSchemaInfo(keyStr, columnsStr);
}
 

protected Object getFieldValue(Object inputValue, Field field) {
    Schema s = AvroUtils.unwrapIfNullable(field.schema());
    if (inputValue != null && inputValue instanceof String && ((String) inputValue).isEmpty()) {
        return emptyStringValue;
    } else if (null == inputValue || inputValue instanceof String) {
        return inputValue;
    } else if (AvroUtils.isSameType(s, AvroUtils._date())) {// TODO improve the performance as no need to get the
                                                            // runtimefield object from map every time
        // if customer set the schema by self instead of retrieve schema function,
        // the snowflake date type like : date, time, timestamp with time zone, timestamp with local time zone,
        // timestamp without time zone all may be the column type in database table
        // please see the test : SnowflakeDateTypeTestIT which show the details about terrible snowflake jdbc date
        // type support, all control by client!
        // so we have to process the date type and format it by different database data type
        boolean isUpperCase = false;
        if (sprops != null) {
            // keep the same logic with the method : getStringSchemaInfo as getStringSchemaInfo is used to init the
            // loader with the right db column name(are you sure?)
            isUpperCase = sprops.convertColumnsAndTableToUppercase.getValue();
        }
        String dbColumnName = field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
        if (dbColumnName == null) {
            dbColumnName = field.name();
        }
        dbColumnName = isUpperCase ? dbColumnName.toUpperCase() : dbColumnName;
        Field runtimeField = dbColumnName2RuntimeField.get(dbColumnName);

        if (runtimeField != null) {
            s = AvroUtils.unwrapIfNullable(runtimeField.schema());
        } else {
            // TODO this is the old action, we keep it if can't fetch the type by the schema db column name
            // consider to adjust it
            Date date = (Date) inputValue;
            return date.getTime();
        }
    }

    return formatIfAnySnowflakeDateType(inputValue, s);
}
 

protected Schema initRuntimeSchemaAndMapIfNecessary() throws IOException {
    if (!dbColumnName2RuntimeField.isEmpty()) {
        return null;
    }
    String tableName = sprops.convertColumnsAndTableToUppercase.getValue() ? sprops.getTableName().toUpperCase()
            : sprops.getTableName();
    Schema runtimeSchema = sink.getSchema(container, processingConnection, tableName);
    if (runtimeSchema != null) {
        for (Field field : runtimeSchema.getFields()) {
            String dbColumnName = field.getProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME);
            dbColumnName2RuntimeField.put(dbColumnName, field);
        }
    }
    return runtimeSchema;
}
 

/**
 * Check if the Avro field is of Date type
 *
 * @param field
 * @return
 */
public static boolean isDateTypeField(Field field) {
    if (field == null) {
        return false;
    }
    if (!Type.LONG.equals(getFieldType(field))) {
        return false;
    }
    String clazz = field.getProp(SchemaConstants.JAVA_CLASS_FLAG);
    String pattr = field.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
    return (clazz != null && clazz.equals(Date.class.getCanonicalName())) || !StringUtils.isEmpty(pattr);
}
 

public static Schema extractSchemaFromAvroWithoutTime(File avroFile)
    throws IOException {
  DataFileStream<GenericRecord> dataStream =
      new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>());
  Schema schema = new Schema();

  for (final Field field : dataStream.getSchema().getFields()) {
    try {
      getColumnType(field);
    } catch (Exception e) {
      LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.",
          field.name(), field.schema().getType());
      continue;
    }
    final String columnName = field.name();
    final String pinotType = field.getProp("pinotType");

    final FieldSpec fieldSpec;
    if (pinotType != null && "METRIC".equals(pinotType)) {
      fieldSpec = new MetricFieldSpec();
    } else {
      fieldSpec = new DimensionFieldSpec();
    }

    fieldSpec.setName(columnName);
    fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName)));
    fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName)));
    schema.addField(fieldSpec);
  }

  dataStream.close();
  return schema;
}
 

/**
 * Writes {@link GenericRecord} as CSV (delimited) record to the
 * {@link OutputStream} using provided delimiter.
 */
public static void write(GenericRecord record, char delimiter, OutputStream out) {
    List<Field> fields = record.getSchema().getFields();

    String delimiterToUse = "";
    try {
        for (Field field : fields) {
            out.write(delimiterToUse.getBytes(StandardCharsets.UTF_8));
            Object fieldValue = record.get(field.name());
            if (null == fieldValue) {
                out.write(new byte[0]);
            } else {
                if (Type.BYTES == field.schema().getType()) {
                    // need to create it from the ByteBuffer it is serialized as.
                    // need to ensure the type is one of the logical ones we support and if so convert it.
                    if(!"decimal".contentEquals(field.getProp("logicalType"))){
                        throw new IllegalArgumentException("The field '" + field.name() + "' has a logical type of '" +
                                field.getProp("logicalType") + "' that is currently not supported.");
                    }

                    JsonNode rawPrecision = field.getJsonProp("precision");
                    if(null == rawPrecision){
                        throw new IllegalArgumentException("The field '" + field.name() + "' is missing the required precision property");
                    }
                    int precision = rawPrecision.asInt();
                    JsonNode rawScale = field.getJsonProp("scale");
                    int scale = null == rawScale ? 0 : rawScale.asInt();

                    // write out the decimal with the precision and scale.
                    NumberFormat numberFormat = DecimalFormat.getInstance();
                    numberFormat.setGroupingUsed(false);
                    normalizeNumberFormat(numberFormat, scale, precision);
                    final String rawValue  = new String(((ByteBuffer)fieldValue).array());
                    out.write(numberFormat.format(new BigDecimal(rawValue)).getBytes(StandardCharsets.UTF_8));
                } else {
                    out.write(fieldValue.toString().getBytes(StandardCharsets.UTF_8));
                }
            }
            if (delimiterToUse.length() == 0) {
                delimiterToUse = String.valueOf(delimiter);
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("Failed to parse AVRO Record", e);
    }
}
 

public RowWriter(List<JDBCSQLBuilder.Column> columnList, Schema inputSchema, Schema componentSchema,
        PreparedStatement statement, boolean debug, String sql) {
    this.debug = debug;

    if (debug) {
        debugUtil = new DebugUtil(sql);
    }

    List<TypeWriter> writers = new ArrayList<TypeWriter>();

    int statementIndex = 0;

    for (JDBCSQLBuilder.Column column : columnList) {
        Field inputField = CommonUtils.getField(inputSchema, column.columnLabel);

        Field componentField = CommonUtils.getField(componentSchema, column.columnLabel);
        int inputValueLocation = inputField.pos();
        String pattern = componentField.getProp("talend.field.pattern");
        statementIndex++;

        Schema basicSchema = AvroUtils.unwrapIfNullable(componentField.schema());
        // TODO any difference for nullable
        // boolean nullable = AvroUtils.isNullable(componentField.schema());

        TypeWriter writer = null;

        if (AvroUtils.isSameType(basicSchema, AvroUtils._string())) {
            writer = new StringTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._int())) {
            writer = new IntTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._date())) {
            writer = new DateTypeWriter(statement, statementIndex, inputValueLocation, pattern);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._decimal())) {
            writer = new BigDecimalTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._long())) {
            writer = new LongTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._double())) {
            writer = new DoubleTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._float())) {
            writer = new FloatTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._boolean())) {
            writer = new BooleanTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._short())) {
            writer = new ShortTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._character())) {
            writer = new CharacterTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._byte())) {
            writer = new ByteTypeWriter(statement, statementIndex, inputValueLocation);
        } else if (AvroUtils.isSameType(basicSchema, AvroUtils._bytes())) {
            writer = new BytesTypeWriter(statement, statementIndex, inputValueLocation);
        } else {
            writer = new ObjectTypeWriter(statement, statementIndex, inputValueLocation);
        }

        writers.add(writer);
    }

    typeWriters = writers.toArray(new TypeWriter[0]);
}