Java Code Examples for org.apache.avro.Schema.Field#addProp()

public static Schema getSchemaForThisFields(String schemaName, FieldDescription[] fields, String[] keys) {
    Schema schema = Schema.createRecord(schemaName, "", "", false);
    List<Field> fieldList = new ArrayList<>();
    if (fields == null) {
        return null;
    }
    for (FieldDescription field : fields) {
        Field f = field.toAvroField();
        for (String key : keys) {
            if (field.getName().equals(key)) {
                f.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
            }
        }
        fieldList.add(f);
    }
    schema.setFields(fieldList);

    return schema;
}
 

@Test
public void fieldWithPiiUnion3Types() throws Exception {
  Schema schema = SchemaBuilder.unionOf().intType().and().booleanType().and().floatType().endUnion();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  try {
    underTest.onVisit(field, breadcrumb);
    fail();
  } catch (InvalidPiiAnnotationException e) {
    assertThat(e.getPath(), is("/a/b"));
  }
  verify(underTest, never()).onPiiField(field, breadcrumb);
}
 

@Test
public void testGenerateNewField() throws Exception {
    Field in = new Schema.Field("email", AvroUtils._string(), "doc", null, Order.ASCENDING);
    in.addProp("test", "testvalue");
    Field out = MarketoUtils.generateNewField(in);
    assertEquals("email", out.name());
    assertEquals("string", out.schema().getType().getName());
    assertEquals("doc", out.doc());
    assertEquals(Order.ASCENDING, out.order());
    assertNotNull(out.getProp("test"));
    assertEquals("testvalue", out.getProp("test"));
}
 

private Field getMigratedField(Field origin, Schema expectedSchema, String expectedDIType) {
    Field expectedField = new Schema.Field(origin.name(), expectedSchema, origin.doc(), origin.defaultVal(), origin.order());
    for (Map.Entry<String, Object> entry : origin.getObjectProps().entrySet()) {
        if ("di.column.talendType".equals(entry.getKey())) {
            expectedField.addProp("di.column.talendType", expectedDIType);
        } else {
            expectedField.addProp(entry.getKey(), entry.getValue());
        }
    }
    return expectedField;
}
 

public static List<Field> getSchemaFieldsList(Schema schema) {
    List<Field> result = new ArrayList<>();
    for (Field f : schema.getFields()) {
        Field nf = new Field(f.name(), f.schema(), f.doc(), f.defaultVal());
        nf.getObjectProps().putAll(f.getObjectProps());
        for (Map.Entry<String, Object> entry : f.getObjectProps().entrySet()) {
            nf.addProp(entry.getKey(), entry.getValue());
        }
        result.add(nf);
    }
    return result;
}
 

public static Schema infer(ResultSetMetaData metadata, Dbms mapping, boolean enableSpecialTableName) throws SQLException {
    List<Field> fields = new ArrayList<>();

    Set<String> existNames = new HashSet<String>();
    int index = 0;
    
    int count = metadata.getColumnCount();
    for (int i = 1; i <= count; i++) {
        int size = metadata.getPrecision(i);
        int scale = metadata.getScale(i);
        boolean nullable = ResultSetMetaData.columnNullable == metadata.isNullable(i);

        int dbtype = metadata.getColumnType(i);
        String fieldName = metadata.getColumnLabel(i);
        String dbColumnName = metadata.getColumnName(i);

        // not necessary for the result schema from the query statement
        boolean isKey = false;

        String columnTypeName = metadata.getColumnTypeName(i).toUpperCase();

        String validName = NameUtil.correct(fieldName, index++, existNames);
        existNames.add(validName);
        
        Field field = sqlType2Avro(size, scale, dbtype, nullable, validName, dbColumnName, null, isKey, mapping,
                columnTypeName);
        if(enableSpecialTableName && !validName.equals(dbColumnName)){
            field.addProp(ENABLE_SPECIAL_TABLENAME,"true");
        }
        fields.add(field);
    }

    return Schema.createRecord("DYNAMIC", null, null, false, fields);
}
 

@Test
public void fieldWithPiiNullableInt() throws Exception {
  Schema schema = SchemaBuilder.unionOf().intType().and().nullType().endUnion();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  try {
    underTest.onVisit(field, breadcrumb);
    fail();
  } catch (InvalidPiiAnnotationException e) {
    assertThat(e.getPath(), is("/a/b"));
  }
  verify(underTest, never()).onPiiField(field, breadcrumb);
}
 

@Test
public void fieldWithPiiUnionNotNullable() throws Exception {
  Schema schema = SchemaBuilder.unionOf().intType().and().booleanType().endUnion();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  try {
    underTest.onVisit(field, breadcrumb);
    fail();
  } catch (InvalidPiiAnnotationException e) {
    assertThat(e.getPath(), is("/a/b"));
  }
  verify(underTest, never()).onPiiField(field, breadcrumb);
}
 

@Test
public void fieldWithNullableBytesPii() throws Exception {
  Schema schema = SchemaBuilder.unionOf().bytesType().and().nullType().endUnion();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  underTest.onVisit(field, breadcrumb);
  verify(underTest).onPiiField(field, breadcrumb);
}
 

@Test
public void fieldWithNullableStringPii() throws Exception {
  Schema schema = SchemaBuilder.unionOf().stringType().and().nullType().endUnion();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  underTest.onVisit(field, breadcrumb);
  verify(underTest).onPiiField(field, breadcrumb);
}
 

private static void setScale(Field field, boolean ignoreScale, int scale) {
    if (ignoreScale) {
        return;
    }

    field.addProp(SchemaConstants.TALEND_COLUMN_PRECISION, String.valueOf(scale));
}
 

@Test
public void fieldWithStringPii() throws Exception {
  Schema schema = SchemaBuilder.builder().stringType();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);
  underTest.onVisit(field, breadcrumb);
  verify(underTest).onPiiField(field, breadcrumb);
}
 

@Test
public void typicalNullableBytes() throws Exception {
  Schema schema = SchemaBuilder.builder().bytesType();
  Schema union = SchemaBuilder.unionOf().type(schema).and().nullType().endUnion();
  Field field = new Field("b", union, null, (Object) null);
  field.addProp(SENSITIVITY, PII);

  underTest.onPiiField(field, null);

  assertThat(schema.getProp("logicalType"), is("pii-bytes"));
}
 

@Test
public void typicalBytes() throws Exception {
  Schema schema = SchemaBuilder.builder().bytesType();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);

  underTest.onPiiField(field, null);

  assertThat(schema.getProp("logicalType"), is("pii-bytes"));
}
 

@Test
public void typicalString() throws Exception {
  Schema schema = SchemaBuilder.builder().stringType();
  Field field = new Field("b", schema, null, (Object) null);
  field.addProp(SENSITIVITY, PII);

  underTest.onPiiField(field, null);

  assertThat(schema.getProp("logicalType"), is("pii-string"));
}
 

@Test(expected = IllegalArgumentException.class)
public void logicalTypeField() throws Exception {
  Field field = new Field("f", SchemaBuilder.builder().intType(), null, (String) null);
  field.addProp(LogicalType.LOGICAL_TYPE_PROP, "date");
  underTest.onVisit(field, breadcrumb);
}
 

private static Field sqlType2Avro(int size, int scale, int dbtype, boolean nullable, String name, String dbColumnName,
        Object defaultValue, boolean isKey, Dbms mapping, String columnTypeName) {
    MappingType<DbmsType, TalendType> mt = mapping.getDbmsMapping(columnTypeName);
    
    Field field = null;
    boolean isIgnoreLength = false;
    boolean isIgnorePrecision = false;
    
    if(mt!=null) {
        TalendType talendType = mt.getDefaultType();
        DbmsType sourceType = mt.getSourceType();
        Schema schema = convertToAvro(talendType);
        field = wrap(nullable, schema, name);
        
        isIgnoreLength = sourceType.isIgnoreLength();
        isIgnorePrecision = sourceType.isIgnorePrecision();
    } else {
        field = wrap(nullable, AvroUtils._string(), name);
    }

    switch (dbtype) {
    case java.sql.Types.VARCHAR:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.INTEGER:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.DECIMAL:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    case java.sql.Types.BIGINT:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.NUMERIC:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    case java.sql.Types.TINYINT:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.DOUBLE:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    case java.sql.Types.FLOAT:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    case java.sql.Types.DATE:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd");
        break;
    case java.sql.Types.TIME:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, "HH:mm:ss");
        break;
    case java.sql.Types.TIMESTAMP:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, "yyyy-MM-dd HH:mm:ss.SSS");
        break;
    case java.sql.Types.BOOLEAN:
        break;
    case java.sql.Types.REAL:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    case java.sql.Types.SMALLINT:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.LONGVARCHAR:
        setPrecision(field, isIgnoreLength, size);
        break;
    case java.sql.Types.CHAR:
        setPrecision(field, isIgnoreLength, size);
        break;
    default:
        setPrecision(field, isIgnoreLength, size);
        setScale(field, isIgnorePrecision, scale);
        break;
    }

    field.addProp(SchemaConstants.TALEND_COLUMN_DB_TYPE, columnTypeName);
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, dbColumnName);

    if (defaultValue != null) {
        field.addProp(SchemaConstants.TALEND_COLUMN_DEFAULT, String.valueOf(defaultValue));
    }

    if (isKey) {
        field.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
    }

    return field;
}
 

@Override
protected Field sqlType2Avro(int size, int scale, int dbtype, boolean nullable, String name, String dbColumnName,
        Object defaultValue, boolean isKey, boolean isAutoIncremented) {
    if (!nullable) {
        //snowflake schema contain empty string as default Values if not specified
        defaultValue = checkNotNullableDefaultValueCorrect(dbtype, defaultValue);
    }
    Field field = null;
    Schema schema = null;
    name = AvroNamesValidationHelper.getAvroCompatibleName(NameUtil.correct(dbColumnName, 0, Collections.<String>emptySet()));
    switch (dbtype) {
    case java.sql.Types.VARCHAR:
    case java.sql.Types.LONGVARCHAR:
    case java.sql.Types.CHAR:
        schema = AvroUtils._string();
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, size);
        break;
    case java.sql.Types.INTEGER:
    case java.sql.Types.DECIMAL:
    case java.sql.Types.BIGINT:
    case java.sql.Types.NUMERIC:
    case java.sql.Types.TINYINT:
    case java.sql.Types.SMALLINT:
        schema = AvroUtils._decimal();
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PRECISION, size);
        field.addProp(SchemaConstants.TALEND_COLUMN_SCALE, scale);
        break;
    case java.sql.Types.DOUBLE:
    case java.sql.Types.FLOAT:
    case java.sql.Types.REAL:
        schema = AvroUtils._double();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    case java.sql.Types.DATE:
        schema = AvroUtils._int();
        LogicalTypes.date().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, SnowflakeConstants.TALEND_DEFAULT_DATE_PATTERN);
        break;
    case java.sql.Types.TIME:
        schema = AvroUtils._int();
        LogicalTypes.timeMillis().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, SnowflakeConstants.TALEND_DAFEULT_TIME_PATTERN);
        /** tell Avro converter how to process Original Avro logical type time
         * if value is "TALEND_DATE", it mean use Talend Date, if not, will use Talend Integer like before
         * we add this only one purpose : for the old job(before current commit), we keep Talend Integer, for new job, we use Talend Date
         *
         * this affect the studio and DI model level which will affect the expected assign value type from input component(snowflakeinput)
         * and the expected passed value type to output component(snowflakeoutput),
         * and we adjust snowflake runtime to make it can process the DATE and INTEGER both for input/output
         * */
        field.addProp(SnowflakeConstants.LOGICAL_TIME_TYPE_AS, SnowflakeConstants.AS_TALEND_DATE);
        break;
    case java.sql.Types.TIMESTAMP:
        schema = AvroUtils._long();
        LogicalTypes.timestampMillis().addToSchema(schema);
        field = wrap(name, schema, nullable, defaultValue);
        field.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, SnowflakeConstants.TALEND_DAFEULT_TIMESTAMP_PATTERN);
        break;
    case java.sql.Types.BOOLEAN:
        schema = AvroUtils._boolean();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    default:
        schema = AvroUtils._string();
        field = wrap(name, schema, nullable, defaultValue);
        break;
    }

    field.addProp(SchemaConstants.TALEND_COLUMN_DB_TYPE, dbtype);
    field.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, dbColumnName);

    //Add property if it is set, otherwise skip
    if (isAutoIncremented) {
        field.addProp(TALEND_FIELD_AUTOINCREMENTED, "true");
    }

    if (defaultValue != null) {
        field.addProp(SchemaConstants.TALEND_COLUMN_DEFAULT, defaultValue);
    }

    if (isKey) {
        field.addProp(SchemaConstants.TALEND_COLUMN_IS_KEY, "true");
    }

    return field;
}
 

public Field toAvroField() {
    Schema fs = null;
    String fname = getName().replaceAll("-", "_");
    //
    fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion();
    switch (getDataType()) {
    case ("string"):
    case ("text"):
    case ("phone"):
    case ("email"):
    case ("url"):
    case ("lead_function"):
    case ("reference"):
        fs = SchemaBuilder.builder().unionOf().nullType().and().stringType().endUnion();
        break;
    case ("integer"):
        fs = SchemaBuilder.builder().unionOf().nullType().and().intType().endUnion();
        break;
    case ("boolean"):
        fs = SchemaBuilder.builder().unionOf().nullType().and().booleanType().endUnion();
        break;
    case ("float"):
    case ("currency"):
        fs = SchemaBuilder.builder().unionOf().nullType().and().floatType().endUnion();
        break;
    case ("date"):
    case ("datetime"):
        fs = SchemaBuilder.builder().unionOf().nullType().and().longType().endUnion();
        fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, getName());
        fs.getTypes().get(1).addProp(SchemaConstants.TALEND_COLUMN_PATTERN, MarketoConstants.DATETIME_PATTERN_REST);
        fs.getTypes().get(1).addProp(SchemaConstants.JAVA_CLASS_FLAG, "java.util.Date");
        break;
    default:
        LOG.warn("Non managed type : {}. for {}. Defaulting to String.", getDataType(), this);
    }
    Field f = new Field(fname, fs, getDisplayName(), (Object) null);
    f.addProp(SchemaConstants.TALEND_COLUMN_DB_COLUMN_NAME, getName());
    if (getLength() != null) {
        f.addProp(SchemaConstants.TALEND_COLUMN_DB_LENGTH, getLength().toString());
    }
    if ("java.util.Date".equals(fs.getTypes().get(1).getProp(SchemaConstants.JAVA_CLASS_FLAG))) {
        f.addProp(SchemaConstants.TALEND_COLUMN_PATTERN, MarketoConstants.DATETIME_PATTERN_REST);
        f.addProp(SchemaConstants.JAVA_CLASS_FLAG, "java.util.Date");
    }
    if (updateable != null && !updateable) {
        f.addProp(SchemaConstants.TALEND_IS_LOCKED, "true");
    }
    //
    if (getId() != null) {
        f.addProp("mktoId", getId().toString());
    }
    f.addProp("mktoType", getDataType());
    return f;
}
 

public static Schema infer(JDBCTableMetadata tableMetadata, Dbms mapping, boolean enableSpecialTableName) throws SQLException {
    DatabaseMetaData databaseMetdata = tableMetadata.getDatabaseMetaData();

    Set<String> keys = getPrimaryKeys(databaseMetdata, tableMetadata.getCatalog(), tableMetadata.getDbSchema(),
            tableMetadata.getTablename());

    Set<String> existNames = new HashSet<String>();
    int index = 0;
    
    try (ResultSet metadata = databaseMetdata.getColumns(tableMetadata.getCatalog(), tableMetadata.getDbSchema(),
            tableMetadata.getTablename(), null)) {
        if (!metadata.next()) {
            return null;
        }

        List<Field> fields = new ArrayList<>();
        String tablename = metadata.getString("TABLE_NAME");

        do {
            int size = metadata.getInt("COLUMN_SIZE");
            int scale = metadata.getInt("DECIMAL_DIGITS");
            int dbtype = metadata.getInt("DATA_TYPE");
            boolean nullable = DatabaseMetaData.columnNullable == metadata.getInt("NULLABLE");

            String columnName = metadata.getString("COLUMN_NAME");
            boolean isKey = keys.contains(columnName);

            String defaultValue = metadata.getString("COLUMN_DEF");
            
            String columnTypeName = metadata.getString("TYPE_NAME");
            
            String validName = NameUtil.correct(columnName, index++, existNames);
            existNames.add(validName);

            Field field = sqlType2Avro(size, scale, dbtype, nullable, validName, columnName, defaultValue, isKey, mapping,
                    columnTypeName);
            if(enableSpecialTableName && !validName.equals(columnName)){
                field.addProp(ENABLE_SPECIAL_TABLENAME,"true");
            }
            fields.add(field);
        } while (metadata.next());

        return Schema.createRecord(NameUtil.correct(tablename, 0, new HashSet<String>()), null, null, false, fields);
    }
}