Java Code Examples for com.streamsets.pipeline.api.Field#Type

@NotNull
private Record makeListRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  ArrayList<Field> fields = new ArrayList<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.add(field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

@NotNull
private Record makeRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.put(fieldName, field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "type",
    description = "Returns the type of the field represented by path 'fieldPath' for the record in context")
public static Field.Type getType(
    @ElParam("fieldPath") String fieldPath) {
  Field.Type type = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field field = record.get(fieldPath);
    if (field != null) {
      type = field.getType();
    }
  }
  return type;
}
 

@NotNull
private Record makeRecord(Map<String, Pair<Field.Type, Object>> fieldMap) {
  Record record = RecordCreator.create();
  Record.Header header = record.getHeader();
  LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
  for (Map.Entry<String, Pair<Field.Type, Object>> entry : fieldMap.entrySet()) {
    String fieldName = entry.getKey();
    Field.Type fieldType = entry.getValue().getLeft();
    Field field = Field.create(fieldType, entry.getValue().getRight());
    if (fieldType == Field.Type.DECIMAL) {
      field.setAttribute(HeaderAttributeConstants.ATTR_SCALE, SCALE);
      field.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, PRECISION);
    }
    fields.put(fieldName, field);
  }
  record.set(Field.create(fields));
  header.setAttribute("table", tableName);
  return record;
}
 

private static void populateColumns(
    Map<String, Field.Type> columnToFieldType,
    Map<String, String> offsetColumns,
    Map<String, String> otherColumns
) {
  columnToFieldType.put(OFFSET_FIELD_NAME, Field.Type.INTEGER);

  offsetColumns.put(
      OFFSET_FIELD_NAME,
      FIELD_TYPE_TO_SQL_TYPE_AND_STRING.get(columnToFieldType.get(OFFSET_FIELD_NAME))
  );

  IntStream.range(0, NUMBER_OF_COLUMNS_PER_TABLE).forEach(columnNumber -> {
    String columnName = COLUMN_NAME_PREFIX + columnNumber;
    columnToFieldType.put(
        columnName,
        OTHER_FIELD_TYPES.get(RANDOM.nextInt(OTHER_FIELD_TYPES.size()))
    );
    otherColumns.put(columnName, FIELD_TYPE_TO_SQL_TYPE_AND_STRING.get(columnToFieldType.get(columnName)));
  });
}
 

private void walkAndCheckField(String fieldName, Field expected, Field actual) {
  Assert.assertEquals(expected.getType(), actual.getType());
  Field.Type type = expected.getType();
  switch (type) {
    case MAP:
    case LIST_MAP:
      Map<String, Field> expectedFieldMap = expected.getValueAsMap();
      Map<String, Field> actualFieldMap = actual.getValueAsMap();
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Expected Field has different keys from Actual Field"), Sets.difference(expectedFieldMap.keySet(), actualFieldMap.keySet()).isEmpty());
      Assert.assertTrue(getMismatchMessage(fieldName, type, "Actual Field has different keys from Expected Field"), Sets.difference(actualFieldMap.keySet(), expectedFieldMap.keySet()).isEmpty());
      for (String key : expectedFieldMap.keySet()) {
        walkAndCheckField(fieldName + "/" + key, expectedFieldMap.get(key), actualFieldMap.get(key));
      }
      break;
    case LIST:
      List<Field> expectedFieldList = expected.getValueAsList();
      List<Field> actualFieldList = actual.getValueAsList();
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Size mistmatch"), expectedFieldList.size(), actualFieldList.size());
      for (int i=0; i < expectedFieldList.size(); i++) {
        walkAndCheckField(fieldName + "[" + i + "]", expectedFieldList.get(i), actualFieldList.get(i));
      }
      break;
    case BYTE_ARRAY:
      Assert.assertArrayEquals(getMismatchMessage(fieldName, type, "Byte Array value mismatch"), expected.getValueAsByteArray(), actual.getValueAsByteArray());
      break;
    default:
      Assert.assertEquals(getMismatchMessage(fieldName, type, "Field Value mismatch"),expected.getValue(), actual.getValue());
  }
}
 

public FieldWithPath(String singleQuoteEscapedPath, String doubleQuoteEscapedPath, Field.Type type, Object value,
    Map<String, String> attributes) {
  this.sqPath = singleQuoteEscapedPath;
  this.dqPath = doubleQuoteEscapedPath;
  this.type = type;
  this.value = value;
  if (attributes == null) {
    this.attributes = null;
  } else {
    this.attributes = new LinkedHashMap<>(attributes);
  }
}
 

public static Field.Type getTypeFromObject(Object result) {
  if(result instanceof Double) {
    return Field.Type.DOUBLE;
  } else if(result instanceof Long) {
    return Field.Type.LONG;
  } else if(result instanceof BigDecimal) {
    return Field.Type.DECIMAL;
  } else if(result instanceof Date) {
    //This can only happen in ${time:now()}
    return Field.Type.DATETIME;
    //For all the timeEL, we currently return String so we are safe.
  } else if(result instanceof Short) {
    return Field.Type.SHORT;
  } else if(result instanceof Boolean) {
    return Field.Type.BOOLEAN;
  } else if(result instanceof Byte) {
    return Field.Type.BYTE;
  } else if(result instanceof byte[]) {
    return Field.Type.BYTE_ARRAY;
  } else if(result instanceof Character) {
    return Field.Type.CHAR;
  } else if(result instanceof Float) {
    return Field.Type.FLOAT;
  } else if(result instanceof Integer) {
    return Field.Type.INTEGER;
  } else if(result instanceof String) {
    return Field.Type.STRING;
  } else if(result instanceof LinkedHashMap) {
    return Field.Type.LIST_MAP;
  } else if(result instanceof Map) {
    return Field.Type.MAP;
  } else if(result instanceof List) {
    return Field.Type.LIST;
  }
  return Field.Type.STRING;
}
 

private StorageType getColumnStorageType(Field.Type fieldType) {
  StorageType storageType;
  switch (fieldType) {
    case BOOLEAN:
    case CHAR:
    case BYTE:
    case SHORT:
    case INTEGER:
    case LONG:
    case FLOAT:
    case DOUBLE:
    case DATE:
    case DATETIME:
    case TIME:
    case DECIMAL:
    case STRING:
      storageType = StorageType.TEXT;
      break;
    case BYTE_ARRAY:
      storageType = StorageType.BINARY;
      break;
    case MAP:
    case LIST:
    case LIST_MAP:
      storageType = StorageType.JSON_STRING;
      break;
    default:
      throw new FieldConversionException(Utils.format("Conversion not defined for: {} ", fieldType));
  }
  return storageType;
}
 

String composeTypeAlert(String fieldPath, Field.Type givenType, Set<Field.Type> supportedTypes) {
  return Utils.format("Field {} have unsupported type of {}. Supported types are {}",
    fieldPath,
    givenType.toString(),
    StringUtils.join(supportedTypes, ",")
  );
}
 

@ElFunction(
    prefix = FIELD_EL_PREFIX,
    name = "type",
    description = "Returns the type of the field in context"
)
public static Field.Type getType() {
  return getFieldInContext().getType();
}
 

FieldMatcher(Field.Type type, Object value) {
  this.type = type;
  this.value = value;
}
 

private static void createRecordsAndExecuteInsertStatements(
    String tableName,
    Statement st,
    Map<String, Field.Type> columnToFieldType,
    boolean multipleRecordsWithSameOffset
) throws Exception {
  int numberOfRecordsInTable = RANDOM.nextInt(MAX_ROWS_PER_TABLE - 1) + 1;
  IntStream.range(0, numberOfRecordsInTable).forEach(recordNum -> {
    Record record = RecordCreator.create();
    LinkedHashMap<String, Field> rootField = new LinkedHashMap<>();

    final int recordNumDivisor = multipleRecordsWithSameOffset ? 100 : 1;
    final int calculatedRecordNum = recordNum / recordNumDivisor;

    rootField.put(OFFSET_FIELD_NAME, Field.create(calculatedRecordNum));


    columnToFieldType.entrySet().stream()
        .filter(columnToFieldTypeEntry -> !columnToFieldTypeEntry.getKey().equals(OFFSET_FIELD_NAME))
        .forEach(columnToFieldTypeEntry  -> {
          String fieldName = columnToFieldTypeEntry.getKey();
          Field.Type fieldType = columnToFieldTypeEntry.getValue();
          rootField.put(fieldName, Field.create(fieldType, generateRandomData(fieldType)));
        });

    if (multipleRecordsWithSameOffset) {
      rootField.put(OFFSET_FIELD_RAW_NAME, Field.create(recordNum));
    }
    record.set(Field.createListMap(rootField));
    List<Record> records = EXPECTED_TABLES_TO_RECORDS.computeIfAbsent(tableName, t -> new ArrayList<>());
    records.add(record);

    try {
      st.addBatch(getInsertStatement(database, tableName, rootField.values()));
    } catch (Exception e) {
      LOG.error("Error Happened", e);
      Throwables.propagate(e);
    }
  });
  st.executeBatch();
}
 

@Before
public void setup() throws Exception {
  context = ContextInfoCreator.createSourceContext("i", false, OnRecordError.TO_ERROR, Collections.EMPTY_LIST);
  record = context.createRecord("123");
  record.set("/", Field.create(new LinkedHashMap<String, Field>()));
  record.set("/byte", Field.create((byte)1));
  record.set("/short", Field.create((short)123));
  record.set("/int", Field.create(123));
  record.set("/long", Field.create(123L));
  record.set("/float", Field.create(123.0f));
  record.set("/double", Field.create(123.0d));
  record.set("/bytes", Field.create("ABC".getBytes(StandardCharsets.UTF_8)));
  record.set("/str", Field.create("ABC"));
  record.set("/bool", Field.create(true));
  DateTime dt = new DateTime(2017, 8, 24, 9, 15, 30, DateTimeZone.UTC); // 2017/8/24 9:15:30
  record.set("/unixtime", Field.create(dt.getMillis() * 1000L));
  Map<String, Field.Type> columnsToFieldTypes = ImmutableMap.<String, Field.Type>builder()
    .put("byte1", Field.Type.BYTE)
    .put("short1", Field.Type.SHORT)
    .put("int1", Field.Type.INTEGER)
    .put("long1", Field.Type.LONG)
    .put("float1", Field.Type.FLOAT)
    .put("double1", Field.Type.DOUBLE)
    .put("bytes", Field.Type.BYTE_ARRAY)
    .put("str", Field.Type.STRING)
    .put("bool1", Field.Type.BOOLEAN)
    .put("unixtime_micro", Field.Type.LONG)
    .build();
  Map<String, String> fieldsToColumns = ImmutableMap.<String, String>builder()
    .put("/byte", "byte1")
    .put("/short", "short1")
    .put("/int", "int1")
    .put("/long", "long1")
    .put("/float", "float1")
    .put("/double", "double1")
    .put("/bytes", "bytes")
    .put("/str", "str")
    .put("/bool", "bool1")
    .put("/unixtime", "unixtime_micro")
    .build();
  Schema schema = new Schema(Arrays.asList(
    new ColumnSchema.ColumnSchemaBuilder("str", Type.STRING).key(true).build(),
    new ColumnSchema.ColumnSchemaBuilder("byte1", Type.INT8).build(),
    new ColumnSchema.ColumnSchemaBuilder("short1", Type.INT16).nullable(true).build(),
    new ColumnSchema.ColumnSchemaBuilder("int1", Type.INT32).build(),
    new ColumnSchema.ColumnSchemaBuilder("long1", Type.INT64).build(),
    new ColumnSchema.ColumnSchemaBuilder("float1", Type.FLOAT).build(),
    new ColumnSchema.ColumnSchemaBuilder("double1", Type.DOUBLE).build(),
    new ColumnSchema.ColumnSchemaBuilder("bytes", Type.BINARY).build(),
    new ColumnSchema.ColumnSchemaBuilder("bool1", Type.BOOL).build(),
    new ColumnSchema.ColumnSchemaBuilder("unixtime_micro", Type.UNIXTIME_MICROS).build()
    ));
  partialRow = new PartialRow(schema);
  kuduRecordConverter = new KuduRecordConverter(columnsToFieldTypes, fieldsToColumns, schema, null);
}
 

private static List<Object> getDifferentDateTimes(Field.Type type) {
  List<Object> dateTimes = new ArrayList<>();
  Calendar calendar = Calendar.getInstance();
  switch (type) {
    case DATE:
      calendar.set(Calendar.HOUR_OF_DAY, 0);
      calendar.set(Calendar.MINUTE, 0);
      calendar.set(Calendar.SECOND, 0);
      calendar.set(Calendar.MILLISECOND, 0);
      calendar.add(Calendar.DATE,  -(BATCHES + 1));
      for (int i = 0 ; i < 5; i++) {
        //Increment 1 DAY for each value
        calendar.add(Calendar.DATE, 1);
        dateTimes.add(calendar.getTime());
      }
      break;
    case TIME:
      calendar.set(Calendar.YEAR, 1970);
      calendar.set(Calendar.MONTH, 0);
      calendar.set(Calendar.DAY_OF_MONTH, 1);
      calendar.add(Calendar.MILLISECOND, -(BATCHES + 1));
      for (int i = 0 ; i < 5; i++) {
        //Increment 1 millisecond for each value
        calendar.add(Calendar.MILLISECOND, 1);
        dateTimes.add(calendar.getTime());
      }
      break;
    case DATETIME:
    case LONG:
      calendar.add(Calendar.MILLISECOND, -(BATCHES + 1));
      for (int i = 0 ; i < 5; i++) {
        //Increment 1 millisecond for each value
        calendar.add(Calendar.MILLISECOND, 1);
        Date currentDateTime = calendar.getTime();
        dateTimes.add(type == Field.Type.LONG ? currentDateTime.getTime() : currentDateTime);
      }
      break;
    default:
      throw new IllegalArgumentException("Unsupported Field Type {}" + type);
  }
  return dateTimes;
}
 

@Override
public Set<Field.Type> supportedTypes() {
  return Sets.newHashSet(Field.Type.LIST, Field.Type.MAP, Field.Type.LIST_MAP);
}
 

public static Field numericAggregationHelper(
    @ElParam("input") List<Field> inputs,
    NumericResultUpdater updater,
    long initialLongValue,
    double initialDoubleValue,
    BigDecimal initialDecimalValue
) {
  if (inputs == null || inputs.isEmpty()) {
    // TODO: something better here?
    return Field.create(0l);
  }
  Field.Type type = null;
  long integralResult = initialLongValue;
  double floatingPointResult = initialDoubleValue;
  BigDecimal decimalResult = initialDecimalValue;
  for (Field input : inputs) {
    if (type == null) {
      type = input.getType();
    } else if (type != input.getType()) {
      throw new IllegalStateException(String.format(
          "Mixed types were detected in input list; encountered %s with value %s, previously saw %s",
          input.getType().name(),
          input.getValue(),
          type.name()
      ));
    }
    switch (type) {
      case INTEGER:
        integralResult = updater.updateLongResult(integralResult, input.getValueAsInteger());
        break;
      case LONG:
        integralResult = updater.updateLongResult(integralResult, input.getValueAsLong());
        break;
      case FLOAT:
        floatingPointResult = updater.updateDoubleResult(floatingPointResult, input.getValueAsFloat());
        break;
      case DOUBLE:
        floatingPointResult = updater.updateDoubleResult(floatingPointResult, input.getValueAsDouble());
        break;
      case DECIMAL:
        decimalResult = updater.updateDecimalResult(decimalResult, input.getValueAsDecimal());
        break;
      default:
        throw new IllegalStateException(String.format(
            "Unsupported type detected in input list; encountered %s with value %s, which is not summable",
            input.getType().name(),
            input.getValue()
        ));
    }
  }

  switch (type) {
    case INTEGER:
    case LONG:
      return Field.create(integralResult);
    case FLOAT:
    case DOUBLE:
      return Field.create(floatingPointResult);
    case DECIMAL:
      return Field.create(decimalResult);
    default:
      throw new IllegalStateException(String.format(
          "Should not reach here (unsupported type for return: %s",
          type.name()
      ));
  }
}
 

XmlType(String xmlType, Field.Type fieldType) {
  this.xmlType = xmlType;
  this.fieldType = fieldType;
}
 

private static Field.Type getFieldType(Schema schema) {
  String logicalType = schema.getProp(LOGICAL_TYPE);
  if(logicalType != null && !logicalType.isEmpty()) {
    switch (logicalType) {
      case LOGICAL_TYPE_DECIMAL:
        return Field.Type.DECIMAL;
      case LOGICAL_TYPE_DATE:
        return Field.Type.DATE;
      case LOGICAL_TYPE_TIME_MILLIS:
        return Field.Type.TIME;
      case LOGICAL_TYPE_TIME_MICROS:
        return Field.Type.LONG;
      case LOGICAL_TYPE_TIMESTAMP_MILLIS:
        return Field.Type.DATETIME;
      case LOGICAL_TYPE_TIMESTAMP_MICROS:
        return Field.Type.LONG;
    }
  }

  switch(schema.getType()) {
    case ARRAY:
      return Field.Type.LIST;
    case BOOLEAN:
      return Field.Type.BOOLEAN;
    case BYTES:
      return Field.Type.BYTE_ARRAY;
    case DOUBLE:
      return Field.Type.DOUBLE;
    case ENUM:
      return Field.Type.STRING;
    case FIXED:
      return Field.Type.BYTE_ARRAY;
    case FLOAT:
      return Field.Type.FLOAT;
    case INT:
      return Field.Type.INTEGER;
    case LONG:
      return Field.Type.LONG;
    case MAP:
      return Field.Type.MAP;
    case NULL:
      return Field.Type.MAP;
    case RECORD:
      return Field.Type.MAP;
    case STRING:
      return Field.Type.STRING;
    default:
      throw new IllegalStateException(Utils.format("Unexpected schema type {}", schema.getType().getName()));
  }
}
 

/**
 * Return set of supported types for this drift detector.
 * Null denotes "I'm supporting all types, please don't do any checks".
 */
public abstract Set<Field.Type> supportedTypes();