Java Code Examples for com.streamsets.pipeline.api.Field#getType()

/**
 * Generate {@link HiveTypeInfo} from the Metadata Record <br>.
 * (Reverse of {@link #generateHiveTypeInfoFieldForMetadataRecord(HiveTypeInfo)})
 *
 * @throws StageException if the metadata field is not valid.
 * @param hiveTypeInfoField hiveTypeInfoField
 * @return {@link HiveTypeInfo}
 * @throws StageException If the record has invalid
 */
@SuppressWarnings("unchecked")
public HiveTypeInfo generateHiveTypeInfoFromMetadataField(Field hiveTypeInfoField) throws StageException {
  if (hiveTypeInfoField.getType() == Field.Type.MAP) {
    Map<String, Field> fields = (Map<String, Field>) hiveTypeInfoField.getValue();
    if (!fields.containsKey(HiveMetastoreUtil.TYPE)
        || !fields.containsKey(HiveMetastoreUtil.EXTRA_INFO)) {
      throw new StageException(Errors.HIVE_17, HiveMetastoreUtil.TYPE_INFO);
    }
    HiveType hiveType = HiveType.getHiveTypeFromString(fields.get(HiveMetastoreUtil.TYPE).getValueAsString());
    String comment = "";
    if(fields.containsKey(HiveMetastoreUtil.COMMENT)) {
      comment = fields.get(HiveMetastoreUtil.COMMENT).getValueAsString();
    }
    return generateHiveTypeInfoFromMetadataField(hiveType, comment, fields.get(HiveMetastoreUtil.EXTRA_INFO));
  } else {
    throw new StageException(Errors.HIVE_17, HiveMetastoreUtil.TYPE_INFO);
  }
}
 

private static int getDelimitedIndex(String header, int startIndex) {
  int index = -1;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (int i= startIndex; index == -1 && i < list.size(); i++) {
        Field element = list.get(i);
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                index = i;
              }
            }
          }
        }
      }
    }
  }
  return index;
}
 

private void doUpsertString(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.STRING) {
    String val = value.getValueAsString();
    pipeline.set(key, val);
    tempRecords.add(new ErrorRecord(record, "String", key, val));
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), " value should be String");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be String"
        )
    );
  }
}
 

private Map<String, Field> flattenEntireRecord(Field rootField) {
  Map<String, Field> ret = new LinkedHashMap<>();
  switch (rootField.getType()) {
    case MAP:
    case LIST_MAP:
      flattenMap("", rootField.getValueAsMap(), ret);
      break;
    case LIST:
      flattenList("", rootField.getValueAsList(), ret);
      break;
    default:
      break;
  }

  return ret;
}
 

private void doUpsertSet(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.sadd(key, val);
        tempRecords.add(new ErrorRecord(record, "Set", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "type",
    description = "Returns the type of the field represented by path 'fieldPath' for the record in context")
public static Field.Type getType(
    @ElParam("fieldPath") String fieldPath) {
  Field.Type type = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field field = record.get(fieldPath);
    if (field != null) {
      type = field.getType();
    }
  }
  return type;
}
 

private static Date getDateForTimeZone(TimeZone timeZone, Field field) throws HiveStageCheckedException {
  Date fieldValue;
  switch (field.getType()) {
    case DATETIME:
      fieldValue = field.getValueAsDate();
      break;
    default:
      throw new HiveStageCheckedException(Errors.HIVE_41, field.getType().name());
  }
  Calendar calendar = Calendar.getInstance();
  calendar.setTime(fieldValue);
  calendar.setTimeZone(timeZone);
  return calendar.getTime();
}
 

private void setFieldsInRecord(Record record, Map<String, Field> fields) {
  for (Map.Entry<String, Field> entry : fields.entrySet()) {
    String columnName = entry.getKey();
    String fieldPath = columnToField.get(columnName);
    Field field = entry.getValue();
    if (fieldPath == null) {
      Field root = record.get();
      // No mapping
      switch (root.getType()) {
        case LIST:
          // Add new field to the end of the list
          fieldPath = "[" + root.getValueAsList().size() + "]";
          Map<String, Field> cell = new HashMap<>();
          cell.put("header", Field.create(columnName));
          cell.put("value", field);
          field = Field.create(cell);
          break;
        case LIST_MAP:
        case MAP:
          // Just use the column name
          fieldPath = columnName;
          break;
        default:
          break;
      }
    }
    record.set(fieldPath, field);
  }
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValue",
    description = "Returns the value of the specified header name")
public static String getDelimitedValue(
    @ElParam("header") String header) {
  String value = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (Field element : list) {
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                if (map.containsKey("value")) {
                  Field valueField = map.get("value");
                  if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                    value = valueField.getValueAsString();
                    break;
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 

public static Map<String, String> getTags(List<String> tagFields, Record record) throws OnRecordErrorException {
  Map<String, String> tags = new HashMap<>();

  for (String fieldPath : tagFields) {
    if (!record.has(fieldPath)) {
      continue;
    }

    Field tagField = record.get(fieldPath);
    switch (tagField.getType()) {
      case MAP:
        // fall through
      case LIST_MAP:
        for (Map.Entry<String, Field> entry : tagField.getValueAsMap().entrySet()) {
          tags.put(entry.getKey(), entry.getValue().getValueAsString());
        }
        break;
      case LIST:
        throw new OnRecordErrorException(Errors.INFLUX_08, fieldPath);
      default:
        tags.put(CollectdRecordConverter.stripPathPrefix(fieldPath), tagField.getValueAsString());
        break;
    }
  }

  return tags;
}
 

private long getTime(Record record) throws OnRecordErrorException {
  Field timeField = record.get(conf.timeField);
  if (timeField.getType() == Field.Type.DATE ||
      timeField.getType() == Field.Type.DATETIME ||
      timeField.getType() == Field.Type.TIME)
  {
    return timeField.getValueAsDatetime().getTime();
  } else if (timeField.getType() == Field.Type.LONG) {
    return timeField.getValueAsLong();
  }

  throw new OnRecordErrorException(Errors.INFLUX_09, timeField.getType());
}
 

private Field processByType(String matchingPath, Field rootField) throws StageException {
  switch (rootField.getType()) {
    case MAP:
    case LIST_MAP:
      if(rootField.getValue() == null) {
        return rootField;
      }
      for (Map.Entry<String, Field> entry : rootField.getValueAsMap().entrySet()) {
        entry.setValue(processByType(matchingPath + "/" + entry.getKey(), entry.getValue()));
      }
      break;
    case LIST:
      if(rootField.getValue() == null) {
        return rootField;
      }
      List<Field> fields = rootField.getValueAsList();
      for(int i = 0; i < fields.size(); i++) {
        fields.set(i, processByType(matchingPath + "[" + i + "]", fields.get(i)));
      }
      break;
    default:
      for(WholeTypeConverterConfig converterConfig : wholeTypeConverterConfigs) {
        if(converterConfig.sourceType == rootField.getType()) {
          rootField = convertField(matchingPath, rootField, converterConfig);
        }
      }
  }

  // Return original field
  return rootField;
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  if (!record.has(listPath)) {
    switch (onStagePreConditionFailure) {
      case TO_ERROR:
        throw new OnRecordErrorException(Errors.LIST_PIVOT_01, record.getHeader().getSourceId(), listPath);
      case CONTINUE:
        return;
      default:
        throw new IllegalStateException("Invalid value for on stage pre-condition failure");
    }
  }

  Field listField = record.get(listPath);
  if (!listField.getType().isOneOf(Field.Type.LIST, Field.Type.LIST_MAP, Field.Type.MAP)) {
    throw new OnRecordErrorException(Errors.LIST_PIVOT_00, listPath);
  }

  List<PivotEntry> entries = Lists.newArrayList();
  if (listField.getType() == Field.Type.LIST) {
    List<Field> fieldList = listField.getValueAsList();
    for (Field field : fieldList) {
      entries.add(new PivotEntry(listPath, field));
    }
  } else {
    Set<Map.Entry<String, Field>> fieldEntries = listField.getValueAsMap().entrySet();
    for (Map.Entry<String, Field> entry : fieldEntries) {
      entries.add(new PivotEntry(entry.getKey(), entry.getValue()));
    }
  }

  pivot(entries, record, batchMaker);
}
 

private void writeFieldToJsonObject(Field field) throws IOException {
  if (field == null || field.getValue() == null) {
    generator.writeNull();
    return;
  }
  switch (field.getType()) {
    case FILE_REF:
      throw new IOException("Cannot serialize FileRef fields.");
    case MAP:
    case LIST_MAP:
      generator.writeStartObject();
      Map<String, Field> map = field.getValueAsMap();
      for (Map.Entry<String, Field> fieldEntry : map.entrySet()) {
        generator.writeFieldName(fieldEntry.getKey());
        writeFieldToJsonObject(fieldEntry.getValue());
      }
      generator.writeEndObject();
      break;
    case LIST:
      generator.writeStartArray();
      List<Field> list = field.getValueAsList();
      for (Field f : list) {
        writeFieldToJsonObject(f);
      }
      generator.writeEndArray();
      break;
    case BOOLEAN:
      generator.writeBoolean(field.getValueAsBoolean());
      break;
    case CHAR:
      generator.writeString(String.valueOf(field.getValueAsChar()));
      break;
    case BYTE:
      generator.writeBinary(new byte[] {field.getValueAsByte()});
      break;
    case SHORT:
      generator.writeNumber(field.getValueAsShort());
      break;
    case INTEGER:
      generator.writeNumber(field.getValueAsInteger());
      break;
    case LONG:
      generator.writeNumber(field.getValueAsLong());
      break;
    case FLOAT:
      generator.writeNumber(field.getValueAsFloat());
      break;
    case DOUBLE:
      generator.writeNumber(field.getValueAsDouble());
      break;
    case DATE:
      generator.writeNumber(field.getValueAsDate().getTime());
      break;
    case DATETIME:
      generator.writeNumber(field.getValueAsDatetime().getTime());
      break;
    case TIME:
      generator.writeNumber(field.getValueAsTime().getTime());
      break;
    case DECIMAL:
      generator.writeNumber(field.getValueAsDecimal());
      break;
    case STRING:
      generator.writeString(field.getValueAsString());
      break;
    case BYTE_ARRAY:
      generator.writeBinary(field.getValueAsByteArray());
      break;
    case ZONED_DATETIME:
      generator.writeString(field.getValueAsString());
      break;
    default:
      throw new IllegalStateException(String.format(
          "Unrecognized field type (%s) in field: %s",
          field.getType().name(),
          field.toString())
      );
  }
}
 

/**
 * Convert the sdc Field to an object for row content
 */
private Object getValueFromField(String fieldPath, Field field) {
  LOG.trace("Visiting Field Path '{}' of type '{}'", fieldPath, field.getType());
  switch (field.getType()) {
    case LIST:
      //REPEATED
      List<Field> listField = field.getValueAsList();
      //Convert the list to map with indices as key and Field as value (Map<Integer, Field>)
      Map<Integer, Field> fields =
          IntStream.range(0, listField.size()).boxed()
              .collect(Collectors.toMap(Function.identity(), listField::get));
      //filter map to remove fields with null value
      fields = fields.entrySet().stream()
          .filter(e -> e.getValue().getValue() != null)
          .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
      //now use the map index to generate field path and generate object for big query write
      return fields.entrySet().stream()
          .map(e -> getValueFromField(fieldPath + "[" + e.getKey() + "]", e.getValue()))
          .collect(Collectors.toList());
    case MAP:
    case LIST_MAP:
      //RECORD
      return field.getValueAsMap().entrySet().stream()
          .filter(me -> me.getValue().getValue() != null)
          .collect(
              Collectors.toMap(
                  Map.Entry::getKey,
                  e -> getValueFromField(fieldPath + "/" + e.getKey(), e.getValue())
              )
          );
    case DATE:
      return dateFormat.format(field.getValueAsDate());
    case TIME:
      return timeFormat.format(field.getValueAsTime());
    case DATETIME:
      return dateTimeFormat.format(field.getValueAsDatetime());
    case BYTE_ARRAY:
      return Base64.getEncoder().encodeToString(field.getValueAsByteArray());
    case DECIMAL:
    case BYTE:
    case CHAR:
    case FILE_REF:
      throw new IllegalArgumentException(Utils.format(Errors.BIGQUERY_12.getMessage(), fieldPath, field.getType()));
    default:
      //Boolean -> Map to Boolean in big query
      //Float, Double -> Map to Float in big query
      //String -> maps to String in big query
      //Short, Integer, Long -> Map to integer in big query
      return field.getValue();
  }
}
 

protected boolean isMap(Field field) {
  return field != null && (field.getType() == Field.Type.LIST_MAP || field.getType() == Field.Type.MAP);
}
 

private int compareIt(Field field, String stringValue, String matchingField) {
  try {
    switch (field.getType()) {
      case BYTE:
        if (field.getValueAsByte() == (Byte) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsByte() < (Byte) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case SHORT:
        if (field.getValueAsShort() == (Short) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsShort() < (Short) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case INTEGER:
        if (field.getValueAsInteger() == (Integer) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsInteger() < (Integer) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case LONG:
        if (field.getValueAsLong() == (Long) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsLong() < (Long) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case FLOAT:
        if (field.getValueAsFloat() == (Float) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsFloat() < (Float) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case DOUBLE:
        if (field.getValueAsDouble() == (Double) convertToType(stringValue, field.getType(), matchingField)) {
          return 0;
        } else if (field.getValueAsDouble() < (Double) convertToType(stringValue, field.getType(), matchingField)) {
          return -1;
        } else {
          return 1;
        }

      case STRING:
        return field.getValueAsString().compareTo(stringValue);

      default:
        throw new IllegalArgumentException(Utils.format(Errors.VALUE_REPLACER_03.getMessage(), field.getType(), matchingField));
    }
  } catch (Exception e) {
    if (e instanceof IllegalArgumentException) {
      throw (IllegalArgumentException)e;
    } else {
      throw new IllegalArgumentException(Utils.format(Errors.VALUE_REPLACER_03.getMessage(), field.getType()), e);
    }
  }
}
 

/**
 * Convert a Record to LinkedHashMap. This is for comparing the structure of incoming Record with cache.
 * Since Avro does not support char, short, and date types, it needs to convert the type to corresponding
 * supported types and change the value in record.
 * @param record incoming Record
 * @return LinkedHashMap version of record. Key is the column name, and value is column type in HiveType
 * @throws HiveStageCheckedException
 * @throws ELEvalException
 */
public static LinkedHashMap<String, HiveTypeInfo> convertRecordToHMSType(
    Record record,
    ELEval scaleEL,
    ELEval precisionEL,
    ELEval commentEL,
    String scaleExpression,
    String precisionExpression,
    String commentExpression,
    ELVars variables,
    boolean convertTimesToString,
    TimeZone timeZone
) throws HiveStageCheckedException, ELEvalException {
  if(!record.get().getType().isOneOf(Field.Type.MAP, Field.Type.LIST_MAP)) {
    throw new HiveStageCheckedException(Errors.HIVE_33, record.getHeader().getSourceId(), record.get().getType().toString());
  }

  LinkedHashMap<String, HiveTypeInfo> columns = new LinkedHashMap<>();
  Map<String, Field> list = record.get().getValueAsMap();
  for(Map.Entry<String,Field> pair:  list.entrySet()) {
    if (StringUtils.isEmpty(pair.getKey())) {
      throw new HiveStageCheckedException(Errors.HIVE_01, "Field name is empty");
    }
    Field currField = pair.getValue();
    switch(currField.getType()) {
      case SHORT:
        currField = Field.create(Field.Type.INTEGER, currField.getValue());
        break;
      case CHAR:
        currField = Field.create(currField.getValueAsString());
        break;
      case DATETIME:
        if (convertTimesToString) {
          currField = Field.create(
              Field.Type.STRING,
              currField.getValue() == null ? null : datetimeFormat.get().format(currField.getValueAsDate())
          );
        } else {
          currField = Field.create(
              Field.Type.DATETIME,
              currField.getValue() == null ? null : getDateForTimeZone(timeZone, currField));
        }
        break;
      case TIME:
        currField = Field.create(
            Field.Type.STRING,
            currField.getValue() == null ? null : timeFormat.get().format(currField.getValueAsTime())
        );
        break;
      default:
        break;
    }

    // Set current field in the context - used by subsequent ELs (decimal resolution, comments, ...)
    FieldPathEL.setFieldInContext(variables, pair.getKey());

    String comment = commentEL.eval(variables, commentExpression, String.class);
    if(!COMMENT_PATTERN.matcher(comment).matches()) {
      throw new HiveStageCheckedException(com.streamsets.pipeline.stage.processor.hive.Errors.HIVE_METADATA_11, pair.getKey(), comment);
    }

    // Update the Field type and value in Record
    pair.setValue(currField);
    HiveType hiveType = HiveType.getHiveTypeforFieldType(currField.getType());
    HiveTypeInfo hiveTypeInfo;
    // Some types requires special checks or alterations
    if (hiveType == HiveType.DECIMAL) {
      int precision = resolveScaleOrPrecisionExpression("precision", precisionEL, variables, precisionExpression, pair.getKey());
      int scale = resolveScaleOrPrecisionExpression("scale", scaleEL, variables, scaleExpression, pair.getKey());
      validateScaleAndPrecision(pair.getKey(), currField, precision, scale);
      hiveTypeInfo = hiveType.getSupport().generateHiveTypeInfoFromRecordField(currField, comment, precision, scale);
      // We need to make sure that all java objects have the same scale
      if(currField.getValue() != null) {
        pair.setValue(Field.create(currField.getValueAsDecimal().setScale(scale)));
      }
    } else {
      hiveTypeInfo = hiveType.getSupport().generateHiveTypeInfoFromRecordField(currField, comment);
    }
    columns.put(pair.getKey().toLowerCase(), hiveTypeInfo);
  }
  return columns;
}
 

private byte[] convertToBinary(Field field, Record record) throws OnRecordErrorException {
  byte[] value;
  switch (field.getType()) {
    case BOOLEAN:
      value = Bytes.toBytes(field.getValueAsBoolean());
      break;
    case BYTE:
      value = Bytes.toBytes(field.getValueAsByte());
      break;
    case BYTE_ARRAY:
      value = field.getValueAsByteArray();
      break;
    case CHAR:
      value = Bytes.toBytes(field.getValueAsChar());
      break;
    case DATE:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.DATE.name(),
          BigtableStorageType.BINARY.name()
      );
    case TIME:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.TIME.name(),
          BigtableStorageType.BINARY.name()
      );
    case DATETIME:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.DATETIME.name(),
          BigtableStorageType.BINARY.name()
      );
    case DECIMAL:
      value = Bytes.toBytes(field.getValueAsDecimal());
      break;
    case DOUBLE:
      value = Bytes.toBytes(field.getValueAsDouble());
      break;
    case FLOAT:
      value = Bytes.toBytes(field.getValueAsFloat());
      break;
    case INTEGER:
      value = Bytes.toBytes(field.getValueAsInteger());
      break;
    case LIST:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.LIST.name(),
          BigtableStorageType.BINARY.name()
      );
    case LIST_MAP:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.LIST_MAP.name(),
          BigtableStorageType.BINARY.name()
      );
    case LONG:
      value = Bytes.toBytes(field.getValueAsLong());
      break;
    case MAP:
      throw new OnRecordErrorException(Errors.BIGTABLE_12,
          Field.Type.MAP.name(),
          BigtableStorageType.BINARY.name(),
          record
      );
    case SHORT:
      value = Bytes.toBytes(field.getValueAsShort());
      break;
    case STRING:
      throw new OnRecordErrorException(record,
          Errors.BIGTABLE_12,
          Field.Type.STRING.name(),
          BigtableStorageType.BINARY.name()
      );
    default:
      throw new OnRecordErrorException(record, Errors.BIGTABLE_13, field.toString());
  }
  return value;
}
 

/**
 * Checks that the decryption input is a valid type, otherwise sends the
 * record to StageException.
 *
 * @param field {@link Field}
 * @return empty {@link Optional} if the record was sent to error
 */
public Optional<Field> checkInputDecrypt(Field field) throws StageException {
  if (field.getType() != Field.Type.BYTE_ARRAY) {
    throw new StageException(CRYPTO_02, field.getType());
  }
  return Optional.of(field);
}