Java Code Examples for com.streamsets.pipeline.api.Field#getValueAsString()

private void doUpsertList(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException{
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.lpush(key, val);
        tempRecords.add(new ErrorRecord(record, "List", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 

private void doUpsertSet(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.LIST) {
    List<Field> values = value.getValueAsList();
    for (Field element : values) {
      if (element != null) {
        String val = element.getValueAsString();
        pipeline.sadd(key, val);
        tempRecords.add(new ErrorRecord(record, "Set", key, val));
      }
    }
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), "value should be List");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be List"
        )
    );
  }
}
 

private void doUpsertString(Record record, List<ErrorRecord> tempRecords, Pipeline pipeline, String key, Field value)
    throws StageException {
  if (value != null && value.getType() == Field.Type.STRING) {
    String val = value.getValueAsString();
    pipeline.set(key, val);
    tempRecords.add(new ErrorRecord(record, "String", key, val));
  } else {
    LOG.error(Errors.REDIS_04.getMessage(), value.getType(), " value should be String");
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            Errors.REDIS_04,
            value.getType(),
            "value should be String"
        )
    );
  }
}
 

@VisibleForTesting
String regExMask(Field field, FieldMaskConfig fieldMaskConfig) {
  String value = field.getValueAsString();
  Matcher matcher = regExToPatternMap.get(fieldMaskConfig.regex).matcher(value);
  if(matcher.matches()) {
    int groupCount = matcher.groupCount();
    //create a masked string of the same length as the original string
    StringBuilder resultString = new StringBuilder();
    for(int i = 0; i < value.length(); i++) {
      resultString.append(MASK_CHAR);
    }
    //for each group that needs to be shown, replace the masked string with the original string characters at those
    //positions
    Set<Integer> groupsToShow = regexToGroupsToShowMap.get(fieldMaskConfig.regex);
    if(groupsToShow != null && !groupsToShow.isEmpty()) {
      for (int i = 1; i <= groupCount; i++) {
        if (groupsToShow.contains(i)) {
          resultString.replace(matcher.start(i), matcher.end(i), matcher.group(i));
        }
      }
    }
    return resultString.toString();
  }
  return field.getValueAsString();
}
 

@VisibleForTesting
InetAddress toAddress(Field field) throws UnknownHostException, OnRecordErrorException {
  switch (field.getType()) {
    case LONG:
    case INTEGER:
      return InetAddresses.fromInteger(field.getValueAsInteger());
    case STRING:
      String ip = field.getValueAsString();
      if (ip == null) {
        throw new OnRecordErrorException(Errors.GEOIP_13);
      }

      ip = ip.trim();
      if (!ip.contains(".") && !ip.contains(":")) {
        return InetAddresses.fromInteger(Integer.parseInt(ip));
      }

      if (!InetAddresses.isInetAddress(ip)) {
        throw new OnRecordErrorException(Errors.GEOIP_06, ip);
      }
      return InetAddresses.forString(ip);
    default:
      throw new IllegalStateException(Utils.format("Unknown field type: ", field.getType()));
  }
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Field field = record.get(fieldPathToParse);
  if (field == null) {
    throw new OnRecordErrorException(Errors.JSONP_00, record.getHeader().getSourceId(), fieldPathToParse);
  } else {
    String value = field.getValueAsString();
    if (value == null) {
      throw new OnRecordErrorException(Errors.JSONP_01, record.getHeader().getSourceId(), fieldPathToParse);
    }
    try (OverrunReader reader = new OverrunReader(new StringReader(value), -1, false, removeCtrlChars)) {
      JsonCharDataParser parser = new JsonCharDataParser(getContext(), "", reader, 0, Mode.MULTIPLE_OBJECTS, -1);
      Field parsed = parser.parseAsField();
      if (parsed != null) {
        record.set(parsedFieldPath, parsed);
      }
    } catch (IOException ex) {
      throw new OnRecordErrorException(Errors.JSONP_03, record.getHeader().getSourceId(), fieldPathToParse,
                                       ex.toString(), ex);
    }
    batchMaker.addRecord(record);
  }
}
 

public static Matcher<Field> fieldWithValue(final String value) {
  return new FieldMatcher(Field.Type.STRING, value) {
    @Override
    protected Object getValueFromField(Field field) {
      return field.getValueAsString();
    }
  };
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Field field = record.get(fieldPathToParse);
  if (field == null) {
    throw new OnRecordErrorException(Errors.LOGP_00, fieldPathToParse, record.getHeader().getSourceId());
  } else {
    String value = field.getValueAsString();
    if (value == null) {
      throw new OnRecordErrorException(Errors.LOGP_01, fieldPathToParse, record.getHeader().getSourceId());
    }
    try (DataParser parser = getContext().getService(LogParserService.class).getLogParser(
        record.getHeader()
              .getSourceId(),
        value
    )) {
      Record r = parser.parse();
      if(r != null) {
        Field parsed = r.get();
        record.set(parsedFieldPath, parsed);
      }
    } catch (IOException | DataParserException ex) {
      throw new OnRecordErrorException(
          Errors.LOGP_03,
          fieldPathToParse,
          record.getHeader().getSourceId(),
          ex.toString(),
          ex
      );
    }
    if (!record.has(parsedFieldPath)) {
      throw new OnRecordErrorException(Errors.LOGP_02, parsedFieldPath, record.getHeader().getSourceId());
    }
    batchMaker.addRecord(record);
  }
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValueAt",
    description = "Returns the value of the specified header name")
public static String getDelimitedValueAt(
    @ElParam("index") int index) {
  String value = null;
  if (index >= 0) {
    Record record = getRecordInContext();
    if (record != null) {
      Field root = record.get();
      if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
        List<Field> list = root.getValueAsList();
        if (index < list.size()) {
          Field element = list.get(index);
          if (element.getType() == Field.Type.MAP && element.getValue() != null) {
            Map<String, Field> map = element.getValueAsMap();
            if (map.containsKey("value")) {
              Field valueField = map.get("value");
              if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                value = valueField.getValueAsString();
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 

@ElFunction(
    prefix = RECORD_EL_PREFIX,
    name = "dValue",
    description = "Returns the value of the specified header name")
public static String getDelimitedValue(
    @ElParam("header") String header) {
  String value = null;
  Record record = getRecordInContext();
  if (record != null) {
    Field root = record.get();
    if (root != null && root.getType() == Field.Type.LIST && root.getValue() != null) {
      List<Field> list = root.getValueAsList();
      for (Field element : list) {
        if (element.getType() == Field.Type.MAP && element.getValue() != null) {
          Map<String, Field> map = element.getValueAsMap();
          if (map.containsKey("header")) {
            Field headerField = map.get("header");
            if (headerField.getType() == Field.Type.STRING && headerField.getValue() != null) {
              if (headerField.getValueAsString().equals(header)) {
                if (map.containsKey("value")) {
                  Field valueField = map.get("value");
                  if (valueField.getType() == Field.Type.STRING && valueField.getValue() != null) {
                    value = valueField.getValueAsString();
                    break;
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return value;
}
 

public static Map<String, String> getOffsetsFromColumns(TableRuntimeContext tableContext, Map<String, Field> fields) throws StageException {
  final Map<String, String> offsets = new HashMap<>();
  for (String offsetColumn : tableContext.getSourceTableContext().getOffsetColumns()) {
    Field field = fields.get(offsetColumn);
    String value;

    if(field.getValue() == null) {
      throw new StageException(JdbcErrors.JDBC_408, offsetColumn);
    }

    if (field.getType().isOneOf(Field.Type.DATE, Field.Type.TIME)) {
      //For DATE/TIME fields store the long in string format and convert back to date when using offset
      //in query
      value = String.valueOf(field.getValueAsDatetime().getTime());
    } else if (field.getType() == Field.Type.DATETIME) {
      //DATETIME is similar to above, but there may also be a nanosecond portion (stored in field attr)
      String nanosAttr = field.getAttribute(JdbcUtil.FIELD_ATTRIBUTE_NANOSECONDS);
      int nanos;
      if (StringUtils.isNotBlank(nanosAttr) && StringUtils.isNumeric(nanosAttr)) {
        nanos = Integer.parseInt(nanosAttr);
      } else {
        nanos = 0;
      }
      value = TableContextUtil.getOffsetValueForTimestampParts(field.getValueAsDatetime().getTime(), nanos);
    } else if(field.getType() == Field.Type.ZONED_DATETIME) {
      value = field.getValueAsZonedDateTime().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
    } else {
      value = field.getValueAsString();
    }
    offsets.put(offsetColumn, value);
  }
  return offsets;
}
 

private void writeLine(Record record, String key) throws IOException, DataGeneratorException{
  Field field = record.get();

  if (field.getType() != Field.Type.LIST && field.getType() != Field.Type.LIST_MAP) {
    throw new DataGeneratorException(Errors.DELIMITED_GENERATOR_00, record.getHeader().getSourceId(), field.getType());
  }

  List<Field> columns = field.getValueAsList();
  List<String> values = new ArrayList<>(columns.size());
  boolean isListMap = field.getType() == Field.Type.LIST_MAP;
  for (int i = 0; i< columns.size(); i++) {
    Field column = columns.get(i);
    String value;
    if (!isListMap) {
      try {
        value = column.getValueAsMap().get(key).getValueAsString();
      } catch (Exception ex) {
        throw new DataGeneratorException(Errors.DELIMITED_GENERATOR_01, record.getHeader().getSourceId(), i,
            column.getType());
      }
    } else {
      value = column.getValueAsString();
    }
    if (value != null) {
      if (replaceNewLines != null) {
        if (value.contains("\n")) {
          value = value.replace("\n", replaceNewLines);
        }
        if (value.contains("\r")) {
          value = value.replace("\r", replaceNewLines);
        }
      }
    } else {
      value = "";
    }
    values.add(value);
  }
  printer.printRecord(values);
}
 

@Override
public void write(Record record) throws IOException, DataGeneratorException {
  if (closed) {
    throw new IOException("Generator has been closed");
  }
  Field field = record.get(fieldPath);
  boolean fieldWritten = false;
  if (field != null && field.getValue() != null) {
    String value;
    try {
      value = field.getValueAsString();
    } catch (Exception ex) {
      throw new DataGeneratorException(Errors.TEXT_GENERATOR_00, record.getHeader().getSourceId(), fieldPath);
    }
    writer.write(value);
    fieldWritten = true;
  } else {
    switch(missingAction) {
      case IGNORE:
        // Do nothing
        break;
      case ERROR:
        throw new DataGeneratorException(Errors.TEXT_GENERATOR_01, record.getHeader().getSourceId(), fieldPath);
      default:
        throw new IllegalStateException("Unknown missing action: " + missingAction);
    }
  }

  if ((fieldWritten || recordSeparatorIfNull) && recordSeparator != null) {
    writer.write(recordSeparator);
  }
}
 

public String getTimestamp() {
  Field entry = getEntryFromField("timestamp");
  return entry.getValueAsString();
}
 

public String getNextLSN() {
  Field entry  = getEntryFromField("nextlsn");
  return entry.getValueAsString();
}
 

public String getXid() {
  Field entry = getEntryFromField("xid");
  return entry.getValueAsString();
}
 

private Field convertStringToTargetType(
    Field field,
    Field.Type targetType,
    Locale dataLocale,
    String dateMask,
    int scale,
    DecimalScaleRoundingStrategy decimalScaleRoundingStrategy,
    DateTimeFormatter dateTimeFormatter
) throws ParseException {
  String stringValue = field.getValueAsString();
  switch(targetType) {
    case BOOLEAN:
      return Field.create(Field.Type.BOOLEAN, Boolean.valueOf(stringValue.trim()));
    case BYTE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue).byteValue());
    case BYTE_ARRAY:
      return Field.create(stringValue.getBytes(StandardCharsets.UTF_8));
    case CHAR:
      return Field.create(stringValue.charAt(0));
    case DATE:
      java.text.DateFormat dateFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDate(dateFormat.parse(stringValue.trim()));
    case DATETIME:
      java.text.DateFormat dateTimeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDatetime(dateTimeFormat.parse(stringValue.trim()));
    case TIME:
      java.text.DateFormat timeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createTime(timeFormat.parse(stringValue.trim()));
    case ZONED_DATETIME:
      return Field.createZonedDateTime(ZonedDateTime.parse(stringValue.trim(), dateTimeFormatter));
    case DECIMAL:
      NumberFormat decimalFormat = NumberFormat.getInstance(dataLocale);
      DecimalFormat df = (DecimalFormat) decimalFormat;
      df.setParseBigDecimal(true);
      Number decimal = df.parse(stringValue.trim());
      BigDecimal bigDecimal = adjustScaleIfNeededForDecimalConversion(new BigDecimal(decimal.toString()), scale, decimalScaleRoundingStrategy);
      Field decimalField = Field.create(Field.Type.DECIMAL, bigDecimal);
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, String.valueOf(bigDecimal.precision()));
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_SCALE, String.valueOf(bigDecimal.scale()));
      return decimalField;
    case DOUBLE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).doubleValue());
    case FLOAT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).floatValue());
    case INTEGER:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).intValue());
    case LONG:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).longValue());
    case SHORT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).shortValue());
    case FILE_REF:
      throw new IllegalArgumentException(Utils.format("Cannot convert String value to type {}", targetType));
    default:
      return field;
  }
}
 

private static Object getValue(
    Descriptors.FieldDescriptor f,
    Field field,
    Record record,
    String protoFieldPath,
    Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
    Map<String, Object> defaultValueMap
) throws DataGeneratorException {
  Object value = null;
  try {
    if (field.getValue() != null) {
      switch (f.getJavaType()) {
        case BOOLEAN:
          value = field.getValueAsBoolean();
          break;
        case BYTE_STRING:
          value = ByteString.copyFrom(field.getValueAsByteArray());
          break;
        case DOUBLE:
          value = field.getValueAsDouble();
          break;
        case ENUM:
          value = f.getEnumType().findValueByName(field.getValueAsString());
          break;
        case FLOAT:
          value = field.getValueAsFloat();
          break;
        case INT:
          value = field.getValueAsInteger();
          break;
        case LONG:
          value = field.getValueAsLong();
          break;
        case STRING:
          value = field.getValueAsString();
          break;
        case MESSAGE:
          Descriptors.Descriptor messageType = f.getMessageType();
          value = sdcFieldToProtobufMsg(
              record, field, protoFieldPath, messageType, messageTypeToExtensionMap, defaultValueMap
          );
          break;
        default:
          throw new DataGeneratorException(Errors.PROTOBUF_03, f.getJavaType().name());
      }
    }
  } catch (IllegalArgumentException e) {
    throw new DataGeneratorException(Errors.PROTOBUF_11, field.getValue(), f.getJavaType().name(), e);
  }
  return value;
}
 

private DocumentMutation populateDocumentMutation(Record rec) throws OnRecordErrorException {
  DocumentMutation documentMutation = MapRJsonDocumentLoader.createDocumentMutation();
  boolean replace = mapRJsonConfigBean.setOrReplace == SetOrReplace.REPLACE;

  if(rec != null && (rec.get().getType() == Field.Type.LIST_MAP || rec.get().getType() == Field.Type.MAP)) {
    Map<String, Field> fields = rec.get().getValueAsMap();
    for(Map.Entry<String, Field> entry : fields.entrySet()) {
      String path = entry.getKey();
      Field field = entry.getValue();

      //don't add the keyField to the document mutation set. that gets added later
      if(entry.getKey().equals(mapRJsonConfigBean.keyField)) {
        continue;
      }
      switch(field.getType()) {
        case DOUBLE:
          double d = field.getValueAsDouble();
          documentMutation = (replace ? documentMutation.setOrReplace(path, d) : documentMutation.set(path, d));
          break;
        case FLOAT:
          float f = field.getValueAsFloat();
          documentMutation = (replace ? documentMutation.setOrReplace(path, f) : documentMutation.set(path, f));
          break;
        case INTEGER:
          int i = field.getValueAsInteger();
          documentMutation = (replace ? documentMutation.setOrReplace(path, i) : documentMutation.set(path, i));
          break;
        case SHORT:
          short s = field.getValueAsShort();
          documentMutation = (replace ? documentMutation.setOrReplace(path, s) : documentMutation.set(path, s));
          break;
        case LONG:
        case DATE:
        case TIME:
        case DATETIME:
          long l = field.getValueAsLong();
          documentMutation = (replace ? documentMutation.setOrReplace(path, l) : documentMutation.set(path, l));
          break;
        case STRING:
          String st = field.getValueAsString();
          documentMutation = (replace ? documentMutation.setOrReplace(path, st) : documentMutation.set(path, st));
          break;
        case BYTE_ARRAY:
          byte[] ba = field.getValueAsByteArray();
          documentMutation = (replace ? documentMutation.setOrReplace(path, ByteBuffer.wrap(ba)) : documentMutation.set(path, ByteBuffer.wrap(ba)));
          break;
        case BOOLEAN:
        case MAP:
        case LIST:
        case LIST_MAP:
        case CHAR:
        case BYTE:
        default:
          throw new OnRecordErrorException(rec, Errors.MAPR_JSON_14, field.getType().name());
      }
    }
  }

  return documentMutation;
}