Java Code Examples for com.streamsets.pipeline.api.Field#create()

public void testEscapingWithQuotes(String specialChar, String pathSpecialChar) {
  RecordImpl r = new RecordImpl("stage", "source", null, null);
  List<Field> list = new ArrayList<>();
  list.add(Field.create(true));
  Field listField = Field.create(list);
  Map<String, Field> map = new HashMap<>();
  map.put("a" + specialChar, listField);
  Field mapField = Field.create(map);
  r.set(mapField);

  Assert.assertEquals(mapField, r.get());
  Assert.assertEquals(mapField, r.get(""));
  Assert.assertEquals(listField, r.get("/'a" + pathSpecialChar + "'"));
  Assert.assertEquals(Field.create(true), r.get("/'a" + pathSpecialChar + "'[0]"));
  Assert.assertNull(r.get("/'a" + pathSpecialChar + "'[1]"));
  Assert.assertTrue(r.has(""));
  Assert.assertTrue(r.has("/'a" + pathSpecialChar + "'"));
  Assert.assertFalse(r.has("/b"));
  Assert.assertTrue(r.has("/'a" + pathSpecialChar + "'[0]"));
  Assert.assertFalse(r.has("/'a" + pathSpecialChar + "'[1]"));
  Assert.assertEquals(Field.create(true), r.delete("/'a" + pathSpecialChar + "'[0]"));
  Assert.assertEquals(Field.create(new ArrayList<Field>()), r.delete("/'a" + pathSpecialChar + "'"));
}
 

@Test
public void testGetNamespaces() throws IOException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);

  Map<String, String> existingNamespaces = ImmutableMap.of("ns1", "NS1");

  Field field = Field.create(0);
  field.setAttribute("xmlns:ns1", "fNS1");
  field.setAttribute("xmlns:ns2", "NS2");
  field.setAttribute("a", "A");

  Map<String, String> expected = ImmutableMap.of("ns1", "fNS1", "ns2", "NS2");

  Map<String, String> got = generator.getNamespaces(existingNamespaces, field);

  Assert.assertEquals(expected, got);
}
 

@VisibleForTesting
static PostgresWalRecord passesOperationFilter(PostgresWalRecord postgresWalRecord, PostgresCDCSource postgresCDCSource) {
  List<PostgresChangeTypeValues> configuredChangeTypes = postgresCDCSource
      .getConfigBean()
      .postgresChangeTypes;

  List<String> changeTypes = new ArrayList<String>();
  for (PostgresChangeTypeValues configuredChangeType : configuredChangeTypes) {
    changeTypes.add(configuredChangeType.getLabel());
  }

  List<Field> changes = postgresWalRecord.getChanges();
  List<Field> filteredChanges = new ArrayList<Field>();

  for (Field change: changes) {
    String changeType = PostgresWalRecord.getTypeFromChangeMap(change.getValueAsMap());
    if (changeTypes.contains(changeType)) {
      filteredChanges.add(change);
    }
  }

  if (filteredChanges == null || filteredChanges.isEmpty()) {
    return null;
  }
  return new PostgresWalRecord(postgresWalRecord, Field.create(filteredChanges));
}
 

@Test
public void testFieldToXmlElementsFieldBasic() throws IOException, DataGeneratorException {
  Writer writer = Mockito.mock(Writer.class);
  XmlCharDataGenerator generator = new XmlCharDataGenerator(writer, true, ImmutableList.of(SCHEMA), true);
  Document document = generator.getDocumentBuilder().newDocument();

  String name = "element";
  Map<String, String> namespaces = ImmutableMap.of("", "", "ns", "NS");
  Field field = Field.create("data");

  List<Element> elements = generator.fieldToXmlElements(document, namespaces, name, field);
  Assert.assertEquals(1, elements.size());
  Element element = elements.get(0);
  Assert.assertEquals("element", element.getNodeName());
  Assert.assertEquals(null, element.getNamespaceURI());
  Assert.assertEquals(1, element.getChildNodes().getLength());
  Assert.assertEquals("data", element.getTextContent());
}
 

private Field getField(String value) {
  if(settings.parseNull() && settings.getNullConstant() != null && settings.getNullConstant().equals(value)) {
    return Field.create(Field.Type.STRING, null);
  }

  return Field.create(Field.Type.STRING, value);
}
 

@Override
protected Field generateExtraInfoFieldForMetadataRecord(HiveTypeInfo hiveTypeInfo) {
  Map<String, Field> extraInfo = new HashMap<>();
  DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)hiveTypeInfo;
  extraInfo.put(SCALE, Field.create(decimalTypeInfo.getScale()));
  extraInfo.put(PRECISION, Field.create(decimalTypeInfo.getPrecision()));
  return Field.create(extraInfo);
}
 

@Override
protected Field convertPrimitiveObject(Object scriptObject) {
  Field field;
  if (scriptObject instanceof Boolean) {
    field = Field.create((Boolean) scriptObject);
  } else if (scriptObject instanceof Character) {
    field = Field.create((Character) scriptObject);
  } else if (scriptObject instanceof Byte) {
    field = Field.create((Byte) scriptObject);
  } else if (scriptObject instanceof Short) {
    field = Field.create((Short) scriptObject);
  } else if (scriptObject instanceof Integer) {
    field = Field.create((Integer) scriptObject);
  } else if (scriptObject instanceof Long) {
    field = Field.create((Long) scriptObject);
  } else if (scriptObject instanceof BigInteger) { // special handling for Jython LONG type
    field = Field.create(((BigInteger) scriptObject).longValue());
  } else if (scriptObject instanceof Float) {
    field = Field.create((Float) scriptObject);
  } else if (scriptObject instanceof Double) {
    field = Field.create((Double) scriptObject);
  } else if (scriptObject instanceof Date) {
    field = Field.createDatetime((Date) scriptObject);
  } else if (scriptObject instanceof BigDecimal) {
    field = Field.create((BigDecimal) scriptObject);
  } else if (scriptObject instanceof String) {
    field = Field.create((String) scriptObject);
  } else if (scriptObject instanceof byte[]) {
    field = Field.create((byte[]) scriptObject);
  } else if (scriptObject instanceof ScriptObjectFactory.ScriptFileRef) {
    field = Field.create(getFileRefFromScriptFileRef((ScriptObjectFactory.ScriptFileRef) scriptObject));
  } else {
    field = ScriptTypedNullObject.getTypedNullFieldFromScript(scriptObject);
    if (field == null) {
      // unable to find field type from scriptObject. Return null String.
      field = Field.create(scriptObject.toString());
    }
  }
  return field;
}
 

@Override
public BaseFieldBuilder<? extends BaseFieldBuilder> end(String... attributes) {
  if (parentBuilder == null) {
    throw new IllegalStateException("Do not call end on the root builder; just call build when finished");
  }
  final Field field = Field.create(Field.Type.MAP, this.builder.build());
  for (Map.Entry<String, String> attr : buildAttributeMap(attributes).entrySet()) {
    field.setAttribute(attr.getKey(), attr.getValue());
  }
  parentBuilder.handleEndChildField(this.field, field);
  return parentBuilder;
}
 

public static Field getMetadataField(Map<String, Object> metadata) {
  if (metadata != null && !metadata.isEmpty()) {
    Map<String, Field> map = new HashMap<>(metadata.size());
    for (Map.Entry<String, Object> e : metadata.entrySet()) {
      if (e.getValue() instanceof String) {
        map.put(e.getKey(), Field.create((String)e.getValue()));
      }
    }
    return Field.create(map);
  }
  return Field.create(Field.Type.MAP, null);
}
 

public Field read() throws IOException, XMLStreamException {
  if (closed) {
    throw new IOException("The parser has been closed");
  }
  Field field = null;
  if (hasNext(xmlEventReader)) {
    int depth = 0;

    // we need to skip first level elements that are not the record delimiter and we have to ignore record delimiter
    // elements deeper than first level
    while (hasNext(xmlEventReader) && !isStartOfRecord()) {
      depth += processNextEvent();
    }
    if (hasNext(xmlEventReader)) {
      StartElement startE = (StartElement) xmlEventReader.getLastMatchingEvent();
      field = parse(xmlEventReader, startE);

      if (preserveRootElement) {
        field = Field.create(Collections.singletonMap(getElementName(startE), field));
      }

      // the while loop consumes the start element for a record, and the parse method above consumes the end
      // so remove it from the stack
      elementNameStack.removeFirst();
    }
    // if advancing, don't evaluate XPath matches
    xmlEventReader.clearLastMatch();
  }
  return field;
}
 

private void setupBaseCDCRecordField() {
  /* Base CDC record only as a Field. To be augmented */

  TestPgMockCDCRecord testPgMockCDCRecord = new TestPgMockCDCRecord("511", "0/0",
      "2018-07-09 10:16:23.815-07");
  baseRecordField = Field.create(testPgMockCDCRecord.getCDCRecord());
}
 

public static Field getMacAddress(byte[] bytes) {
  StringBuilder sb = new StringBuilder();
  for (int i = 0; i < bytes.length; i++) {
    sb.append(String.format("%02X", bytes[i]));
    if (i < bytes.length - 1) {
      sb.append(":");
    }
  }
  return Field.create(sb.toString());
}
 

@Test
public void testFieldToXmlElementsFieldMap() throws IOException, DataGeneratorException {
  Map<String, Field> map = ImmutableMap.of("entry", Field.create("data"));
  Field field = Field.create(map);
  testFieldToXmlElementsFieldMap(field);
}
 

private Record createRecord(String name, String anotherName) {
  Field name1 = Field.create(name);
  Field name2 = Field.create(anotherName);
  Map<String, Field> nameMap1 = new HashMap<>();
  nameMap1.put("name", name1);
  Map<String, Field> nameMap2 = new HashMap<>();
  nameMap2.put("name", name2);

  Field name3 = Field.create(name);
  Field name4 = Field.create(anotherName);
  Map<String, Field> nameMap3 = new HashMap<>();
  nameMap3.put("name", name3);
  Map<String, Field> nameMap4 = new HashMap<>();
  nameMap4.put("name", name4);

  Field name5 = Field.create("madhu");
  Field name6 = Field.create("girish");
  Map<String, Field> nameMap5 = new HashMap<>();
  nameMap5.put("name", name5);
  Map<String, Field> nameMap6 = new HashMap<>();
  nameMap6.put("name", name6);

  Field first = Field.create(Field.Type.LIST, ImmutableList.of(Field.create(nameMap1), Field.create(nameMap2)));
  Field second = Field.create(Field.Type.LIST, ImmutableList.of(Field.create(nameMap3), Field.create(nameMap4)));
  Field third = Field.create(Field.Type.LIST, ImmutableList.of(Field.create(nameMap5), Field.create(nameMap6)));

  Map<String, Field> noe = new HashMap<>();
  noe.put("streets", Field.create(ImmutableList.of(first, second)));

  Map<String, Field> cole = new HashMap<>();
  cole.put("streets", Field.create(ImmutableList.of(third)));


  Map<String, Field> sfArea = new HashMap<>();
  sfArea.put("noe", Field.create(noe));

  Map<String, Field> utahArea = new HashMap<>();
  utahArea.put("cole", Field.create(cole));


  Map<String, Field> california = new HashMap<>();
  california.put("SanFrancisco", Field.create(sfArea));

  Map<String, Field> utah = new HashMap<>();
  utah.put("SantaMonica", Field.create(utahArea));

  Map<String, Field> map = new LinkedHashMap<>();
  map.put("USA", Field.create(Field.Type.LIST,
      ImmutableList.of(Field.create(california), Field.create(utah))));

  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  return record;
}
 

private void handleHashingForTarget(
    Record record,
    HashType hashType,
    Set<String> fieldsToHashForThisConfig,
    Set<String> fieldsDontExist,
    String targetField,
    String headerAttribute,
    boolean includeRecordHeader,
    boolean useSeparator,
    char separator
) throws StageException {
  // If there is nothing to hash, don't create bogus output hash
  if(fieldsToHashForThisConfig.isEmpty()) {
    return;
  }

  String hashVal = generateHash(
      record,
      hashType,
      fieldsToHashForThisConfig,
      includeRecordHeader,
      useSeparator,
      separator
  );
  if (!targetField.isEmpty()) {
    Field newField = Field.create(hashVal);
    //Handle already existing field.
    if (record.has(targetField)) {
      record.set(targetField, newField);
    } else {
      try {
        record.set(targetField, newField);

      } catch(IllegalArgumentException ex) {
        throw new OnRecordErrorException(Errors.HASH_05, newField.getType().name(), ex);
      }

      if (!record.has(targetField)) {
        fieldsDontExist.add(targetField);
      }
    }
  }
  if (!headerAttribute.isEmpty()) {
    record.getHeader().setAttribute(headerAttribute, hashVal);
  }
}
 

@SuppressWarnings("unchecked")
public static Field jsonToField(Object json) throws IOException {
  Field field;
  if (json == null) {
    field = Field.create(Field.Type.STRING, null);
  } else if (json instanceof List) {
    List jsonList = (List) json;
    List<Field> list = new ArrayList<>(jsonList.size());
    for (Object element : jsonList) {
      list.add(jsonToField(element));
    }
    field = Field.create(list);
  } else if (json instanceof Map) {
    Map<String, Object> jsonMap = (Map<String, Object>) json;
    Map<String, Field> map = new LinkedHashMap<>();
    for (Map.Entry<String, Object> entry : jsonMap.entrySet()) {
      map.put(entry.getKey(), jsonToField(entry.getValue()));
    }
    field = Field.create(map);
  } else if (json instanceof String) {
    field = Field.create((String) json);
  } else if (json instanceof Boolean) {
    field = Field.create((Boolean) json);
  } else if (json instanceof Character) {
    field = Field.create((Character) json);
  } else if (json instanceof Byte) {
    field = Field.create((Byte) json);
  } else if (json instanceof Short) {
    field = Field.create((Short) json);
  } else if (json instanceof Integer) {
    field = Field.create((Integer) json);
  } else if (json instanceof Long) {
    field = Field.create((Long) json);
  } else if (json instanceof Float) {
    field = Field.create((Float) json);
  } else if (json instanceof Double) {
    field = Field.create((Double) json);
  } else if (json instanceof byte[]) {
    field = Field.create((byte[]) json);
  } else if (json instanceof Date) {
    field = Field.createDatetime((Date) json);
  } else if (json instanceof BigDecimal) {
    field = Field.create((BigDecimal) json);
  } else if (json instanceof UUID) {
    field = Field.create(json.toString());
  } else {
    throw new IOException(Utils.format("Not recognized type '{}', value '{}'", json.getClass(), json));
  }
  return field;
}
 

public static Field createFieldForMetadata(Object metadataObject) {
  if (metadataObject == null) {
    return Field.create("");
  }
  if (metadataObject instanceof Boolean) {
    return Field.create((Boolean) metadataObject);
  } else if (metadataObject instanceof Character) {
    return Field.create((Character) metadataObject);
  } else if (metadataObject instanceof Byte) {
    return Field.create((Byte) metadataObject);
  } else if (metadataObject instanceof Short) {
    return Field.create((Short) metadataObject);
  } else if (metadataObject instanceof Integer) {
    return Field.create((Integer) metadataObject);
  } else if (metadataObject instanceof Long) {
    return Field.create((Long) metadataObject);
  } else if (metadataObject instanceof Float) {
    return Field.create((Float) metadataObject);
  } else if (metadataObject instanceof Double) {
    return Field.create((Double) metadataObject);
  } else if (metadataObject instanceof Date) {
    return Field.createDatetime((Date) metadataObject);
  } else if (metadataObject instanceof BigDecimal) {
    return Field.create((BigDecimal) metadataObject);
  } else if (metadataObject instanceof String) {
    return Field.create((String) metadataObject);
  } else if (metadataObject instanceof byte[]) {
    return Field.create((byte[]) metadataObject);
  } else if (metadataObject instanceof Collection) {
    Iterator iterator = ((Collection)metadataObject).iterator();
    List<Field> fields = new ArrayList<>();
    while (iterator.hasNext()) {
      fields.add(createFieldForMetadata(iterator.next()));
    }
    return Field.create(fields);
  } else if (metadataObject instanceof Map) {
    boolean isListMap = (metadataObject instanceof LinkedHashMap);
    Map<String, Field> fieldMap = isListMap? new LinkedHashMap<>() : new HashMap<>();
    Map<Object, Object> map = (Map)metadataObject;
    for (Map.Entry<Object, Object> entry : map.entrySet()) {
      fieldMap.put(entry.getKey().toString(), createFieldForMetadata(entry.getValue()));
    }
    return isListMap? Field.create(Field.Type.LIST_MAP, fieldMap) : Field.create(fieldMap);
  } else {
    return Field.create(metadataObject.toString());
  }
}
 

private Field convertStringToTargetType(
    Field field,
    Field.Type targetType,
    Locale dataLocale,
    String dateMask,
    int scale,
    DecimalScaleRoundingStrategy decimalScaleRoundingStrategy,
    DateTimeFormatter dateTimeFormatter
) throws ParseException {
  String stringValue = field.getValueAsString();
  switch(targetType) {
    case BOOLEAN:
      return Field.create(Field.Type.BOOLEAN, Boolean.valueOf(stringValue.trim()));
    case BYTE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue).byteValue());
    case BYTE_ARRAY:
      return Field.create(stringValue.getBytes(StandardCharsets.UTF_8));
    case CHAR:
      return Field.create(stringValue.charAt(0));
    case DATE:
      java.text.DateFormat dateFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDate(dateFormat.parse(stringValue.trim()));
    case DATETIME:
      java.text.DateFormat dateTimeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createDatetime(dateTimeFormat.parse(stringValue.trim()));
    case TIME:
      java.text.DateFormat timeFormat = new SimpleDateFormat(dateMask, Locale.ENGLISH);
      return Field.createTime(timeFormat.parse(stringValue.trim()));
    case ZONED_DATETIME:
      return Field.createZonedDateTime(ZonedDateTime.parse(stringValue.trim(), dateTimeFormatter));
    case DECIMAL:
      NumberFormat decimalFormat = NumberFormat.getInstance(dataLocale);
      DecimalFormat df = (DecimalFormat) decimalFormat;
      df.setParseBigDecimal(true);
      Number decimal = df.parse(stringValue.trim());
      BigDecimal bigDecimal = adjustScaleIfNeededForDecimalConversion(new BigDecimal(decimal.toString()), scale, decimalScaleRoundingStrategy);
      Field decimalField = Field.create(Field.Type.DECIMAL, bigDecimal);
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_PRECISION, String.valueOf(bigDecimal.precision()));
      decimalField.setAttribute(HeaderAttributeConstants.ATTR_SCALE, String.valueOf(bigDecimal.scale()));
      return decimalField;
    case DOUBLE:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).doubleValue());
    case FLOAT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).floatValue());
    case INTEGER:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).intValue());
    case LONG:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).longValue());
    case SHORT:
      return Field.create(NumberFormat.getInstance(dataLocale).parse(stringValue.trim()).shortValue());
    case FILE_REF:
      throw new IllegalArgumentException(Utils.format("Cannot convert String value to type {}", targetType));
    default:
      return field;
  }
}
 

@Override
protected Field processField(Record record, byte[] fieldData) throws OnRecordErrorException {
  return Field.create(Base64.encodeBase64(fieldData, false, urlSafe));
}
 

public Field build() {
  return Field.create(listMap ? Field.Type.LIST_MAP : Field.Type.MAP, builder.build());
}