Java Code Examples for com.streamsets.pipeline.api.Record#getEscapedFieldPaths()

@Test
public void testParseWithExtraColumnsAllowed() throws Exception {
  OverrunReader reader = new OverrunReader(new StringReader("a,b,c\n1,2,3,4"), 1000, true, false);
  DelimitedDataParserSettings settings = DelimitedDataParserSettings.builder()
      .withSkipStartLines(0)
      .withFormat(CSVFormat.DEFAULT)
      .withHeader(CsvHeader.WITH_HEADER)
      .withMaxObjectLen(-1)
      .withRecordType(CsvRecordType.LIST_MAP)
      .withParseNull(false)
      .withNullConstant(null)
      .withAllowExtraColumns(true)
      .withExtraColumnPrefix("_abc_")
      .build();
  DataParser parser = new DelimitedCharDataParser(getContext(), "id", reader, 0, settings);

  Record record = parser.parse();
  Assert.assertNotNull(record);
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  Set<String> expectedFieldPaths = ImmutableSet.of("", "/a", "/b", "/c", "/_abc_01");
  assertTrue(fieldPaths.containsAll(expectedFieldPaths));
}
 

@VisibleForTesting
List<String> getValueFields(Record record) throws OnRecordErrorException {
  List<String> fields = new ArrayList<>();
  Set<String> fieldPaths = record.getEscapedFieldPaths();

  for (String fieldPath : fieldPaths) {
    if (!isValueField(fieldPath)) {
      continue;
    }
    fields.add(stripPathPrefix(fieldPath));
  }

  if (fields.isEmpty()) {
    throw new OnRecordErrorException(Errors.INFLUX_06);
  }

  return fields;
}
 

private void setInputConfigFields(Record record) {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  for (TensorInputConfig inputConfig : conf.inputConfigs) {
    Set<String> inputConfigFields = new HashSet<>();
    for(String f: inputConfig.fields) {
      List<String> matchingFieldPaths = FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          f,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      );
      inputConfigFields.addAll(matchingFieldPaths);
    }
    inputConfig.setResolvedFields(new ArrayList<>(inputConfigFields));
  }
}
 

@Test
public void testNestedKeep8() throws StageException {
  /*
   */
  Record record = createNestedRecord();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[2]/SantaClara/main/streets[*][*]/name12*"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);
    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertTrue(resultRecord.has("/USA[0]/SantaClara/main/streets[1][1]/name12AB"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SantaClara/main/streets[0][0]/name1"));


  } finally {
    runner.runDestroy();
  }
}
 

protected List<String> getFieldsToHash(Record record) {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  List<String> fields = new ArrayList<>();
  if (fieldsToHash != null) {
    for(String field : fieldsToHash) {
      List<String> matchingFieldPaths = FieldRegexUtil.getMatchingFieldPaths(field, fieldPaths);
      Collections.sort(matchingFieldPaths);
      fields.addAll(matchingFieldPaths);
    }
  } else {
    fields = new ArrayList<>(record.getEscapedFieldPaths());
    Collections.sort(fields);
  }
  return fields;
}
 

public static void validateWholeFileRecord(Record record) {
  Set<String> fieldPathsInRecord = record.getEscapedFieldPaths();
  Utils.checkArgument(
      fieldPathsInRecord.containsAll(MANDATORY_FIELD_PATHS),
      Utils.format(
          "Record does not contain the mandatory fields {} for Whole File Format.",
          COMMA_JOINER.join(Sets.difference(MANDATORY_FIELD_PATHS, fieldPathsInRecord))
      )
  );
}
 

protected static void insertRows(String insertTemplate, List<Record> records) throws SQLException {
  try (Statement st = connection.createStatement()) {
    for (Record record : records) {
      List<String> values = new ArrayList<>();
      for (String fieldPath : record.getEscapedFieldPaths()) {
        //Skip root field
        if (!fieldPath.equals("")) {
          values.add(getStringRepOfFieldValueForInsert(record.get(fieldPath)));
        }
      }
      st.addBatch(String.format(insertTemplate, values.toArray()));
    }
    st.executeBatch();
  }
}
 

private void processByField(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  final Set<String> fieldPaths = record.getEscapedFieldPaths();
  for(FieldTypeConverterConfig fieldTypeConverterConfig : fieldTypeConverterConfigs) {
    for(String fieldToConvert : fieldTypeConverterConfig.fields) {
      final List<String> matchingFieldPaths = new LinkedList<>(FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          fieldToConvert,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      ));
      if (matchingFieldPaths.isEmpty()) {
        // FieldPathExpressionUtil.evaluateMatchingFieldPaths does NOT return the supplied param in its result
        // regardless, like FieldRegexUtil#getMatchingFieldPaths did, so we add manually here
        matchingFieldPaths.add(fieldToConvert);
      }
      for (String matchingField : matchingFieldPaths) {
        Field field = record.get(matchingField);
        if(field == null) {
          LOG.trace("Record does not have field {}. Ignoring conversion.", matchingField);
        } else {
          record.set(matchingField, convertField(matchingField, field, fieldTypeConverterConfig));
        }
      }
    }
  }
  batchMaker.addRecord(record);
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Set<String> fieldPaths = record.getEscapedFieldPaths();
  List<String> nonStringFields = new ArrayList<>();
  // For each individual configuration entry
  for(FieldMaskConfig fieldMaskConfig : activeFieldMaskConfigs) {
    // For each configured field expression
    for (String toMask : fieldMaskConfig.fields) {
      // Find all actual fields that matches given configured expression
      for (String matchingFieldPath : FieldPathExpressionUtil.evaluateMatchingFieldPaths(
          toMask,
          fieldPathEval,
          fieldPathVars,
          record,
          fieldPaths
      )) {
        if (record.has(matchingFieldPath)) {
          Field field = record.get(matchingFieldPath);
          if (field.getType() != Field.Type.STRING) {
            nonStringFields.add(matchingFieldPath);
          } else {
            if (field.getValue() != null) {
              Field newField = Field.create(maskField(field, fieldMaskConfig));
              record.set(matchingFieldPath, newField);
            }
          }
        }
      }
    }
  }
  if (nonStringFields.isEmpty()) {
    batchMaker.addRecord(record);
  } else {
    throw new OnRecordErrorException(Errors.MASK_00, StringUtils.join(nonStringFields, ", "), record.getHeader().getSourceId());
  }
}
 

@Test
public void testNestedKeep9() throws StageException {
  /*check for embedded *
   */
  Record record = createNestedRecord();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[2]/SantaClara/ma*/streets[*][*]/name1"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);
    Set <String> r = resultRecord.getEscapedFieldPaths();


    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertFalse(resultRecord.has("/USA[0]/SantaClara/main/streets[1][1]/name12AB"));
    Assert.assertTrue(resultRecord.has("/USA[0]/SantaClara/main/streets[0][0]/name1"));


  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testNestedKeep10() throws StageException {
  /* check for embedded ? and * at end of field and after /
   */
  Record record = createNestedRecordForRegex();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[?]/San*/*/streets[*][*]/name?2"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);

    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noel/streets[0][0]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][1]/name22"));
    Assert.assertFalse(resultRecord.has("/USA[1]/SantaClara/main/streets[0][0]/name21"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name1234"));


  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testNestedKeep11() throws StageException {
  /*
   check for regex * at the end of the line
   */
  Record record = createNestedRecordForRegex();
  ProcessorRunner runner = new ProcessorRunner.Builder(FieldFilterDProcessor.class)
      .addConfiguration("fields", ImmutableList.of(
          "/USA[?]/San*/*/streets[*][*]/name2*"))
      .addConfiguration("filterOperation", FilterOperation.KEEP)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));

    Record resultRecord = output.getRecords().get("a").get(0);

    Assert.assertNotNull(resultRecord);
    Set<String> paths = resultRecord.getEscapedFieldPaths();
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name12"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noel/streets[0][0]/name12"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][1]/name22"));
    Assert.assertTrue(resultRecord.has("/USA[1]/SantaClara/main/streets[0][0]/name21"));
    Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name1234"));


  } finally {
    runner.runDestroy();
  }
}
 

private static void validateRootLevelType(Record record) throws OnRecordErrorException {
  for (String fieldPath : record.getEscapedFieldPaths()) {
    if (fieldPath.isEmpty()) {
      Field.Type type = record.get(fieldPath).getType();
      if (type != Field.Type.MAP && type != Field.Type.LIST_MAP) {
        throw new OnRecordErrorException(record, Errors.HBASE_29, type);
      }
      break;
    }
  }
}
 

private void doImplicitFieldMapping(
    Put p,
    Record record,
    StringBuilder errorMsgBuilder,
    Set<String> explicitFields,
    String timeDriver,
    boolean ignoreMissingField,
    boolean ignoreInvalidColumn,
    String hbaseRowKey
) throws OnRecordErrorException {
  validateRootLevelType(record);
  Date recordTime = getRecordTime(record, timeDriver);
  for (String fieldPath : record.getEscapedFieldPaths()) {
    if (!fieldPath.isEmpty() && !fieldPath.equals(hbaseRowKey) && !explicitFields.contains(fieldPath)) {
      String fieldPathColumn = fieldPath;
      if (fieldPath.charAt(0) == '/') {
        fieldPathColumn = fieldPath.substring(1);
      }
      HBaseColumn hbaseColumn = hbaseConnectionHelper.getColumn(fieldPathColumn.replace("'", ""));
      if (hbaseColumn.getCf().isPresent() && hbaseColumn.getQualifier().isPresent()) {
        byte[] value = getBytesForValue(record, fieldPath, null, ignoreMissingField);
        addCell(p, hbaseColumn.getCf().get(), hbaseColumn.getQualifier().get(), recordTime, value);
      } else if (ignoreInvalidColumn) {
        String errorMessage = Utils.format(Errors.HBASE_28.getMessage(),
            fieldPathColumn,
            KeyValue.COLUMN_FAMILY_DELIMITER
        );
        LOG.warn(errorMessage);
        errorMsgBuilder.append(errorMessage);
      } else {
        throw new OnRecordErrorException(record, Errors.HBASE_28, fieldPathColumn, KeyValue.COLUMN_FAMILY_DELIMITER);
      }
    }
  }
}