Java Code Examples for com.streamsets.pipeline.api.Record#get()

@Override
public void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Field original = record.get(originFieldPath);
  try {
    if (original != null) {
      if (original.getType() != Field.Type.BYTE_ARRAY) {
        throw new OnRecordErrorException(
            CommonError.CMN_0100, original.getType().toString(), original.getValue().toString(), record.toString());
      }
      record.set(resultFieldPath, processField(record, original.getValueAsByteArray()));
    }
    batchMaker.addRecord(record);
  } catch (OnRecordErrorException error) {
    errorRecordHandler.onError(
        new OnRecordErrorException(
            record,
            error.getErrorCode(),
            error.getParams()
        )
    );
  }
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  if (!record.has(fieldPathToSerialize)) {
    throw new OnRecordErrorException(Errors.JSON_00, fieldPathToSerialize, record.getHeader().getSourceId());
  }

  Field field = record.get(fieldPathToSerialize);
  if (field.getValue() == null) {
    throw new OnRecordErrorException(Errors.JSON_01, fieldPathToSerialize);
  }
  if (!supportedFieldTypes.contains(field.getType())) {
    throw new OnRecordErrorException(Errors.JSON_02, fieldPathToSerialize, field.getType());
  }

  Record tempRecord = getContext().createRecord(record.getHeader().getSourceId());
  tempRecord.set(field);
  Writer writer = new StringWriter();
  try (JsonCharDataGenerator generator = new JsonCharDataGenerator(getContext(), writer, Mode.MULTIPLE_OBJECTS)) {
    generator.write(tempRecord);
  } catch (IOException e) {
    throw new OnRecordErrorException(Errors.JSON_03, e.toString(), e);
  }

  record.set(outputFieldPath, Field.create(writer.toString().trim()));
  batchMaker.addRecord(record);
}
 

private void setFieldsInRecord(Record record, Map<String, Field>fields) {
  for (Map.Entry<String, Field> entry : fields.entrySet()) {
    String columnName = entry.getKey();
    String fieldPath = columnsToFields.get(columnName);
    Field field = entry.getValue();
    if (fieldPath == null) {
      Field root = record.get();
      // No mapping
      switch (root.getType()) {
        case LIST:
          // Add new field to the end of the list
          fieldPath = "[" + root.getValueAsList().size() + "]";
          Map<String, Field> cell = new HashMap<>();
          cell.put("header", Field.create(columnName));
          cell.put("value", field);
          field = Field.create(cell);
          break;
        case LIST_MAP:
        case MAP:
          // Just use the column name
          fieldPath = "/" + columnName;
          break;
        default:
          break;
      }
    }
    record.set(fieldPath, field);
  }
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  ELVars vars = getContext().createELVars();
  RecordEL.setRecordInContext(vars, record);

  for(ReplaceRule rule : conf.rules) {
    // Generate list of applicable paths to given expression
    List<String> fieldPaths = FieldPathExpressionUtil.evaluateMatchingFieldPaths(
      rule.fields,
      pathEval,
      vars,
      record,
      record.getEscapedFieldPaths()
    );

    if(fieldPaths.isEmpty() && conf.onStagePreConditionFailure == OnStagePreConditionFailure.TO_ERROR) {
      throw new OnRecordErrorException(record, Errors.FIELD_REPLACER_00, rule.fields);
    }

    // Perform the replacement
    for(String path : fieldPaths) {
      if(!record.has(path) && conf.onStagePreConditionFailure == OnStagePreConditionFailure.TO_ERROR) {
        throw new OnRecordErrorException(record, Errors.FIELD_REPLACER_00, rule.fields);
      }

      Field field = record.get(path);
      FieldEL.setFieldInContext(vars, path, null, field);

      if(rule.setToNull) {
        record.set(path, Field.create(field.getType(), null));
      } else {
        Object result = replacementEval.eval(vars, rule.replacement, Object.class);
        record.set(path, Field.create(FieldUtils.getTypeFromObject(result), result));
      }

    }
  }

  batchMaker.addRecord(record);
}
 

@Test
public void generateString() throws Exception {
  DataGeneratorConfig config = new DataGeneratorConfig();
  config.outputType = OutputType.STRING;
  config.targetField = "/";

  ProcessorRunner runner = new ProcessorRunner.Builder(DataGeneratorDProcessor.class, new DataGeneratorProcessor(config))
    .addOutputLane("out")
    .addService(DataFormatGeneratorService.class, new SdkJsonDataFormatGeneratorService())
    .build();

  runner.runInit();
  try {
    Map<String, Field> map = new LinkedHashMap<>();
    map.put("name", Field.create("StreamSets Inc."));
    map.put("age", Field.create(Field.Type.LONG, 4));

    Record record = RecordCreator.create("s", "s:1");
    record.set(Field.create(map));

    StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    List<Record> outputRecords = output.getRecords().get("out");
    assertEquals(1, outputRecords.size());

    Record outputRecord = outputRecords.get(0);
    assertTrue(outputRecord.has("/"));
    assertFalse(outputRecord.has("/name"));
    assertFalse(outputRecord.has("/age"));

    Field rootField = outputRecord.get();
    assertEquals(Field.Type.STRING, rootField.getType());
    assertEquals("{\"name\":\"StreamSets Inc.\",\"age\":4}", rootField.getValueAsString());
  } finally {
    runner.runDestroy();
  }
}
 

private void writeLine(Record record, String key) throws IOException, DataGeneratorException{
  Field field = record.get();

  if (field.getType() != Field.Type.LIST && field.getType() != Field.Type.LIST_MAP) {
    throw new DataGeneratorException(Errors.DELIMITED_GENERATOR_00, record.getHeader().getSourceId(), field.getType());
  }

  List<Field> columns = field.getValueAsList();
  List<String> values = new ArrayList<>(columns.size());
  boolean isListMap = field.getType() == Field.Type.LIST_MAP;
  for (int i = 0; i< columns.size(); i++) {
    Field column = columns.get(i);
    String value;
    if (!isListMap) {
      try {
        value = column.getValueAsMap().get(key).getValueAsString();
      } catch (Exception ex) {
        throw new DataGeneratorException(Errors.DELIMITED_GENERATOR_01, record.getHeader().getSourceId(), i,
            column.getType());
      }
    } else {
      value = column.getValueAsString();
    }
    if (value != null) {
      if (replaceNewLines != null) {
        if (value.contains("\n")) {
          value = value.replace("\n", replaceNewLines);
        }
        if (value.contains("\r")) {
          value = value.replace("\r", replaceNewLines);
        }
      }
    } else {
      value = "";
    }
    values.add(value);
  }
  printer.printRecord(values);
}
 

@Test
public void testErrorLogFormat() throws StageException, IOException {

  LogParserServiceImpl logParserServiceImpl = getDataParserService();
  logParserServiceImpl.logParserServiceConfig.logMode = LogMode.APACHE_ERROR_LOG_FORMAT;

  logParserService.init();

  Map<String, Field> map = new LinkedHashMap<>();
  map.put("text", Field.create(APACHE_ERROR_LOG_FORMAT_LINE));
  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  Record parsedRecord = logParserServiceImpl.getLogParser(
      record.getHeader().getSourceId(),
      record.get("/text").getValueAsString()
  ).parse();

  Field parsed = parsedRecord.get();
  parsedRecord.set("/log", parsed);

  Assert.assertFalse(parsedRecord.has("/log/truncated"));

  Assert.assertTrue(parsedRecord.has("/log/" + Constants.TIMESTAMP));
  Assert.assertEquals("Wed Oct 11 14:32:52 2000", parsedRecord.get("/log/" + Constants.TIMESTAMP).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/log/" + Constants.LOGLEVEL));
  Assert.assertEquals("error", parsedRecord.get("/log/" + Constants.LOGLEVEL).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/log/" + Constants.CLIENTIP));
  Assert.assertEquals("127.0.0.1", parsedRecord.get("/log/" + Constants.CLIENTIP).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/log/" + Constants.MESSAGE));
  Assert.assertEquals("client denied by server configuration1: /export/home/live/ap/htdocs/test1",
      parsedRecord.get("/log/" + Constants.MESSAGE).getValueAsString());

}
 

@Test
public void testLog4jFormat() throws StageException, IOException {

  LogParserServiceImpl logParserServiceImpl = getDataParserService();
  logParserServiceImpl.logParserServiceConfig.logMode = LogMode.LOG4J;

  logParserService.init();

  Map<String, Field> map = new LinkedHashMap<>();
  map.put("text", Field.create(LOG_4J_FORMAT_LINE));
  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  Record parsedRecord = logParserServiceImpl.getLogParser(
      record.getHeader().getSourceId(),
      record.get("/text").getValueAsString()
  ).parse();

  Field parsed = parsedRecord.get();
  parsedRecord.set("/", parsed);

  Assert.assertFalse(parsedRecord.has("/truncated"));

  Assert.assertTrue(parsedRecord.has("/" + Constants.TIMESTAMP));
  Assert.assertEquals("2015-03-20 15:53:31,161", parsedRecord.get("/" + Constants.TIMESTAMP).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/" + Constants.SEVERITY));
  Assert.assertEquals("DEBUG", parsedRecord.get("/" + Constants.SEVERITY).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/" + Constants.CATEGORY));
  Assert.assertEquals("PipelineConfigurationValidator", parsedRecord.get("/" + Constants.CATEGORY).getValueAsString());

  Assert.assertTrue(parsedRecord.has("/" + Constants.MESSAGE));
  Assert.assertEquals("Pipeline 'test:preview' validation. valid=true, canPreview=true, issuesCount=0",
      parsedRecord.get("/" + Constants.MESSAGE).getValueAsString());

}
 

@Test
public void testProduceFullFileWithStackTrace() throws Exception {

  LogParserServiceImpl logParserServiceImpl = getDataParserService();
  logParserServiceImpl.logParserServiceConfig.logMode = LogMode.LOG4J;
  logParserServiceImpl.logParserServiceConfig.regex = REGEX;
  logParserServiceImpl.logParserServiceConfig.fieldPathsToGroupName = REGEX_CONFIG;

  List<ConfigIssue> issues = logParserService.init();

  Map<String, Field> map = new LinkedHashMap<>();
  map.put("text", Field.create(LOG_LINE_WITH_STACK_TRACE));
  Record record = RecordCreator.create("s", "s:1");
  record.set(Field.create(map));

  try {
    Record parsedRecord = logParserServiceImpl.getLogParser(
        record.getHeader().getSourceId(),
        record.get("/text").getValueAsString()
    ).parse();

    Field parsed = parsedRecord.get();
    parsedRecord.set("/", parsed);
  } catch (DataParserException ex) {
    Assert.assertTrue(ex.getErrorCode().toString().equals("LOG_PARSER_03"));
  }


}
 

private DefaultLeapFrame convertRecordsToLeapFrame(List<Record> records) throws StageException {
  StructType inputSchema = mLeapPipeline.inputSchema();
  List<StructField> structFieldList = leapFrameSupport.getFields(inputSchema);
  List<Row> mLeapRows = new ArrayList<>();

  List<Record> errorRecords = new ArrayList<>();

  for(Record record: records) {
    List<Object> rowValues = new ArrayList<>();
    boolean foundError = false;
    for (StructField structField: structFieldList) {
      Field input = record.get(fieldNameMap.get(structField.name()));
      if (input != null) {
        rowValues.add(input.getValue());
      } else {
        errorRecordHandler.onError(new OnRecordErrorException(
            record,
            Errors.MLEAP_04,
            fieldNameMap.get(structField.name()),
            record.getHeader().getSourceId()
        ));
        errorRecords.add(record);
        foundError = true;
      }
    }

    if (!foundError) {
      mLeapRows.add(leapFrameBuilder.createRowFromIterable(rowValues));
    }
  }

  // Remove all error records from records to list before it used in processTransformOutput
  records.removeAll(errorRecords);

  return leapFrameBuilder.createFrame(mLeapPipeline.inputSchema(), mLeapRows);
}
 

private void setFieldsInRecord(Record record, Map<String, Field> fields) {
  for (Map.Entry<String, Field> entry : fields.entrySet()) {
    String columnName = entry.getKey();
    String fieldPath = columnToField.get(columnName);
    Field field = entry.getValue();
    if (fieldPath == null) {
      Field root = record.get();
      // No mapping
      switch (root.getType()) {
        case LIST:
          // Add new field to the end of the list
          fieldPath = "[" + root.getValueAsList().size() + "]";
          Map<String, Field> cell = new HashMap<>();
          cell.put("header", Field.create(columnName));
          cell.put("value", field);
          field = Field.create(cell);
          break;
        case LIST_MAP:
        case MAP:
          // Just use the column name
          fieldPath = columnName;
          break;
        default:
          break;
      }
    }
    record.set(fieldPath, field);
  }
}
 

@Override
public void funnel(Record record, PrimitiveSink sink) {
  for (String path : getFieldsToHash(record)) {
    Field field = record.get(path);
    if (field == null) {
      throw new IllegalArgumentException(
          Utils.format("Field Path {}  does not exist in the record", path)
      );
    }
    if (field.getValue() != null) {
      switch (field.getType()) {
        case BOOLEAN:
          sink.putBoolean(field.getValueAsBoolean());
          break;
        case CHAR:
          sink.putChar(field.getValueAsChar());
          break;
        case BYTE:
          sink.putByte(field.getValueAsByte());
          break;
        case SHORT:
          sink.putShort(field.getValueAsShort());
          break;
        case INTEGER:
          sink.putInt(field.getValueAsInteger());
          break;
        case LONG:
          sink.putLong(field.getValueAsLong());
          break;
        case FLOAT:
          sink.putFloat(field.getValueAsFloat());
          break;
        case DOUBLE:
          sink.putDouble(field.getValueAsDouble());
          break;
        case DATE:
          sink.putLong(field.getValueAsDate().getTime());
          break;
        case TIME:
          sink.putLong(field.getValueAsTime().getTime());
          break;
        case DATETIME:
          sink.putLong(field.getValueAsDatetime().getTime());
          break;

        case DECIMAL:
        case STRING:
          sink.putString(field.getValueAsString(), Charset.defaultCharset());
          break;

        case BYTE_ARRAY:
          sink.putBytes(field.getValueAsByteArray());
          break;
        case FILE_REF:
          throw new IllegalStateException(
              Utils.format(
                  "Hashing not supported for field: {} of type {}",
                  path,
                  field.getType()
              )
          );
        default:
          break;
      }
    } else {
      sink.putBoolean(true);
    }
    if(useSeparators) {
      sink.putString(java.nio.CharBuffer.wrap(new char[] {separator}), Charset.forName("UTF-8"));
    }
  }

  if (this.includeRecordHeader) {
    for (String attrName : record.getHeader().getAttributeNames()) {
      String headerAttr = record.getHeader().getAttribute(attrName);
      if (headerAttr != null) {
        sink.putString(headerAttr, Charset.defaultCharset());
      } else {
        sink.putBoolean(true);
      }

      if(useSeparators) {
        sink.putString(java.nio.CharBuffer.wrap(new char[] {separator}), Charset.forName("UTF-8"));
      }
    }
  }
}
 

@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  Path tempParquetFile = null;
  try {
    validateRecord(record);

    Map<String, Field> sourceFileMetaData = record.get(FileRefUtil.FILE_INFO_FIELD_PATH).getValueAsMap();
    if (sourceFileMetaData.get(FILENAME) == null) {
      throw new TransformerStageCheckedException(Errors.CONVERT_03, FileRefUtil.FILE_INFO_FIELD_PATH + "/" + FILENAME);
    }

    String sourceFileName = sourceFileMetaData.get(FILENAME).getValueAsString();
    tempParquetFile = getAndValidateTempFilePath(record, sourceFileName);

    InputStream is = getAvroInputStream(record);

    DataFileStream<GenericRecord> fileReader = getFileReader(is, sourceFileName);

    writeParquet(sourceFileName, fileReader, tempParquetFile);

    Map<String, Object> metadata = generateHeaderAttrs(tempParquetFile);

    // build parquet OutputStream
    FileRef newFileRef = new WholeFileTransformerFileRef.Builder().filePath(tempParquetFile.toString()).bufferSize(
        jobConfig.wholeFileMaxObjectLen).rateLimit(FileRefUtil.evaluateAndGetRateLimit(rateLimitElEval,
        variables,
        jobConfig.rateLimit
    )).createMetrics(true).build();

    // move the file info field to source file info field
    Field fileInfo = record.get(FileRefUtil.FILE_INFO_FIELD_PATH);

    record.set(FileRefUtil.getWholeFileRecordRootField(newFileRef, metadata));
    record.set(FileRefUtil.WHOLE_FILE_SOURCE_FILE_INFO_PATH, fileInfo);

    batchMaker.addRecord(record);
  } catch (TransformerStageCheckedException ex) {
    if (tempParquetFile != null) {
      try {
        handleOldTempFiles(tempParquetFile);
      } catch (IOException ex1) {
        LOG.error("failed to delete temporary parquet file : {}", tempParquetFile.toString(), ex1);
      }
    }
    LOG.error(ex.getMessage(), ex.getParams(), ex);
    errorRecordHandler.onError(new OnRecordErrorException(record, ex.getErrorCode(), ex.getParams()));
  }
}
 

private void doBatch(Batch batch) throws StageException {
  Iterator<Record> records = batch.getRecords();
  List<ErrorRecord> tempRecord = new ArrayList<>();
  Pipeline p;

  try {
    p = jedis.pipelined();

    while (records.hasNext()) {
      Record record = records.next();
      for (RedisFieldMappingConfig parameters : conf.redisFieldMapping) {
        String key = null;
        // Special treatment is only given to deletes -
        // all other records will be handled as an upsert.
        if (OperationType.DELETE_CODE == getOperationFromHeader(record)) {
          key = getDeleteKey(record, parameters);
          doDeleteRecord(record, tempRecord, p, key);
          continue;
        }

        if (record.has(parameters.keyExpr)) {
          key = record.get(parameters.keyExpr).getValueAsString();
        }
        Field value = record.get(parameters.valExpr);

        if (key != null && value != null) {
          switch (parameters.dataType) {
            case STRING:
              doUpsertString(record, tempRecord, p, key, value);
              break;
            case LIST:
              doUpsertList(record, tempRecord, p, key, value);
              break;
            case SET:
              doUpsertSet(record, tempRecord, p, key, value);
              break;
            case HASH:
              doUpsertHash(record, tempRecord, p, key, value);
              break;
            default:
              LOG.error(Errors.REDIS_05.getMessage(), parameters.dataType);
              errorRecordHandler.onError(new OnRecordErrorException(record, Errors.REDIS_05, parameters.dataType));
              break;
          }
        } else {
          LOG.warn(Errors.REDIS_07.getMessage(), parameters.keyExpr, parameters.valExpr, record);
        }

        // set the expire time
        if (parameters.ttl > 0) {
          p.expire(key, parameters.ttl);
        }
      }
    }

    List<Object> results = p.syncAndReturnAll();

    int index = 0;
    for (Object result : results) {
      if (!("OK".equals(result) || Long.class.equals(result == null ? null : result.getClass()))) {
        LOG.error(
            Errors.REDIS_03.getMessage(),
            tempRecord.get(index).operation,
            tempRecord.get(index).key,
            tempRecord.get(index).value
        );
        errorRecordHandler.onError(new OnRecordErrorException(
            tempRecord.get(index).record,
            Errors.REDIS_03,
            tempRecord.get(index).operation,
            tempRecord.get(index).key,
            tempRecord.get(index).value,
            result.toString()
        ));
      }
      index++;
    }
    retries = 0;
  } catch (JedisException ex) {
    handleException(ex, batch, tempRecord);
  }
}
 

@Test
public void move() throws StageException {
  final FieldMapperProcessorConfig config = new FieldMapperProcessorConfig();
  config.operateOn = OperateOn.FIELD_PATHS;
  config.mappingExpression = "/outputs/${f:name()}";
  config.conditionalExpression = "${str:startsWith(f:path(), '/inputs')}";
  config.structureChangeAllowed = true;

  final ProcessorRunner runner = new ProcessorRunner.Builder(
      FieldMapperDProcessor.class,
      new FieldMapperProcessor(config)
  ).addOutputLane("a").build();
  runner.runInit();

  try {
    final MapFieldBuilder builder = MapFieldBuilder.builder();
    builder.startListMap("outputs").end().startListMap("inputs")
        .add("first", 1)
        .add("second", 2)
        .add("third", 3)
        .end().startListMap("leaveMeAlone")
        .add("important1", "foo")
        .add("important2", "bar")
        .end();

    final Record record = RecordCreator.create("s", "s:1");
    record.set(builder.build());

    final StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    final Record outputRecord = output.getRecords().get("a").get(0);
    final Field outputs = assertInputsEmptyAndReturnOutputs(outputRecord);
    assertThat(outputs, Matchers.mapFieldWithEntry("first", 1));
    assertThat(outputs, Matchers.mapFieldWithEntry("second", 2));
    assertThat(outputs, Matchers.mapFieldWithEntry("third", 3));

    // make sure the "leaveMeAlone" map was left alone
    final Field leaveMeAloneMap = outputRecord.get("/leaveMeAlone");
    assertThat(leaveMeAloneMap, notNullValue());
    assertThat(leaveMeAloneMap, Matchers.mapFieldWithEntry("important1", "foo"));
    assertThat(leaveMeAloneMap, Matchers.mapFieldWithEntry("important2", "bar"));
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void changeNamesWithLists() throws StageException {
  final FieldMapperProcessorConfig config = new FieldMapperProcessorConfig();
  config.operateOn = OperateOn.FIELD_NAMES;
  // replace dots, carets, and slashes in field names with underscore
  config.mappingExpression = "${str:replaceAll(f:name(), '[A-Z]', '_')}";
  config.structureChangeAllowed = true;

  final ProcessorRunner runner = new ProcessorRunner.Builder(
      FieldMapperDProcessor.class,
      new FieldMapperProcessor(config)
  ).addOutputLane("a").build();
  runner.runInit();

  try {
    final MapFieldBuilder builder = MapFieldBuilder.builder();
    builder.startListMap("outputs").end().startListMap("inputs")
        .add("Foo", 1)
        .add("Bar", 2)
        .startList("BazList")
        .add("listItem1")
        .add("listItem2")
        .add("listItem3")
        .end() // end BazList
        .end(); // end inputs

    final Record record = RecordCreator.create("s", "s:1");
    record.set(builder.build());

    final StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    final Record outputRecord = output.getRecords().get("a").get(0);
    final Field outputs = assertInputsAndReturnOutputs(outputRecord, false);
    assertTrue(outputs.getValueAsMap().isEmpty());
    final Field inputs = outputRecord.get("/inputs");
    assertThat(inputs, notNullValue());
    assertThat(inputs.getType(), IS_A_MAP);
    assertThat(inputs, Matchers.mapFieldWithEntry("_oo", 1));
    assertThat(inputs, Matchers.mapFieldWithEntry("_ar", 2));
    final Field bazList = inputs.getValueAsMap().get("_az_ist");
    assertThat(bazList, notNullValue());
    assertThat(bazList.getType(), equalTo(Field.Type.LIST));
    assertThat(bazList, Matchers.listFieldWithValues("listItem1", "listItem2", "listItem3"));
  } finally {
    runner.runDestroy();
  }
}
 

@Test
public void testMultiCharDelimiters() throws Exception {
  // multiple character field separator
  final String multiCharFieldDelimiter = "||";
  // add a null constant and field for it in the input data
  final String nullConstant = "\\N";
  // use a strange line delimiter (not newline)
  final String lineDelimiter = "%%FOO%%";

  // craft the input data
  final String inputData = String.format(
      "field1%1$sfield2%1$sfield3%3$s" +
          "one%1$stwo%1$sthree%3$s" +
          "four%1$s%2$s%1$ssix%3$s" +
          "seven%1$seight%1$snine",
      multiCharFieldDelimiter,
      nullConstant,
      lineDelimiter
  );

  // get the index of the start of each "line", for offset checking later
  final int indexOfFirstLine = inputData.indexOf("one");
  final int indexOfSecondLine = inputData.indexOf("four");
  final int indexOfThirdLine = inputData.indexOf("seven");

  final OverrunReader reader = new OverrunReader(new StringReader(inputData), 1000, true, false);
  final DelimitedDataParserSettings settings = DelimitedDataParserSettings.builder()
      .withSkipStartLines(0)
      .withMultiCharacterFieldDelimiter(multiCharFieldDelimiter)
      .withHeader(CsvHeader.WITH_HEADER)
      .withMaxObjectLen(4096)
      .withRecordType(CsvRecordType.LIST_MAP)
      .withParseNull(true)
      .withNullConstant(nullConstant)
      .withAllowExtraColumns(false)
      .withMultiCharQuoteChar('"')
      .withMultiCharEscapeChar('\\')
      .withMultiCharacterLineDelimiter(lineDelimiter)
      .build();
  final DataParser parser = new DelimitedCharDataParser(getContext(), "id", reader, 0, settings);

  Assert.assertEquals("0", parser.getOffset());

  final Record record1 = parser.parse();
  final Field rootField1 = record1.get();
  assertThat(rootField1, Matchers.mapFieldWithEntry("field1", "one"));
  assertThat(rootField1, Matchers.mapFieldWithEntry("field2", "two"));
  assertThat(rootField1, Matchers.mapFieldWithEntry("field3", "three"));

  Assert.assertEquals(String.valueOf(indexOfSecondLine - indexOfFirstLine), parser.getOffset());

  final Record record2 = parser.parse();
  final Field rootField2 = record2.get();
  assertThat(rootField2, Matchers.mapFieldWithEntry("field1", "four"));
  assertThat(rootField2, Matchers.mapFieldWithEntry("field2", (String) null));
  assertThat(rootField2, Matchers.mapFieldWithEntry("field3", "six"));

  Assert.assertEquals(String.valueOf(indexOfThirdLine - indexOfFirstLine), parser.getOffset());

  final Record record3 = parser.parse();
  final Field rootField3 = record3.get();
  assertThat(rootField3, Matchers.mapFieldWithEntry("field1", "seven"));
  assertThat(rootField3, Matchers.mapFieldWithEntry("field2", "eight"));
  assertThat(rootField3, Matchers.mapFieldWithEntry("field3", "nine"));

  Assert.assertEquals(String.valueOf(inputData.length() - indexOfFirstLine), parser.getOffset());

  final Record record4 = parser.parse();
  assertThat(record4, nullValue());

  parser.close();
}
 

@Test
public void testRandomDataGenerator() throws Exception {

  RandomDataGeneratorSource.DataGeneratorConfig stringData = new RandomDataGeneratorSource.DataGeneratorConfig();
  stringData.field = "name";
  stringData.type = RandomDataGeneratorSource.Type.STRING;

  RandomDataGeneratorSource.DataGeneratorConfig intData = new RandomDataGeneratorSource.DataGeneratorConfig();
  intData.field = "age";
  intData.type = RandomDataGeneratorSource.Type.INTEGER;

  RandomDataGeneratorSource.DataGeneratorConfig longData = new RandomDataGeneratorSource.DataGeneratorConfig();
  longData.field = "milliSecondsSinceBirth";
  longData.type = RandomDataGeneratorSource.Type.LONG;

  RandomDataGeneratorSource.DataGeneratorConfig dateData = new RandomDataGeneratorSource.DataGeneratorConfig();
  dateData.field = "dob";
  dateData.type = RandomDataGeneratorSource.Type.DATE;

  RandomDataGeneratorSource.DataGeneratorConfig doubleData = new RandomDataGeneratorSource.DataGeneratorConfig();
  doubleData.field = "salary";
  doubleData.type = RandomDataGeneratorSource.Type.DOUBLE;

  RandomDataGeneratorSource.DataGeneratorConfig decimalData = new RandomDataGeneratorSource.DataGeneratorConfig();
  decimalData.field = "decimal";
  decimalData.type = RandomDataGeneratorSource.Type.DECIMAL;
  decimalData.scale = 2;
  decimalData.precision = 5;

  RandomDataGeneratorSource.DataGeneratorConfig zonedDateTimeData =
      new RandomDataGeneratorSource.DataGeneratorConfig();
  zonedDateTimeData.field = "zonedDateTime";
  zonedDateTimeData.type = RandomDataGeneratorSource.Type.ZONED_DATETIME;

  final PushSourceRunner runner = new PushSourceRunner.Builder(RandomDataGeneratorSource.class)
    .addConfiguration("dataGenConfigs",
        Arrays.asList(stringData, dateData, doubleData, longData, intData, decimalData, zonedDateTimeData))
    .addConfiguration("rootFieldType", RandomDataGeneratorSource.RootType.MAP)
    .addConfiguration("delay", 0)
    .addConfiguration("batchSize", 1000)
    .addConfiguration("numThreads", 1)
    .addConfiguration("eventName", "secret-name")
    .addOutputLane("a")
    .build();
  runner.runInit();
  try {
    final List<Record> records = new ArrayList<>();
    runner.runProduce(Collections.<String, String>emptyMap(), 1, new PushSourceRunner.Callback() {
      @Override
      public void processBatch(StageRunner.Output output) {
        records.clear();
        records.addAll(output.getRecords().get("a"));
        runner.setStop();
      }
    });
    runner.waitOnProduce();

    Assert.assertEquals(1, records.size());
    Record record =  records.get(0);
    Assert.assertEquals(Field.Type.STRING, record.get("/name").getType());
    Assert.assertEquals(Field.Type.INTEGER, record.get("/age").getType());
    Assert.assertEquals(Field.Type.LONG, record.get("/milliSecondsSinceBirth").getType());
    Assert.assertEquals(Field.Type.DATE, record.get("/dob").getType());
    Assert.assertEquals(Field.Type.DOUBLE, record.get("/salary").getType());

    Field decimalField = record.get("/decimal");
    Assert.assertNotNull(decimalData);
    Assert.assertEquals(Field.Type.DECIMAL, decimalField.getType());
    Assert.assertEquals("5", decimalField.getAttribute(HeaderAttributeConstants.ATTR_PRECISION));
    Assert.assertEquals("2", decimalField.getAttribute(HeaderAttributeConstants.ATTR_SCALE));

    Assert.assertEquals(Field.Type.ZONED_DATETIME, record.get("/zonedDateTime").getType());
    Assert.assertTrue(record.get("/zonedDateTime").getValue() instanceof ZonedDateTime);
  } finally {
    runner.runDestroy();
  }
}
 

private SDCMetricsJson createSdcMetricJson(Record currentRecord) throws IOException {
  SDCMetricsJson sdcMetricsJson = new SDCMetricsJson();
  sdcMetricsJson.setSdcId(currentRecord.get("/" + AggregatorUtil.SDC_ID).getValueAsString());
  Field value =  currentRecord.get("/" + AggregatorUtil.MASTER_SDC_ID);
  if (value != null) {
    sdcMetricsJson.setMasterSdcId(value.getValueAsString());
  }
  sdcMetricsJson.setTimestamp(currentRecord.get("/" + AggregatorUtil.TIMESTAMP).getValueAsLong());
  sdcMetricsJson.setAggregated(currentRecord.get("/" + AggregatorUtil.IS_AGGREGATED).getValueAsBoolean());
  Map<String, Field> valueAsListMap = currentRecord.get("/" + AggregatorUtil.METADATA).getValueAsMap();
  if (valueAsListMap != null && !valueAsListMap.isEmpty()) {
    // Metadata is not available as of now, make it mandatory once available
    Map<String, String> metadata = new HashMap<>();
    for (Map.Entry<String, Field> e : valueAsListMap.entrySet()) {
      metadata.put(e.getKey(), e.getValue().getValueAsString());
    }
    metadata.put(DPM_PIPELINE_COMMIT_ID, pipelineCommitId);
    metadata.put(DPM_JOB_ID, jobId);

    Field timeSeriesAnalysisField =  currentRecord.get("/" + AggregatorUtil.TIME_SERIES_ANALYSIS);
    if (timeSeriesAnalysisField != null) {
      metadata.put(AggregatorUtil.TIME_SERIES_ANALYSIS, timeSeriesAnalysisField.getValueAsString());
    }
    // SDC's from 3.2 will have the last record field
    Field isLastRecord = currentRecord.get("/" + AggregatorUtil.LAST_RECORD);
    if (isLastRecord != null) {
      if (Boolean.valueOf(isLastRecord.getValueAsString())) {
        LOG.info(
            "Got last metric record from {} running job {}",
            sdcMetricsJson.getSdcId(),
            jobId
        );
      }
      metadata.put(AggregatorUtil.LAST_RECORD, isLastRecord.getValueAsString());
    }
    sdcMetricsJson.setMetadata(metadata);
  }
  String metricRegistryJson = currentRecord.get("/" + AggregatorUtil.METRIC_JSON_STRING).getValueAsString();
  sdcMetricsJson.setMetrics(ObjectMapperFactory.get().readValue(metricRegistryJson, MetricRegistryJson.class));
  return sdcMetricsJson;
}
 

private void checkFieldIssueContinue(
    Record record,
    HasherConfig hasherConfig,
    Set<String> expectedValidFields,
    Map<String, Field> expectedVal
) throws StageException {
  StageRunner.Output output;
  FieldHasherProcessor processor;
  ProcessorRunner runner;

  final Set<String> validFieldsFromTheProcessor = registerCallbackForValidFields();

  processor = PowerMockito.spy(
      new FieldHasherProcessor(
          hasherConfig,
          OnStagePreConditionFailure.CONTINUE
      )
  );

  runner = new ProcessorRunner.Builder(FieldHasherDProcessor.class, processor)
      .addOutputLane("a").build();
  runner.runInit();

  try {
    output = runner.runProcess(Arrays.asList(record));
    Assert.assertEquals("Valid Fields Size mismatch", expectedValidFields.size(), validFieldsFromTheProcessor.size());
    Assert.assertTrue("Expected Valid Fields Not Present", validFieldsFromTheProcessor.containsAll(expectedValidFields));
    Assert.assertEquals(1, output.getRecords().get("a").size());
    Record outputRecord = output.getRecords().get("a").get(0);

    Field field = outputRecord.get();
    Assert.assertTrue(field.getValue() instanceof Map);

    Map<String, Field> result = field.getValueAsMap();
    Assert.assertEquals("Expected fields does not match: ", expectedVal.size(), result.size());

    Set<String> resultFieldPaths = new HashSet<String>();
    for (Map.Entry<String, Field> entry : result.entrySet()) {
      String fieldKey = entry.getKey();
      Field outputField = entry.getValue();
      Field expectedField  = expectedVal.get(fieldKey);
      Assert.assertEquals("Expected Type not present for field:" + fieldKey, expectedField.getType(), outputField.getType());
      Assert.assertEquals("Expected Value not present for field: " + fieldKey, expectedField.getValue(), outputField.getValue());
      resultFieldPaths.add("/" + fieldKey);
    }
  } catch (StageException e) {
    Assert.fail("Should not throw an exception when On Stage Precondition Continue");
  } finally {
    runner.runDestroy();
  }
}