org.springframework.batch.item.ItemProcessor Java Examples

@Bean
@Qualifier("compositeItemProcessorr")
public ItemProcessor<Document,Document> compositeItemProcessor() {
    CompositeItemProcessor processor = new CompositeItemProcessor<>();
    ArrayList<ItemProcessor<Document,Document>> delegates = new ArrayList<>();

    if(tikaItemProcessor !=null) delegates.add(tikaItemProcessor);
    if(pdfBoxItemProcessor !=null) delegates.add(pdfBoxItemProcessor);
    if(metadataItemProcessor !=null) delegates.add(metadataItemProcessor);
    if(dBLineFixerItemProcessor !=null) delegates.add(dBLineFixerItemProcessor);
    if(gateItemProcessor !=null) delegates.add(gateItemProcessor);
    if(deIdDocumentItemProcessor !=null) delegates.add(deIdDocumentItemProcessor);
    if(webserviceDocumentItemProcessor !=null) delegates.add(webserviceDocumentItemProcessor);
    if(pdfGenerationItemProcessor !=null) delegates.add(pdfGenerationItemProcessor);
    if(thumbnailGenerationItemProcessor !=null) delegates.add(thumbnailGenerationItemProcessor);


    delegates.add(jsonMakerItemProcessor);
    processor.setDelegates(delegates);
    return processor;
}
 

private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            String validateResult = ValidatorUtil.validate(item);
            if(StringUtils.isNotEmpty(validateResult)){
                throw new SeedException("数据校验未通过-->" + validateResult);
            }
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            item.setRealName(item.getRealName() + "--00");
            item.setAge(item.getAge() + 10);
            item.setId(null);
            item.setUpdateTime(null);
            item.setCreateTime(new Date());
            LogUtil.getLogger().info("转换后==>{}", ReflectionToStringBuilder.toString(item));
            return item;
        }
    };
}
 

@Bean(name = STEP_PROCESS_CSV_FILE)
public Step readCsvFileAndPopulateDbTable(
        StepBuilderFactory stepBuilderFactory,
        PlatformTransactionManager platformTransactionManager,
        @Qualifier(DATA_READER) ItemReader<JavaChampion> itemReader,
        @Qualifier(DATA_PROCESSOR) ItemProcessor<JavaChampion, JavaChampion> itemProcessor,
        @Qualifier(DATA_WRITER) ItemWriter<JavaChampion> itemWriter) {

    StepBuilder builder = stepBuilderFactory.get(STEP_PROCESS_CSV_FILE);

    return builder.<JavaChampion, JavaChampion>chunk(10)
            .reader(itemReader)
            .processor(itemProcessor)
            .writer(itemWriter)
            .transactionManager(platformTransactionManager)
            .build();
}
 

/**
     * step步骤，包含ItemReader，ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "vtollStep1")
    public Step vtollStep1(StepBuilderFactory stepBuilderFactory,
                           @Qualifier("vtollReader") ItemReader<BudgetVtoll> reader,
                           @Qualifier("vtollWriter") ItemWriter<BudgetVtoll> writer,
                           @Qualifier("vtollProcessor") ItemProcessor<BudgetVtoll, BudgetVtoll> processor) {
        return stepBuilderFactory
                .get("vtollStep1")
                .<BudgetVtoll, BudgetVtoll>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行，一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 

/**
     * step步骤，包含ItemReader，ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "cantonStep1")
    public Step cantonStep1(StepBuilderFactory stepBuilderFactory,
                           @Qualifier("cantonReader") ItemReader<Canton> reader,
                           @Qualifier("cantonWriter") ItemWriter<Canton> writer,
                           @Qualifier("cantonProcessor") ItemProcessor<Canton, Canton> processor) {
        return stepBuilderFactory
                .get("cantonStep1")
                .<Canton, Canton>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行，一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 

/**
     * step步骤，包含ItemReader，ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "logStep1")
    public Step logStep1(StepBuilderFactory stepBuilderFactory,
                         @Qualifier("logReader") ItemReader<Log> reader,
                         @Qualifier("logWriter") ItemWriter<Log> writer,
                         @Qualifier("logProcessor") ItemProcessor<Log, Log> processor) {
        return stepBuilderFactory
                .get("logStep1")
                .<Log, Log>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行，一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 

/**
     * step步骤，包含ItemReader，ItemProcessor和ItemWriter
     *
     * @param stepBuilderFactory
     * @param reader
     * @param writer
     * @param processor
     * @return
     */
    @Bean(name = "zappStep1")
    public Step zappStep1(StepBuilderFactory stepBuilderFactory,
                          @Qualifier("appReader") ItemReader<App> reader,
                          @Qualifier("appWriter") ItemWriter<App> writer,
                          @Qualifier("appProcessor") ItemProcessor<App, App> processor) {
        return stepBuilderFactory
                .get("zappStep1")
                .<App, App>chunk(5000)//批处理每次提交5000条数据
                .reader(reader)//给step绑定reader
                .processor(processor)//给step绑定processor
                .writer(writer)//给step绑定writer
                .faultTolerant()
                .retry(Exception.class)   // 重试
                .noRetry(ParseException.class)
                .retryLimit(1)           //每条记录重试一次
                .skip(Exception.class)
                .skipLimit(200)         //一共允许跳过200次异常
//                .taskExecutor(new SimpleAsyncTaskExecutor()) //设置每个Job通过并发方式执行，一般来讲一个Job就让它串行完成的好
//                .throttleLimit(10)        //并发任务数为 10,默认为4
                .build();
    }
 

private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            String validateResult = ValidatorUtil.validate(item);
            if(StringUtils.isNotEmpty(validateResult)){
                throw new SeedException("数据校验未通过-->" + validateResult);
            }
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            item.setRealName(item.getRealName() + "--11");
            item.setAge(item.getAge() - 5);
            LogUtil.getLogger().info("转换后==>{}", ReflectionToStringBuilder.toString(item));
            return item;
        }
    };
}
 

private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            if(item.getRealName().contains("玄玉")){
                throw new SeedException("这是测试断点续跑的异常");
            }
            try {
                TimeUnit.SECONDS.sleep(4);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            return item;
        }
    };
}
 

@Bean
public ItemProcessor<Long, Address> addressPrinterProcessor() {
    return new ItemProcessor<Long, Address>() {

        @Autowired
        private BatchTaskAddressDao batchTaskAddressDAO;

        @Autowired
        private AddressDao addressDAO;

        @Override
        public Address process(final Long id) throws Exception {
            this.batchTaskAddressDAO.updateProcessingState(id, ProcessingState.PRINTED);
            return this.addressDAO.getAddressById(id);
        }
    };
}
 

@Bean
public Step step2() {
	return this.stepBuilderFactory.get("step2").chunk(3).faultTolerant()
			.skip(IllegalStateException.class).skipLimit(100)
			.reader(new SkipItemReader())
			.processor(new ItemProcessor<Object, Object>() {
				@Override
				public String process(Object item) throws Exception {
					return String.valueOf(Integer.parseInt((String) item) * -1);
				}
			}).writer(new SkipItemWriter()).build();
}
 

private ItemProcessor<Person, Person> processor(){
    return new ItemProcessor<Person, Person>() {
        @Override
        public Person process(Person item) {
            LogUtil.getLogger().info("读取到-->{}", ReflectionToStringBuilder.toString(item));
            try {
                TimeUnit.SECONDS.sleep(6);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            return item;
        }
    };
}
 

@Bean
@Qualifier("compositeSlaveStep")
public Step compositeSlaveStep(
                    ItemReader<Document> reader,
        @Qualifier("compositeItemProcessor") ItemProcessor<Document, Document> processor,
        @Qualifier("compositeESandJdbcItemWriter") ItemWriter<Document> writer,
        @Qualifier("slaveTaskExecutor")TaskExecutor taskExecutor,
        @Qualifier("nonFatalExceptionItemProcessorListener")
                            ItemProcessListener nonFatalExceptionItemProcessorListener,
        //@Qualifier("targetDatasourceTransactionManager")PlatformTransactionManager manager,
        StepBuilderFactory stepBuilderFactory
) {
    FaultTolerantStepBuilder stepBuilder = stepBuilderFactory.get("compositeSlaveStep")
            .<Document, Document> chunk(chunkSize)
            .reader(reader)
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipLimit(skipLimit)
            .skip(WebserviceProcessingFailedException.class);
    if (env.acceptsProfiles("jdbc_out_map")) {
      stepBuilder = stepBuilder.skip(InvalidDataAccessApiUsageException.class);
    }
    return stepBuilder.noSkip(Exception.class)
     //       .listener(nonFatalExceptionItemProcessorListener)
            .listener(new SkipListener())
            .taskExecutor(taskExecutor)
            .build();
}
 

@Bean
Job job(JobBuilderFactory jobBuilderFactory,
 StepBuilderFactory stepBuilderFactory, JdbcTemplate template,
 ItemReader<Contact> fileReader,
 ItemProcessor<Contact, Contact> emailProcessor,
 ItemWriter<Contact> jdbcWriter) {

 Step setup = stepBuilderFactory.get("clean-contact-table")
  .tasklet((contribution, chunkContext) -> {
   template.update("delete from CONTACT");
   return RepeatStatus.FINISHED;
  }).build();

 Step fileToJdbc = stepBuilderFactory.get("file-to-jdbc-fileToJdbc")
  .<Contact, Contact>chunk(5)
  // <1>
  .reader(fileReader).processor(emailProcessor).writer(jdbcWriter)
  .faultTolerant().skip(InvalidEmailException.class)
  // <2>
  .skipPolicy((Throwable t, int skipCount) -> {
   LogFactory.getLog(getClass()).info("skipping ");
   return t.getClass().isAssignableFrom(InvalidEmailException.class);
  }).retry(HttpStatusCodeException.class) // <3>
  .retryLimit(2).build();

 return jobBuilderFactory.get("etl") // <4>
  .start(setup).next(fileToJdbc).build();
}
 

@Bean
ItemProcessor<Contact, Contact> validatingProcessor(
 EmailValidationService emailValidationService) {
 return item -> {
  boolean valid = emailValidationService.isEmailValid(item.getEmail());
  item.setValidEmail(valid);
  if (!valid)
   throw new InvalidEmailException(item.getEmail());
  return item;
 };
}
 

@Bean
public Step addressMigrationStep(final StepBuilderFactory stepBuilderFactory,
                                 final ItemStreamReader<BatchTaskAddress> addressReader,
                                 final ItemProcessor<BatchTaskAddress, BatchTaskAddress> addressMigrationProcessor,
                                 final ItemWriter<BatchTaskAddress> addressWriter) throws Exception {
    return stepBuilderFactory.get("addressMigrationStep")
            .<BatchTaskAddress, BatchTaskAddress>chunk(1)
            .reader(addressReader)
            .processor(addressMigrationProcessor)
            .writer(addressWriter)
            .build();
}
 

@Bean
public Step addressPrinterStep(final StepBuilderFactory stepBuilderFactory,
                               final ItemStreamReader<Long> addressPrinterReader,
                               final ItemProcessor<Long, Address> addressPrinterProcessor,
                               final ItemWriter<Address> addressPrinterWriter) throws Exception {
    return stepBuilderFactory
            .get("addressPrinterStep")
            .<Long, Address>chunk(1)
            .reader(addressPrinterReader)
            .processor(addressPrinterProcessor)
            .writer(addressPrinterWriter)
            .allowStartIfComplete(Boolean.TRUE)
            .build();
}
 

@Bean
public Step skipPolicyStep(@Qualifier("skippingItemProcessor") ItemProcessor<Transaction, Transaction> processor,
                           ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("skipPolicyStep")
            .<Transaction, Transaction>chunk(10)
            .reader(itemReader(invalidInputCsv))
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipPolicy(new CustomSkipPolicy())
            .build();
}
 

@Bean
protected Step processLines(ItemReader<Line> reader, ItemProcessor<Line, Line> processor, ItemWriter<Line> writer) {
    return steps.get("processLines").<Line, Line> chunk(2)
      .reader(reader)
      .processor(processor)
      .writer(writer)
      .build();
}
 

@Bean
public Step retryStep(@Qualifier("retryItemProcessor") ItemProcessor<Transaction, Transaction> processor,
  ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory.get("retryStep")
      .<Transaction, Transaction>chunk(10)
      .reader(itemReader(inputCsv))
      .processor(processor)
      .writer(writer)
      .faultTolerant()
      .retryLimit(3)
      .retry(ConnectTimeoutException.class)
      .retry(DeadlockLoserDataAccessException.class)
      .build();
}
 

@Bean
protected Step step1(@Qualifier("itemProcessor") ItemProcessor<Transaction, Transaction> processor, ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("step1")
            .<Transaction, Transaction> chunk(10)
            .reader(itemReader(inputCsv))
            .processor(processor)
            .writer(writer)
            .build();
}
 

@Bean
public Step skippingStep(@Qualifier("skippingItemProcessor") ItemProcessor<Transaction, Transaction> processor,
                         ItemWriter<Transaction> writer) throws ParseException {
    return stepBuilderFactory
            .get("skippingStep")
            .<Transaction, Transaction>chunk(10)
            .reader(itemReader(invalidInputCsv))
            .processor(processor)
            .writer(writer)
            .faultTolerant()
            .skipLimit(2)
            .skip(MissingUsernameException.class)
            .skip(NegativeAmountException.class)
            .build();
}
 

@Bean
public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<Student> reader, ItemWriter<Student> writer,
        ItemProcessor<Student,Student> processor) {
    return stepBuilderFactory
            .get("step1")
            .<Student, Student>chunk(65000) //批处理每次提交 65000 条数据
            .reader(reader) // 给 step 绑定 reader
            .processor(processor) // 给 step 绑定 processor
            .writer(writer) // 给 step 绑定 writer
            .build();
}
 

@Bean @StepScope
ItemProcessor personProcessor(
        @Qualifier("personNameCaseChange") ItemProcessor personNameCaseChange,
        @Value("#{jobParameters['" + LAST_NAME_PREFIX + "']}") String lastNamePrefix) {
    CompositeItemProcessor p = new CompositeItemProcessor();
    p.setDelegates(newArrayList(
            personNameFilter(Optional.ofNullable(lastNamePrefix).orElseGet(() -> environment.getProperty(LAST_NAME_PREFIX))),
            personNameCaseChange));
    return p;
}
 

@Bean
public Step step1(ItemReader<CustomerCredit> itemReader,
		ItemProcessor<CustomerCredit, CustomerCredit> itemProcessor,
		ItemWriter<CustomerCredit> itemWriter) {
	return this.stepBuilderFactory.get("step1")
			.<CustomerCredit, CustomerCredit>chunk(2)
			.reader(itemReader)
			.processor(itemProcessor)
			.writer(itemWriter)
			.build();
}
 

@Bean(name = DATA_PROCESSOR)
public ItemProcessor<JavaChampion, JavaChampion> processor() {
    return objectFromRow -> {
        log.info("Java Champion found: " + objectFromRow);
        return objectFromRow;
    };
}
 

@Bean
Step personStep(@Qualifier("personProcessor") ItemProcessor personProcessor) {
    return steps.get("personStep")
            .allowStartIfComplete(true)
            .<Person, Person>chunk(3)
            .reader(personReader())
            .processor(personProcessor)
            .writer(personWriter())
            .build();
}
 

@Bean @StepScope
ItemProcessor personProcessor(
        @Qualifier("personNameCaseChange") ItemProcessor personNameCaseChange,
        @Value("#{jobParameters['" + LAST_NAME_PREFIX + "']}") String lastNamePrefix) {
    CompositeItemProcessor p = new CompositeItemProcessor();
    p.setDelegates(newArrayList(
            personNameFilter(Optional.ofNullable(lastNamePrefix).orElseGet(() -> environment.getProperty(LAST_NAME_PREFIX))),
            personNameCaseChange));
    return p;
}
 

@Bean @StepScope
ItemProcessor personNameCaseChange(@Value("#{jobParameters['upperCase']}") Boolean upperCaseParam) {
    boolean upperCase = upperCaseParam == null ? false : upperCaseParam;
    log.info("personNameCaseChange(upperCase={})", upperCase);
    return new FunctionItemProcessor<Person, Person>(p -> new Person(
            upperCase ? p.firstName.toUpperCase() : p.firstName.toLowerCase(),
            upperCase ? p.lastName.toUpperCase() : p.lastName.toLowerCase()));
}
 

@Bean
Step personStep(@Qualifier("personProcessor") ItemProcessor personProcessor) {
    return steps.get("personStep")
            .allowStartIfComplete(true)
            .<Person, Person>chunk(3)
            .reader(personReader())
            .processor(personProcessor)
            .writer(personWriter())
            .build();
}