Spring batch app does not process all items

Viewed 87

I have a spring batch app which is weird that I dont know why does not process all items ,I am using range partition and compositeprocessor for the data transform. If my reader reads 5787 records for example, it is an example because they can be more, it only processes 5704 recors and the rest remain unprocessed. I hope someone can help me , thanks in advance.

My dataitemprocessor

public class data implements ItemProcessor<beangenerico,ThreadLocal<List<beanAccountCollect>>> {

    Logger logger = Logger.getLogger(data.class);

    private String SP_SQL = "{call GetDetailAccount(?)}";
    private String SELECT = "{call myspbyblocks (?,?)}";
    private beanAccountCollect b;
    private ThreadLocal<List<beanAccountCollect>> listbeanAccC = new ThreadLocal<List<beanAccountCollect>>();
    private ThreadLocal<List<beanCustomer>> listbeanc=new ThreadLocal<List<beanCustomer>>();

    @Autowired
    private  JdbcTemplate jdbcTemplate;

    @Override
    public ThreadLocal<List<beanAccountCollect>> process(beangenerico rangos)  {
        // TODO Auto-generated method stub

        listbeanAccC.set(new ArrayList<beanAccountCollect>());

        try {
        listbeanc = this.jdbcTemplate.query(SELECT,new Object [] {rangos.getIni(),rangos.getFin()},new CustomerResultSetExtractor());

            for(beanCustomer bc : listbeanc.get())  {

                b = new beanAccountCollect();
                b.setUsernetwork(bc.getUsernetwork());
                b.setTipoagente(bc.getTipoagente());
                b.setLbpar(this.jdbcTemplate.query(SP_SQL,new Object [] {bc.getCuenta()},new BeanAccountResulSetExtractor(this.jdbcTemplate)));

                listbeanAccC.get().add(b);
            }

        }catch (Exception e) {
            logger.error(e);
        }

        return listbeanAccC;
    }

    public void setJdbcTemplate(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
    }
}

This is my transform data

public class transformDataWS implements ItemProcessor<ThreadLocal<List<beanAccountCollect>>, ThreadLocal<List<beanNewMessageBeanP>>>
{

    Logger logger = Logger.getLogger(transformDataWS.class);

    private ThreadLocal<List<beanNewMessageBeanP>> lstbnmbp = new ThreadLocal<List<beanNewMessageBeanP>>();

    private beanNewMessageBeanP bnmbp;

    public ThreadLocal<List<beanNewMessageBeanP>> process(ThreadLocal<List<beanAccountCollect>> list) throws Exception {    

            // TODO Auto-generated method stub
        lstbnmbp.set(new ArrayList<beanNewMessageBeanP>());
        List<beanParameter> lbeanPar=null;
        List<NMessagePEntryBeanParray> lNMPEBa = null;

        for(beanAccountCollect bc:list.get()) {

            NewMessageParametrosEntryBeanP nb = new NewMessageParametrosEntryBeanP();
            NMessagePEntryBeanParray bar= null;
            NewMessageParametrosEntryBeanP [] ba = null;
            lNMPEBa = new ArrayList<NMessagePEntryBeanParray>();
            lbeanPar = new ArrayList<beanParameter>();
            lbeanPar = bc.getLbpar();
            bnmbp = new beanNewMessageBeanP();

            bnmbp.setTipoagente(bc.getTipoagente());
            bnmbp.setUsernetwork(bc.getUsernetwork());


            if(lbeanPar!=null) {

                for(beanParameter bpar : lbeanPar) {

                    ba = new NewMessageParametrosEntryBeanP[54];
                    bar = new NMessagePEntryBeanParray();


                    ba[0] = new NewMessageParametrosEntryBeanP();
                    ba[0].setKey("PaymentQ");
                    ba[0].setValue(bpar.getPQ());

                    ba[1] = new NewMessageParametrosEntryBeanP();
                    ba[1].setKey("PaymentReest");
                    ba[1].setValue(bpar.getPR());

                    ba[2] = new NewMessageParametrosEntryBeanP();
                    ba[2].setKey("DelayCte");
                    ba[2].setValue(bpar.getDelayCte());

                    ba[3] = new NewMessageParametrosEntryBeanP();
                    ba[3].setKey("DelayRange");ba[3].setValue(bpar.getDelayR());

                    ba[4] = new NewMessageParametrosEntryBeanP();
                    ba[4].setKey("C4");ba[4].setValue(bpar.getC3());

                    ba[5] = new NewMessageParametrosEntryBeanP();
                    ba[5].setKey("C6");ba[5].setValue(bpar.getC6());

                    ba[6] = new NewMessageParametrosEntryBeanP();
                    ba[6].setKey("Banddict");ba[6].setValue(bpar.getBanddict());

                    ba[7] = new NewMessageParametrosEntryBeanP();
                    ba[7].setKey("Street");ba[7].setValue(bpar.getStreet());

                    ba[8] = new NewMessageParametrosEntryBeanP();
                    ba[8].setKey("Stree_1");ba[8].setValue(bpar.getStreet1());

                    //....

                    ba[53] = new NewMessageParametrosEntryBeanP();
                    ba[53].setKey("Zone");ba[53].setValue(bpar.getZone());

                    bar.setArr(ba);
                    lNMPEBa.add(bar);
                }
            }
            bnmbp.setNmespebarr(lNMPEBa);

            lstbnmbp.get().add(bnmbp);
        }
         return lstbnmbp;
    }
}

This is my config job

@EnableBatchProcessing
@Configuration
@Import({DBConfiguration.class})
@ComponentScan({"com.mycompany.batch.config","com.mycompany.batch.mapper","com.mycompany.batch.model","com.mycompany.batch.particion","com.mycompan.batch.procesos","com.mycompany.batch.reader","com.mycompany.batch.writers"})
@PropertySource("file:pruebas.properties")
public class ConfigJobBatch {

    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;

    @Autowired
    @Qualifier("sqlserverDataSource")
    private DataSource dataSource;

    @Autowired
    Environment envws;


    @Bean(name = "demoPartitionStep")
    public Step step1Manager(Step slaveStep) {
        return stepBuilderFactory.get("step1.manager")
            .<String, String>partitioner("step1", demoPartitioner())
            .step(slaveStep)
            .gridSize(numerohilos())
            .taskExecutor(taskExecutor())
            .build();
    }

    @Bean(name = "demoPartitioner", destroyMethod = "")
    public Partitioner demoPartitioner() {
        RangePartitioner partitioner = new RangePartitioner();
        return partitioner;
    }

    // slave step
    @Bean 
    public Step slaveStep(ItemReader<beangenerico> demoReader,ItemWriter BeanAccCollectionWriter)
    {
        return stepBuilderFactory.get("slaveStep")
                .chunk(1)
                .reader(demoReader)
                .processor(compositeProcessor())
                .writer(BeanAccCollectionWriter)
                .taskExecutor(taskExecutor())
                .build();
    }

    @Bean
    public CompositeItemProcessor compositeProcessor() {

        List<ItemProcessor> delegates = new ArrayList<>(2);
        delegates.add(CustomerProccesor());
        delegates.add(beanDataItemProccesor());
        CompositeItemProcessor processor = new CompositeItemProcessor();

        processor.setDelegates(delegates);
        return processor;
    }

    /***FIXME debemos instanciar los processor como spring bean sino el spring no lee y no toma en cuenta la capa dao o service***/
    @Bean
    public CustomerItemProcessor CustomerProccesor(){
        return new CustomerItemProcessor();
    }

    @Bean
    public beanDataItemProccesor beanDataItemProccesor(){
        return new beanDataItemProccesor();
    }

    /***FIXME debemos instanciar los processor como bean sino no toma en cuenta la capa dao o service***/
    @Bean
    public CustomItemProcessListener listener() {
        return new CustomItemProcessListener();
    }

    @Bean(name = "demoWriter")
    @StepScope
    public ItemWriter< beangenerico> CustomItemWriter() {
        // TODO Auto-generated method stub
        CustomItemWriter wri = new CustomItemWriter();
        return wri;
    }

    @Bean(name = "testWriter")
    @StepScope
    public ItemWriter<ThreadLocal<CopyOnWriteArrayList<beangen>>> testItemWriter() {
        // TODO Auto-generated method stub
        TestWriter wri = new TestWriter();
        return wri;
    }

    @Bean(name = "BeanAccCollectionWriter")
    @StepScope
    public ItemWriter<ThreadLocal<List<NewMessage>>> BeanAccItemWriter() {  
        // TODO Auto-generated method stub
        BeanAccItemWriter wri = new BeanAccItemWriter();
        return wri;
    }

    @Bean(name="flatFileItemWriterPartition")
    @StepScope
    public FlatFileItemWriter<beangen> slaveWriter(
            @Value("#{stepExecutionContext[fromId]}") int fromId,@Value("#{stepExecutionContext[toId]}")int toId ) {
      FlatFileItemWriter<beangen> reader = new FlatFileItemWriter<beangen>();
      reader.setResource(new FileSystemResource(
          "csv/users.processed" + fromId + "-" + toId + ".csv"));
      //reader.setAppendAllowed(false);
      reader.setLineAggregator(new DelimitedLineAggregator<beangen>() {{
        setDelimiter(",");
        setFieldExtractor(new BeanWrapperFieldExtractor<beangen>() {{
          setNames(new String[]{"usernetwork","cuenta","atributo","atributo2"});
        }});
      }});
      return reader;
    }

    @Bean(name="tempRecordsWriter")
    @StepScope
    public ListDelegateWriter ListDelegateWriter(@Qualifier("flatFileItemWriterPartition")FlatFileItemWriter<beangen> writer) {
        // TODO Auto-generated method stub
        ListDelegateWriter wri = new ListDelegateWriter();
        wri.setDelegate(writer);

        return wri;
    }

    @Bean(name = "demoReader")
    @StepScope
    public ItemReader<beangenerico> myreader(@Value("#{stepExecutionContext['fromId']}") int minValue,@Value("#{stepExecutionContext['toId']}") int maxValue){
        Myreader fr = new Myreader(minValue,maxValue);
        return fr;
    }

    @Bean
    public TaskExecutor taskExecutor() {
        return new SimpleAsyncTaskExecutor("spring_batch");
    }

    @Bean
    public Job job(@Qualifier("demoPartitionStep") Step demoPartitionStep) {
        return this.jobBuilderFactory.get("job")
                .start(demoPartitionStep)
                .build();
    }

    @Bean
    public StepExecuListner steplistener() {
        return new StepExecuListner();
    }

    public static int numerohilos() {
        /****ciclo para hilos usando rango y numero de hilos a calcular***
          **************N_threads = N_cpu * U_cpu * (1 + W / C) *************************************
          ***N_cpu = Runtime.getRuntime().availableProcessors()**
          *******/
        int numcpu = Runtime.getRuntime().availableProcessors();
        int numthread = numcpu*1*(1+10);
        int gridSize=numthread;
        return gridSize;
    }
}
1 Answers

I fix my code, this helped .

public class data implements ItemProcessor<beangenerico,ThreadLocal<List<beanAccountCollect>>> {

    Logger logger = Logger.getLogger(data.class);
    
    private String SP_SQL = "{call GetDetailAccount(?)}";
    private String SELECT = "{call myspbyblocks (?,?)}";
    private beanAccountCollect b;
    private ThreadLocal<List<beanAccountCollect>> listbeanAccC = new ThreadLocal<List<beanAccountCollect>>();
    private ThreadLocal<List<beanCustomer>> listbeanc=new ThreadLocal<List<beanCustomer>>();
    
    @Autowired
    private  JdbcTemplate jdbcTemplate;
    
    @Override
    public ThreadLocal<List<beanAccountCollect>> process(beangenerico rangos)  {
        // TODO Auto-generated method stub

        List<beanParameter> lbeanPar=null;
        listbeanAccC.set(new ArrayList<beanAccountCollect>());
        
        try {
        listbeanc = this.jdbcTemplate.query(SELECT,new Object [] {rangos.getIni(),rangos.getFin()},new CustomerResultSetExtractor());
        
            for(beanCustomer bc : listbeanc.get())  {

                b = new beanAccountCollect();
                lbeanPar = new ArrayList<beanParameter>();
                
                lbeanPar = this.jdbcTemplate.query(SP_SQL,new Object [] {bc.getCuenta()},new BeanAccountResulSetExtractor(this.jdbcTemplate));
                
                b.setUsernetwork(bc.getUsernetwork());
                b.setTipoagente(bc.getTipoagente());
                b.setLbpar(lbeanPar);
                                                        
                listbeanAccC.get().add(b);
            }

        }catch (Exception e) {
            logger.error(e);
        }

        return listbeanAccC;
    }

    public void setJdbcTemplate(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
    }
}
Related