使用Spring batch从S3读取大文件的最佳方法

<dependency>
    <groupId>software.amazon.awssdk</groupId>
    <artifactId>s3</artifactId>
    <version>2.17.85</version>
</dependency>

@Configuration
@EnableBatchProcessing
public class S3FileProcessingJobConfig {

    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;

    @Bean
    public ItemReader<String> s3FileReader(AmazonS3 amazonS3, String bucketName, String key) {
        S3Object s3Object = amazonS3.getObject(bucketName, key);
        S3ObjectInputStream inputStream = s3Object.getObjectContent();
        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));

        return new IteratorItemReader<>(reader.lines().iterator());
    }

    @Bean
    public ItemProcessor<String, String> s3FileProcessor() {
        // 处理文件内容的逻辑
        return item -> item.toUpperCase();
    }

    @Bean
    public ItemWriter<String> s3FileWriter() {
        // 将处理后的内容写入目标位置的逻辑
        return items -> {
            for (String item : items) {
                System.out.println(item);
            }
        };
    }

    @Bean
    public Step s3FileProcessingStep(ItemReader<String> reader, ItemProcessor<String, String> processor, ItemWriter<String> writer) {
        return stepBuilderFactory.get("s3FileProcessingStep")
                .<String, String>chunk(100)
                .reader(reader)
                .processor(processor)
                .writer(writer)
                .build();
    }

    @Bean
    public Job s3FileProcessingJob(Step s3FileProcessingStep) {
        return jobBuilderFactory.get("s3FileProcessingJob")
                .start(s3FileProcessingStep)
                .build();
    }
}