Export大数据量导出和打包

{
    "className": "ValideData",         //导出的数据的实体类，类中有别名和顺序相关的注解
    "createUser": "",				//操作人
    "downLoadNo": "202203181504732568468066304",  	//下载流水号
    "fileName": "机卡绑定",          //文件名      fileName+HHmmssSSS.xlsx
    "keys": [						//redis key的数据，分批获取数据
    ],
    "remark": "机卡绑定",				//备注（不关注）
    "type": "机卡绑定"					//导出类型（不关注）
}

<dependency>
  <groupId>org.apache.commons</groupId>
  <artifactId>commons-compress</artifactId>
  <version>1.21</version>
</dependency>

public void bigDataExport(PortDto dto) throws Exception {
    long start = System.currentTimeMillis();
    log.info("开始导出，批次号：<{}>, 开始时间：{}", dto.getDownLoadNo(), DateUtil.now());

    //修改导出记录
    LambdaUpdateWrapper<PortDto> updateWrapper = new LambdaUpdateWrapper<>();
    updateWrapper.eq(PortDto::getDownLoadNo, dto.getDownLoadNo());
    //生成导出记录
    int row = this.baseMapper.update(dto, updateWrapper);
    if (row > 0) {
        log.info("批次号：<{}>准备生成文件", dto.getDownLoadNo());
        try {
            Iterator<String> iterator = keys.iterator();
            Workbook workbook = null;
            ExportParams params = new ExportParams();

            //加载导出数据实体类
            Class<?> aClass = Class.forName(entityBasePackage + dto.getClassName());

            int element = 0;
            while (iterator.hasNext()) {
                String key = iterator.next();
                Collection<?> list = getList(key, aClass);
                element += list.size();
                workbook = ExcelExportUtil.exportBigExcel(params, aClass, list);

                //文件数据达到阈值
                if (element >= maxDataCount) {
                    String fileName = dto.getFileName() + "_" + DateUtil.format(new Date(),
                            "HHmmssSSS") + ".xlsx";
                    ExcelExportUtil.closeExportBigExcel();
                    FileOutputStream fos =
                            new FileOutputStream(fileProp.getPath().getPath() + fileName);
                    workbook.write(fos);
                    fos.close();
                    element = 0;
                    //更新地址
                    Map<String, Object> map = new HashMap<>();
                    map.put("downloadNo", dto.getDownLoadNo());
                    map.put("filePath", fileProp.getPath().getPath() + fileName);
                    map.put("createTime", new Date());
                    this.baseMapper.insertPathRecord(map);
                    log.info("文件写入完成，文件名：{}", fileName);
                    continue;
                }
                iterator.remove();
            }

            //写入剩余文件
            if (element != 0) {
                String fileName = dto.getFileName() + "_" + DateUtil.format(new Date(),
                        "HHmmssSSS") + ".xlsx";
                ExcelExportUtil.closeExportBigExcel();
                FileOutputStream fos = new FileOutputStream(fileProp.getPath().getPath() + fileName);
                workbook.write(fos);
                fos.close();
                element = 0;
                //更新地址
                Map<String, Object> map = new HashMap<>();
                map.put("downloadNo", dto.getDownLoadNo());
                map.put("filePath", fileProp.getPath().getPath() + fileName);
                map.put("createTime", new Date());
                this.baseMapper.insertPathRecord(map);
                log.info("文件写入完成，文件名：{}", fileName);
            }

            long end = System.currentTimeMillis();
            log.info("导出结束，批次号：<{}>, 结束时间：{}， 耗时：{}", dto.getDownLoadNo(), DateTime.of(end),
                    DateUtil.formatBetween(end - start));
        } catch (Exception e) {
            log.info("批次号<{}>导出异常：", dto.getDownLoadNo(), e);
            throw new BusinessException("");
        } finally {
            log.info("批次号<{}>生成文件结束，准备压缩文件，修改状态", dto.getDownLoadNo());
            //合并文件到导出文件记录主表
            //当只有一个文件记录时直接更新主表文件地址
            List<PortDto> recordList = exportDao.getPathRecord(dto);
            if (recordList.size() > 1) {
                //zipPath
                dto.setFilePath(zcat(dto, recordList));
            } else {
                //xlsxPath
                dto.setFilePath(recordList.size()==0? "":recordList.get(0).getFilePath());
            }
            updateWrapper.clear();
            updateWrapper.set(PortDto::getFilePath, dto.getFilePath());
            updateWrapper.set(PortDto::getSuccessTime, new Date());
            updateWrapper.set(PortDto::getStatus, "1");
            updateWrapper.eq(PortDto::getDownLoadNo, dto.getDownLoadNo());
            this.baseMapper.update(null, updateWrapper);
            log.info("批次号<{}>更新下载记录表文件地址，修改状态成功", dto.getDownLoadNo());
        }
    }
}

<dependency>
	<groupId>cn.hutool</groupId>
	<artifactId>hutool-all</artifactId>
	<version>5.7.3</version>
</dependency>

/**
 *  多文件压缩
 * @param dto 导出信息
 * @Param recordList 文件路径
 * @return void
 * @throws
 * @author Surpass
 * @date 2022/3/17 9:59
 */
private String zcat(PortDto dto, List<PortDto> recordList) throws Exception {
    //临时文件
        List<String> tempFileList = new ArrayList<>();
        try {
            String destFilePath = fileProp.getPath().getPath() + IdUtil.fastSimpleUUID() + ".temp";
            tempFileList.add(destFilePath);
            File destFile = new File(destFilePath);
            Archiver archiver = CompressUtil.createArchiver(
                    CharsetUtil.CHARSET_UTF_8,
                    ArchiveStreamFactory.ZIP,
                    destFile
            );
            for (PortDto portDto : recordList) {
                String ext = portDto.getFilePath().substring(portDto.getFilePath().lastIndexOf("."));
                String tempFilePath = fileProp.getPath().getPath() + IdUtil.fastSimpleUUID() + ext;
                tempFileList.add(tempFilePath);
                File file1 = HttpUtil.downloadFileFromUrl(fastBaseUrl + portDto.getFilePath(), tempFilePath);
                archiver.add(file1);
            }
            archiver.finish();
            archiver.close();

            //上传
            InputStream is = new FileInputStream(destFile);
            return fastDFSClient.uploadFile(is, is.available(), ArchiveStreamFactory.ZIP);
        } finally {
            log.info("压缩包上传完成，删除临时文件");
            tempFileList.forEach(FileUtil::del);
        }
}

/**
 *   查询redis数据
 * @param key
 * @param cls
 * @return java.util.Collection<?>
 * @throws
 * @author Surpass
 * @date 2022/3/18 15:51
 */
private Collection<?> getList(String key, Class<?> cls) {
    List<String> list = redis.getList(key);
    return list.stream().map(item -> JSONObject.parseObject(item, cls)).collect(Collectors.toList());
}