多線程導出

2000萬數據量導出200萬數據，10個字段，耗時16s.

@Configuration @Slf4j public class AsyncTaskPoolConfig {@Value("${async.executor.thread.core_pool_size}")private int corePoolSize;@Value("${async.executor.thread.max_pool_size}")private int maxPoolSize;@Value("${async.executor.thread.queue_capacity}")private int queueCapacity;@Value("${async.executor.thread.name.prefix}")private String namePrefix;@Bean(name = "taskExecutor")public Executor asyncServiceExecutor() {log.warn("start taskExecutor");//在這里修改ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();//配置核心線程數executor.setCorePoolSize(corePoolSize);//配置最大線程數executor.setMaxPoolSize(maxPoolSize);//配置隊列大小executor.setQueueCapacity(queueCapacity);//配置線程池中的線程的名稱前綴executor.setThreadNamePrefix(namePrefix);// rejection-policy：當pool已經達到max size的時候，如何處理新任務// CALLER_RUNS：不在新線程中執行任務，而是有調用者所在的線程來執行executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());//執行初始化executor.initialize();return executor;} }

配置

spring:# DataSource Configdatasource:druid:url: jdbc:mysql://127.0.0.1:3306/maruko?useUnicode=true&characterEncoding=utf-8&useSSL=falseusername: rootpassword: 123456initial-size: 5max-active: 10min-idle: 10max-wait: 20#配置間隔多久才進行一次檢測，檢測需要關閉的空閑連接，單位是毫秒time-between-eviction-runs-millis: 60000 # 異步線程配置 # 配置核心線程數 async:executor:thread:core_pool_size: 10# 配置最大線程數max_pool_size: 20# 配置隊列大小queue_capacity: 30# 配置線程池中的線程的名稱前綴name:prefix: async-excl- @GetMapping(value = "/exportEasyExcel2")public void exportEasyExcel2() {long start = System.currentTimeMillis();// 數據的總數int dataTotalCount = 2000000;int limit = 150000;// 計算出多少頁，即循環次數int count = dataTotalCount / limit + (dataTotalCount % limit > 0 ? 1 : 0);CountDownLatch countDownLatch = new CountDownLatch(count);for (int i = 1; i <= count; i++) {Map<String, Object> map = new HashMap<>(3);map.put("start", (i - 1) * limit);map.put("end", limit);map.put("path", "d:\\excel\\");iUserService.excuteAsyncTaskDatabase(map, countDownLatch);}try {countDownLatch.await(); //保證之前的所有的線程都執行完成，才會走下面的；// 這樣就可以在下面拿到所有線程執行完的集合結果} catch (Exception e) {log.error("阻塞異常:" + e.getMessage());} finally {long end = System.currentTimeMillis();log.info("excel任務執行完畢,共耗時： " + (end - start) + "ms");}} @Override@Async("taskExecutor")public void excuteAsyncTaskDatabase(Map<String, Object> map, CountDownLatch cdl) {try {long startTime = System.currentTimeMillis();List<String> title = Arrays.asList("主鍵ID", "用戶名稱", "郵箱", "手機號", "性別", "密碼", "年齡", "創建時間", "更新時間");List<String> key = Arrays.asList("id", "name", "email", "phone", "gender", "password", "age", "create_time", "update_time");List<Map<String, Object>> list = baseMapper.queryList(map);log.info("線程：" + Thread.currentThread().getName() + "讀取數據耗時 ：" + (System.currentTimeMillis() - startTime) + "ms,查詢到數據量為：" + list.size());String filePath = map.get("path").toString() + UUID.randomUUID() + ".xlsx";// 調用導出的文件方法Workbook workbook = SxssfWorkbookUtil.createWorkbook(title, key, list);//將workbook轉換為文件ExcelUtil.workbookToFile(workbook, filePath);long endTime = System.currentTimeMillis();log.info("線程：" + Thread.currentThread().getName() + "導出excel" + map.get("start") + ".xlsx成功 ,耗時 ：" + (endTime - startTime) + "ms");log.info("剩余任務數 ===========================> " + cdl.getCount());} catch (Exception e) {log.error("文件生成異常" + e.getMessage());} finally {// 執行完線程數減1cdl.countDown();}} /*** 大文件導出** @param title 列標題頭* @param key 字段英文* @param data 數據* @return*/public static SXSSFWorkbook createWorkbook(List<String> title, List<String> key, List<Map<String, Object>> data) {try {//創建poi導出數據對象SXSSFWorkbook sxssfWorkbook = new SXSSFWorkbook(100);//創建sheet頁SXSSFSheet sheet = sxssfWorkbook.createSheet();//設置表頭信息SXSSFRow headRow = sheet.createRow(0);//設置標題頭for (int j = 0; j < title.size(); j++) {headRow.createCell(j).setCellValue(title.get(j));}//遍歷數據for (int i = 0; i < data.size(); i++) {SXSSFRow dataRow = sheet.createRow(sheet.getLastRowNum() + 1);for (int j = 0; j < key.size(); j++) {dataRow.createCell(j).setCellValue(data.get(i).get(key.get(j)).toString());}}return sxssfWorkbook;} catch (Exception e) {log.error(e.getMessage());}return null;}

總結

以上是生活随笔為你收集整理的poi实现多线程大数据导出的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。