Skip to content

D
dsk-operate-sys-cscec
Commit c5f54a1c authored by lcl's avatar lcl
Browse files
Options

数据解析批量插入优化

parent be0554b4
Hide whitespace changes
Inline Side-by-side
Showing with 86 additions and 63 deletions
dsk-module/dsk-biz-api/src/main/java/com/dsk/component/DataAnalysisComponent.java
View file @ c5f54a1c
......@@ -54,71 +54,69 @@ public class DataAnalysisComponent {
if (ObjectUtils.isEmpty(fileList)) return;
//文件处理
for (CbProjectFile file : fileList) {
// if (file.getDelFlag() == 0) {
//文件下载
InputStream inputStream = ossService.downFileIO(file.getFileOssId());
if (ObjectUtil.isNull(inputStream)) {
file.setFileParseStatus(3);
file.setFailRemark("文件数据不存在");
projectFileService.updateById(file);
break;
}
//解析数据
List<CbQuantitySummary> importList = new ExcelUtils<>(CbQuantitySummary.class).importExcelAllSheet(inputStream, 1);
if (importList.isEmpty()) {
//文件下载
InputStream inputStream = ossService.downFileIO(file.getFileOssId());
if (ObjectUtil.isNull(inputStream)) {
file.setFileParseStatus(3);
file.setFailRemark("文件数据不存在");
projectFileService.updateById(file);
break;
}
//解析数据
List<CbQuantitySummary> importList = new ExcelUtils<>(CbQuantitySummary.class).importExcelAllSheet(inputStream, 1);
if (importList.isEmpty()) {
file.setFileParseStatus(3);
file.setFailRemark("表格中不存在待导入数据!");
projectFileService.updateById(file);
break;
}
List<CbQuantitySummary> quantitySummaryList = importList.stream().parallel()
.filter(item -> !ObjectUtils.isEmpty(item.getCbName()))
.peek(item -> {
item.setProjectId(bo.getProjectId());
item.setCbStage(bo.getCbStage());
item.setCbProjectFileId(file.getId());
}).collect(Collectors.toList());
if (quantitySummaryList.isEmpty()) {
throw new ServiceException("表格中不存在有效数据!");
}
transactionTemplate.execute(status -> {
try {
// //分批次插入
// if (quantitySummaryList.size() > 1000) {
// int index = 0;
// int sum = quantitySummaryList.size();
// while (index < sum) {
// List<CbQuantitySummary> divideList = quantitySummaryList.subList(index, Math.min(index + 1000, sum));
// boolean b = quantitySummaryService.saveBatch(divideList);
// if (!b) {
// throw new ServiceException("数据插入失败！");
// }
// index += 1000;
// }
// } else {
// boolean b = quantitySummaryService.saveBatch(quantitySummaryList);
// if (!b) {
// throw new ServiceException("数据插入失败！");
// }
// }
boolean a = quantitySummaryService.batchInsert(quantitySummaryList);
if (!a) {
throw new ServiceException("数据插入失败！");
}
file.setFileParseStatus(2);
boolean b = projectFileService.updateById(file);
if (!b) {
throw new ServiceException("文件状态更新失败！");
}
} catch (Exception e) {
status.setRollbackOnly();
file.setFileParseStatus(3);
file.setFailRemark("表格中不存在待导入数据!");
file.setFailRemark(e.getMessage());
projectFileService.updateById(file);
break;
}
List<CbQuantitySummary> quantitySummaryList = importList.stream().parallel()
.filter(item -> !ObjectUtils.isEmpty(item.getCbName()))
.peek(item -> {
item.setProjectId(bo.getProjectId());
item.setCbStage(bo.getCbStage());
item.setCbProjectFileId(file.getId());
}).collect(Collectors.toList());
if (quantitySummaryList.isEmpty()) {
throw new ServiceException("表格中不存在有效数据!");
}
transactionTemplate.execute(status -> {
try {
//分批次插入
if (quantitySummaryList.size() > 1000) {
int index = 0;
int sum = quantitySummaryList.size();
while (index < sum) {
List<CbQuantitySummary> divideList = quantitySummaryList.subList(index, Math.min(index + 1000, sum));
boolean b = quantitySummaryService.saveBatch(divideList);
if (!b) {
throw new ServiceException("数据插入失败！");
}
index += 1000;
}
} else {
boolean b = quantitySummaryService.saveBatch(quantitySummaryList);
if (!b) {
throw new ServiceException("数据插入失败！");
}
}
file.setFileParseStatus(2);
boolean b = projectFileService.updateById(file);
if (!b) {
throw new ServiceException("文件状态更新失败！");
}
} catch (Exception e) {
status.setRollbackOnly();
file.setFileParseStatus(3);
file.setFailRemark(e.getMessage());
projectFileService.updateById(file);
}
return Boolean.TRUE;
});
// } else {
// quantitySummaryService.remove(Wrappers.<CbQuantitySummary>lambdaQuery().eq(CbQuantitySummary::getCbProjectFileId, file.getId()));
// projectFileService.removeById(file);
// }
return Boolean.TRUE;
});
}
}
}
dsk-module/dsk-biz-api/src/main/java/com/dsk/cscec/domain/CbQuantitySummary.java
View file @ c5f54a1c
......@@ -11,6 +11,7 @@ import java.io.Serializable;
import com.dsk.common.annotation.Excel;
import lombok.Data;
import org.springframework.util.ObjectUtils;
/**
* 成本-工料汇总基本表(CbQuantitySummary)表实体类
......@@ -146,7 +147,7 @@ public class CbQuantitySummary implements Serializable {
private String num;
public void setNum(String num) {
this.number = Integer.valueOf(num);
this.number = ObjectUtils.isEmpty(num) ? null : Integer.valueOf(num);
}
}
dsk-module/dsk-biz-api/src/main/java/com/dsk/cscec/mapper/CbQuantitySummaryMapper.java
View file @ c5f54a1c
......@@ -20,6 +20,7 @@ import java.util.Map;
*/
public interface CbQuantitySummaryMapper extends BaseMapperPlus<CbQuantitySummaryMapper,CbQuantitySummary,CbQuantitySummary> {
int batchInsert(@Param("list") List<CbQuantitySummary> list);
List<Map<String, Object>> selectSubject(CbProjectBaseBo bo);
......@@ -29,5 +30,7 @@ public interface CbQuantitySummaryMapper extends BaseMapperPlus<CbQuantitySummar
List<CbQuantitySummaryListVo> selectListBySubject(CbQuantitySummaryListBo bo);
}
dsk-module/dsk-biz-api/src/main/java/com/dsk/cscec/service/ICbQuantitySummaryService.java
View file @ c5f54a1c
......@@ -28,5 +28,7 @@ public interface ICbQuantitySummaryService extends IService<CbQuantitySummary> {
void pushData(CbQuantitySummaryActual bo);
boolean batchInsert(List<CbQuantitySummary> list);
}
dsk-module/dsk-biz-api/src/main/java/com/dsk/cscec/service/impl/CbQuantitySummaryServiceImpl.java
View file @ c5f54a1c
......@@ -139,5 +139,11 @@ public class CbQuantitySummaryServiceImpl extends ServiceImpl<CbQuantitySummaryM
//TODO 推送数据
}
@Override
public boolean batchInsert(List<CbQuantitySummary> list) {
int i = baseMapper.batchInsert(list);
return i == list.size();
}
}
dsk-module/dsk-biz-api/src/main/resources/mapper/cscec/CbQuantitySummaryMapper.xml
View file @ c5f54a1c
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.dsk.cscec.mapper.CbQuantitySummaryMapper">
<insert id="batchInsert" parameterType="com.dsk.cscec.domain.CbQuantitySummary">
INSERT INTO cscec_bms.cb_quantity_summary
(project_id, `number`, cb_stage, cb_subject_name, company_no, org_no, cb_name, job_content,
calculation_rule, unit, material_description, guide_price, bid_unit_price, unit_price_difference, quantity,
combined_price, combined_price_tax, brand_name, bid_source, remark, cb_project_file_id)
VALUES
<foreach collection="list" item="item" separator="," >
(#{item.projectId}, #{item.number}, #{item.cbStage}, #{item.cbSubjectName}, #{item.companyNo}, #{item.orgNo}, #{item.cbName},
#{item.jobContent},#{item.calculationRule},#{item.unit},#{item.materialDescription},#{item.guidePrice},
#{item.bidUnitPrice},#{item.unitPriceDifference},#{item.quantity},#{item.combinedPrice},#{item.combinedPriceTax},
#{item.brandName},#{item.bidSource},#{item.remark},#{item.cbProjectFileId})
</foreach>
</insert>
<select id="selectSubject" resultType="java.util.Map">
select
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment