From f706e73c30f849496f050e27ee3708aa7184469f Mon Sep 17 00:00:00 2001 From: zt Date: Thu, 18 Sep 2025 18:27:29 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=BD=E5=B7=A5=E4=BA=BA=E5=91=98=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/application-prod.yml | 7 + .../controller/TransferDataController.java | 6 +- .../domain/ConstructionUserNew.java | 20 +++ .../dromara/transferData/domain/OldFile.java | 21 +++ .../dromara/transferData/domain/UserFile.java | 39 +++++ .../mapper/TransferDataMapper.java | 42 ++++- .../service/TransferDataService.java | 160 ++++++++++++++++-- 7 files changed, 271 insertions(+), 24 deletions(-) create mode 100644 xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/ConstructionUserNew.java create mode 100644 xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/OldFile.java create mode 100644 xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/UserFile.java diff --git a/xinnengyuan/ruoyi-admin/src/main/resources/application-prod.yml b/xinnengyuan/ruoyi-admin/src/main/resources/application-prod.yml index 71431c26..1623c947 100644 --- a/xinnengyuan/ruoyi-admin/src/main/resources/application-prod.yml +++ b/xinnengyuan/ruoyi-admin/src/main/resources/application-prod.yml @@ -63,6 +63,13 @@ spring: url: jdbc:mysql://192.168.110.2:13386/zmkgc?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true username: zmkgc password: nWKDKRNRT48tFBdh + slave1: + lazy: true + type: ${spring.datasource.type} + driverClassName: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://192.168.110.2:13386/zmkgprod?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true + username: zmkgprod + password: MaY8nehwWkJriWPm # # 从库数据源 # slave: # lazy: true diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java index 04fb35d0..426133e3 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java @@ -197,9 +197,9 @@ public class TransferDataController { } - @GetMapping("/transferSpecialWorkPic") - private void handleFaceImage() { - transferDataService.handleSpecialWorkPic(); + @GetMapping("/handleFile") + private void handleFile() { + transferDataService.handleFile(); } diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/ConstructionUserNew.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/ConstructionUserNew.java new file mode 100644 index 00000000..95cfc358 --- /dev/null +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/ConstructionUserNew.java @@ -0,0 +1,20 @@ +package org.dromara.transferData.domain; + +import lombok.Data; + +@Data +public class ConstructionUserNew { + private Long id; + + private Long sysUserId; + + private Long goId; + + private String yhkPic; + + private String sfzFrontPic; + + private String sfzBackPic; + + private String specialWorkPic; +} diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/OldFile.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/OldFile.java new file mode 100644 index 00000000..5f64933b --- /dev/null +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/OldFile.java @@ -0,0 +1,21 @@ +package org.dromara.transferData.domain; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class OldFile { + + private Long id; + + private Long userId; + + private String userImgType; + + private String name; + + private String path; + +} diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/UserFile.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/UserFile.java new file mode 100644 index 00000000..fbe3f728 --- /dev/null +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/UserFile.java @@ -0,0 +1,39 @@ +package org.dromara.transferData.domain; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; + +/** + * 施工人员文件存储对象 sub_construction_user_file + * + * @author lilemy + * @date 2025-04-01 + */ +@Data +public class UserFile { + + + /** + * 用户id + */ + private Long userId; + + /** + * 文件类型 + */ + private String fileType; + + /** + * 文件路径 + */ + private String path; + + + private Long goId; + +} diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java index 18b1e373..82b0c40a 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java @@ -1,13 +1,9 @@ package org.dromara.transferData.mapper; import com.baomidou.dynamic.datasource.annotation.DS; -import org.apache.ibatis.annotations.Insert; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; -import org.apache.ibatis.annotations.Select; +import org.apache.ibatis.annotations.*; import org.dromara.project.domain.BusAttendance; -import org.dromara.transferData.domain.ConstructionUserCopy; -import org.dromara.transferData.domain.OldAttendance; +import org.dromara.transferData.domain.*; import java.util.List; @@ -40,4 +36,38 @@ public interface TransferDataMapper { "" + "") int saveBatchCopy(@Param("list") List busAttendanceList); + + + @Select("select id,go_id,sfz_front_pic,sfz_back_pic,yhk_pic,special_work_pic from sub_construction_user_copy1") + List getConstructionUserList(); + + + @DS("slave1") + @Select("") + List getOldFileList1(List userIds); + + @DS("slave") + @Select("") + List getOldFileList(List userIds); + + @Select("") + List getUserFileList(List goIds); } diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java index da66374c..1b8cb69e 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java @@ -1,5 +1,11 @@ package org.dromara.transferData.service; +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.conditions.query.Query; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.toolkit.Wrappers; import jakarta.activation.MimetypesFileTypeMap; import jakarta.annotation.Resource; @@ -15,6 +21,10 @@ import org.dromara.project.service.IBusAttendanceRuleService; import org.dromara.project.service.IBusAttendanceService; import org.dromara.system.domain.vo.SysOssVo; import org.dromara.system.service.ISysOssService; +import org.dromara.transferData.domain.ConstructionUserNew; +import org.dromara.transferData.domain.OldFile; +import org.dromara.transferData.domain.UserFile; +import org.dromara.transferData.mapper.TransferDataMapper; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; @@ -29,10 +39,13 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.stream.Collectors; +import static kotlin.reflect.jvm.internal.impl.builtins.StandardNames.FqNames.list; + /** * @Author 铁憨憨 * @Date 2025/9/14 10:53 @@ -55,6 +68,8 @@ public class TransferDataService { @Resource @Qualifier("attendanceAsyncPool") // 对应之前 AsyncConfig 中定义的线程池Bean名 private Executor attendanceAsyncExecutor; + @Resource + private TransferDataMapper transferDataMapper; // 两个候选基础URL @@ -62,8 +77,10 @@ public class TransferDataService { "http://xny.yj-3d.com:7464", "http://xny.yj-3d.com:7363" }; + /** * 异步处理照片并批量保存（修复线程安全 + 性能优化） + * * @param batchList 原始批量数据（浅拷贝） */ // 2. 修复后的照片批量异步处理逻辑 @@ -164,7 +181,7 @@ public class TransferDataService { try (InputStream inputStream = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()).body()) { // 从响应头获取真实Content-Type（比文件名解析更准确） HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); - if(response.statusCode() == 200){ + if (response.statusCode() == 200) { contentType = response.headers().firstValue("Content-Type").orElse(contentType); long contentLength = response.headers().firstValueAsLong("Content-Length").orElse(-1); @@ -197,21 +214,21 @@ public class TransferDataService { //获取到施工人员信息，然后处理图片 List list1 = constructionUserService.list(); for (SubConstructionUser subConstructionUser : list1) { - if(subConstructionUser.getFacePic()!=null){ + if (subConstructionUser.getFacePic() != null) { Long l = handleSinglePhoto1(subConstructionUser.getFacePic()); - subConstructionUser.setFacePic(l==null?"":l.toString()); + subConstructionUser.setFacePic(l == null ? "" : l.toString()); } - if(subConstructionUser.getSfzFrontPic()!=null){ + if (subConstructionUser.getSfzFrontPic() != null) { Long l = handleSinglePhoto1(subConstructionUser.getSfzFrontPic()); - subConstructionUser.setSfzFrontPic(l==null?"":l.toString()); + subConstructionUser.setSfzFrontPic(l == null ? "" : l.toString()); } - if(subConstructionUser.getSfzBackPic()!=null){ + if (subConstructionUser.getSfzBackPic() != null) { Long l = handleSinglePhoto1(subConstructionUser.getSfzBackPic()); - subConstructionUser.setSfzBackPic(l==null?"":l.toString()); + subConstructionUser.setSfzBackPic(l == null ? "" : l.toString()); } - if(subConstructionUser.getYhkPic()!=null){ + if (subConstructionUser.getYhkPic() != null) { Long l = handleSinglePhoto1(subConstructionUser.getYhkPic()); - subConstructionUser.setYhkPic(l==null?"":l.toString()); + subConstructionUser.setYhkPic(l == null ? "" : l.toString()); } objects.add(subConstructionUser); } @@ -230,8 +247,8 @@ public class TransferDataService { String[] split = path.split(","); for (String s : split) { Long l = handleSinglePhoto1(s); - if(l!=null){ - subConstructionUser.setPath(subConstructionUser.getPath().replace(s,l.toString())); + if (l != null) { + subConstructionUser.setPath(subConstructionUser.getPath().replace(s, l.toString())); } } } @@ -243,7 +260,6 @@ public class TransferDataService { } - private Long handleSinglePhoto1(String facePicRelativePath) { if (facePicRelativePath == null || facePicRelativePath.isEmpty()) { return null; @@ -272,7 +288,7 @@ public class TransferDataService { try (InputStream inputStream = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()).body()) { // 从响应头获取真实Content-Type（比文件名解析更准确） HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); - if(response.statusCode() == 200){ + if (response.statusCode() == 200) { contentType = response.headers().firstValue("Content-Type").orElse(contentType); long contentLength = response.headers().firstValueAsLong("Content-Length").orElse(-1); @@ -291,7 +307,7 @@ public class TransferDataService { // 所有URL和重试都失败 log.error("照片处理失败，所有URL重试完毕，relativePath={}", normalizedPath); - return null; + return null; } @@ -301,13 +317,127 @@ public class TransferDataService { for (SubConstructionUser constructionUser : list) { String facePicRelativePath = constructionUser.getSpecialWorkPic(); Long l = handleSinglePhoto1(facePicRelativePath); - constructionUser.setSpecialWorkPic(l==null?"":l.toString()); + constructionUser.setSpecialWorkPic(l == null ? "" : l.toString()); } constructionUserService.updateBatchById(list); } + public void handleFile() { + List constructionUserList = transferDataMapper.getConstructionUserList(); + log.info("开始处理特殊图片，一共{}条数据", constructionUserList.size()); + List list1 = constructionUserList.stream().map(ConstructionUserNew::getGoId).toList(); + //zmkgprod + List oldFileList1 = transferDataMapper.getOldFileList1(list1); + // 将 oldFileList1 转换为 Map> 结构 + Map> resultMap1 = oldFileList1.stream() + .collect(Collectors.groupingBy( + OldFile::getUserId, + Collectors.toMap( + OldFile::getUserImgType, + OldFile::getPath, + (existing, replacement) -> replacement // 如果有重复的 key，保留后面的值 + ) + )); + //zmkgc + List oldFileList = transferDataMapper.getOldFileList(list1); + + // 将 oldFileList1 转换为 Map> 结构 + Map> resultMap = oldFileList.stream() + .collect(Collectors.groupingBy( + OldFile::getUserId, + Collectors.toMap( + OldFile::getUserImgType, + OldFile::getPath, + (existing, replacement) -> replacement // 如果有重复的 key，保留后面的值 + ) + )); + //本地文件 + + //身份证（正面）,0 + //身份证（反面）,1 + //银行卡,2 + //特种作业证,3 + //合同,4 + //体检报告,5 + //安全责任书,6 + //岗位危险告知书,7 + //安全技术交底,8 + //三级安全教育,10 + + for (ConstructionUserNew constructionUser : constructionUserList) { + Long goId = constructionUser.getGoId(); + Map map = resultMap1.get(goId); + if (map != null) { + handle0to3(map, constructionUser); + handle4to10(map,goId); + } + + Map map1 = resultMap.get(goId); + if (map1 != null) { + handle0to3(map1, constructionUser); + handle4to10(map1,goId); + } + } + } + + private void handle0to3(Map map, ConstructionUserNew constructionUser) { + String s = map.get("0"); + if (s != null) { + Long l = handleSinglePhoto1(s); + if (l != null) { + constructionUser.setSfzFrontPic(l.toString()); + } + } + + String s1 = map.get("1"); + if (s1 != null) { + Long l = handleSinglePhoto1(s1); + if (l != null) { + constructionUser.setSfzBackPic(l.toString()); + } + } + String s2 = map.get("2"); + if (s2 != null) { + Long l = handleSinglePhoto1(s2); + if (l != null) { + constructionUser.setYhkPic(l.toString()); + } + } + String s3 = map.get("3"); + if (s3 != null) { + Long l = handleSinglePhoto1(s3); + if (l != null) { + constructionUser.setSpecialWorkPic(l.toString()); + } + } + SubConstructionUser constructionUser1 = BeanUtil.copyProperties(constructionUser, SubConstructionUser.class); + constructionUserService.updateById(constructionUser1); + } + + private void handle4to10(Map map,Long goId){ + List list1 = constructionUserFileService.list(Wrappers.lambdaQuery() + .eq(SubConstructionUserFile::getUserId, goId) + ); + for (SubConstructionUserFile subConstructionUserFile : list1) { + String s = map.get(subConstructionUserFile.getFileType()); + if (s != null) { + String[] split = s.split(","); + List path = new ArrayList<>(); + for (String s1 : split) { + Long l = handleSinglePhoto1(s1); + if (l != null) { + path.add(l); + } + } + if (CollectionUtil.isNotEmpty(path)) { + subConstructionUserFile.setPath(path.stream().map(String::valueOf).collect(Collectors.joining(","))); + } + } + } + constructionUserFileService.updateBatchById(list1); + } }