数据

xinnengyuan/ruoyi-admin/src/main/resources/application-dev.yml

@@ -60,6 +60,13 @@ spring:
           url: jdbc:mysql://192.168.110.2:13386/zmkgdev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
           username: zmkgdev
           password: JhYxREf25AXdy3h8
+        slave1:
+          lazy: true
+          type: ${spring.datasource.type}
+          driverClassName: com.mysql.cj.jdbc.Driver
+          url: jdbc:mysql://192.168.110.2:13386/zmkgprod?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
+          username: zmkgprod
+          password: MaY8nehwWkJriWPm
       #        oracle:
       #          type: ${spring.datasource.type}
       #          driverClassName: oracle.jdbc.OracleDriver

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUser.java

@@ -200,4 +200,8 @@ public class SubConstructionUser extends BaseEntity {
      * 首次入职时间
      */
     private LocalDate firstDate;
+
+    private Long goId;
+
+    private String goOpenid;
 }

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUserFile.java

@@ -57,4 +57,6 @@ public class SubConstructionUserFile implements Serializable {
      */
     private String remark;
 
+    private Long goId;
+
 }

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java

@@ -203,5 +203,9 @@ public class TransferDataController {
     }
 
 
+    @GetMapping("/userTransfer")
+    private void userTransfer() {
+        transferDataService.userTransfer();
+    }
 
 }

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/User.java

@@ -0,0 +1,174 @@
+package org.dromara.transferData.domain;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+import java.math.BigDecimal;
+import java.util.Date;
+
+/**
+ * 用户信息实体类
+ * 对应数据库表结构
+ */
+@Data
+public class User {
+
+    /**
+     * 主键ID
+     */
+    private Long id;
+
+    /**
+     * 微信id
+     */
+    private String openid;
+
+
+    /**
+     * 班组id
+     */
+    private Long teamId;
+
+    private String teamName;
+
+    /**
+     * 人脸照
+     */
+    private String pacePhoto;
+
+    /**
+     * 人员姓名
+     */
+    private String userName;
+
+    /**
+     * 项目id
+     */
+    private Long projectId;
+
+    /**
+     * 状态（0在职 1离职）-字典position_status
+     */
+    private String status = "0";
+
+
+    /**
+     * 创建者
+     */
+    private String createBy = "1";
+
+    /**
+     * 更新者
+     */
+    private String updateBy = "1";
+
+    /**
+     * 创建时间
+     */
+    private Date createdAt;
+
+    /**
+     * 更新时间
+     */
+    private Date updatedAt;
+
+    /**
+     * 电话
+     */
+    private String phone;
+
+    /**
+     * 1:男,2女,3保密
+     */
+    private String sex;
+
+    /**
+     * 身份证民族
+     */
+    private String sfzNation;
+
+    /**
+     * 身份证号码
+     */
+    private String sfzNumber;
+
+    /**
+     * 身份证有效开始期
+     */
+    private String sfzStart;
+
+    /**
+     * 身份证有效结束期
+     */
+    private String sfzEnd;
+
+    /**
+     * 身份证地址
+     */
+    private String sfzSite;
+
+    /**
+     * 身份证出生日期
+     */
+    private String sfzBirth;
+
+    /**
+     * 籍贯
+     */
+    private String nativePlace;
+
+    /**
+     * 银行卡号
+     */
+    private String yhkNumber;
+
+    /**
+     * 开户行
+     */
+    private String yhkOpeningBank;
+
+    /**
+     * 持卡人
+     */
+    private String yhkCardholder;
+
+    /**
+     * 工种（字典）
+     */
+    private String typeOfWork;
+
+    /**
+     * 打卡(1启用打卡 2禁止打卡)
+     */
+    private String clock = "2";
+
+    /**
+     * 劳务公司id
+     */
+    private Long labourserviceId;
+
+    /**
+     * 入场时间
+     */
+    private String entryDate;
+
+    /**
+     * 离场时间
+     */
+    private String leaveDate;
+
+    /**
+     * 薪水（此字段为0表示此字段无效）
+     */
+    private BigDecimal salary = new BigDecimal("0.00");
+
+    /**
+     * 是否开启项目备案（小程序）1开启 2不开启
+     */
+    private String projectRecord = "2";
+
+
+}
+

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java

@@ -16,6 +16,10 @@ public interface TransferDataMapper {
     @Select("select * from bus_attendance where  project_id in (59,60)")
     List<OldAttendance> getData();
 
+    @DS("slave1")
+    @Select("select * from bus_construction_user where project_id in (59,60) and deleted_at is null")
+    List<User> getUserList();
+
 
     @Select("select id,sys_user_id,project_id,user_name from sub_construction_user_copy1 where go_openid = #{openId}")
     ConstructionUserCopy getConstructionUserCopy(@Param("openId") String openId);
@@ -38,7 +42,8 @@ public interface TransferDataMapper {
     int saveBatchCopy(@Param("list") List<BusAttendance> busAttendanceList);
 
 
-    @Select("select id,go_id,sfz_front_pic,sfz_back_pic,yhk_pic,special_work_pic from sub_construction_user_copy1")
+    @Select("select id,go_id,sfz_front_pic,sfz_back_pic,yhk_pic,special_work_pic from sub_construction_user " +
+        "where go_id between 107 and 893 order by go_id")
     List<ConstructionUserNew> getConstructionUserList();
 
 

xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java

@@ -11,6 +11,7 @@ import jakarta.activation.MimetypesFileTypeMap;
 import jakarta.annotation.Resource;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.SerializationUtils;
+import org.dromara.common.core.enums.UserType;
 import org.dromara.common.core.service.OssService;
 import org.dromara.contractor.domain.SubConstructionUser;
 import org.dromara.contractor.domain.SubConstructionUserFile;
@@ -19,10 +20,14 @@ import org.dromara.contractor.service.ISubConstructionUserService;
 import org.dromara.project.domain.BusAttendance;
 import org.dromara.project.service.IBusAttendanceRuleService;
 import org.dromara.project.service.IBusAttendanceService;
+import org.dromara.system.domain.SysUser;
 import org.dromara.system.domain.vo.SysOssVo;
+import org.dromara.system.domain.vo.SysUserVo;
 import org.dromara.system.service.ISysOssService;
+import org.dromara.system.service.ISysUserService;
 import org.dromara.transferData.domain.ConstructionUserNew;
 import org.dromara.transferData.domain.OldFile;
+import org.dromara.transferData.domain.User;
 import org.dromara.transferData.domain.UserFile;
 import org.dromara.transferData.mapper.TransferDataMapper;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -36,12 +41,14 @@ import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
 import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
 import java.time.Duration;
-import java.util.ArrayList;
+import java.time.LocalDate;
-import java.util.List;
+import java.time.ZoneId;
-import java.util.Map;
+import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
+import java.util.concurrent.Semaphore;
 import java.util.stream.Collectors;
 
 import static kotlin.reflect.jvm.internal.impl.builtins.StandardNames.FqNames.list;
@@ -70,6 +77,8 @@ public class TransferDataService {
     private Executor attendanceAsyncExecutor;
     @Resource
     private TransferDataMapper transferDataMapper;
+    @Resource
+    private ISysUserService userService;
 
 
     // 两个候选基础URL
@@ -371,13 +380,13 @@ public class TransferDataService {
             Map<String, String> map = resultMap1.get(goId);
             if (map != null) {
                 handle0to3(map, constructionUser);
-                handle4to10(map,goId);
+                handle4to10(map, goId);
             }
 
             Map<String, String> map1 = resultMap.get(goId);
             if (map1 != null) {
                 handle0to3(map1, constructionUser);
-                handle4to10(map1,goId);
+                handle4to10(map1, goId);
             }
         }
 
@@ -414,11 +423,13 @@ public class TransferDataService {
                 constructionUser.setSpecialWorkPic(l.toString());
             }
         }
+
         SubConstructionUser constructionUser1 = BeanUtil.copyProperties(constructionUser, SubConstructionUser.class);
+        log.info("开始修改数据，id：{}", constructionUser1.getId());
         constructionUserService.updateById(constructionUser1);
     }
 
-    private void handle4to10(Map<String, String> map,Long goId){
+    private void handle4to10(Map<String, String> map, Long goId) {
         List<SubConstructionUserFile> list1 = constructionUserFileService.list(Wrappers.<SubConstructionUserFile>lambdaQuery()
             .eq(SubConstructionUserFile::getUserId, goId)
         );
@@ -438,6 +449,357 @@ public class TransferDataService {
                 }
             }
         }
+//        log.info("开始修改附件，数量：{}", list1.size());
         constructionUserFileService.updateBatchById(list1);
     }
+
+
+    public void userTransfer() {
+        List<User> userList = transferDataMapper.getUserList();
+        log.info("开始处理用户数据，数量：{}", userList.size());
+
+        // 设置每批处理的数据量
+        int batchSize = 300;
+        // 计算总批次数
+        int totalBatches = (int) Math.ceil((double) userList.size() / batchSize);
+
+        // 将数据分批
+        List<List<User>> batches = new ArrayList<>();
+        for (int i = 0; i < totalBatches; i++) {
+            int startIndex = i * batchSize;
+            int endIndex = Math.min((i + 1) * batchSize, userList.size());
+            batches.add(userList.subList(startIndex, endIndex));
+        }
+
+        // 使用并行流处理每批数据
+        batches.parallelStream().forEach(batch -> {
+            try {
+                log.info("开始处理批次，数据量：{}", batch.size());
+                handleUser(batch);
+                log.info("批次处理完成");
+            } catch (Exception e) {
+                log.error("处理批次时发生错误，数据量：{}", batch.size(), e);
+            }
+        });
+
+        log.info("所有用户数据处理完成，总批次数：{}", totalBatches);
+    }
+
+
+    public void handleUser(List<User> userList) {
+        //查询附件
+        List<Long> list1 = userList.stream().map(User::getId).toList();
+        List<OldFile> oldFileList = transferDataMapper.getOldFileList1(list1);
+
+        // 将 oldFileList1 转换为 Map<Long, Map<String, String>> 结构
+        Map<Long, Map<String, String>> oldFileMap = oldFileList.stream()
+            .collect(Collectors.groupingBy(
+                OldFile::getUserId,
+                Collectors.toMap(
+                    OldFile::getUserImgType,
+                    OldFile::getPath,
+                    (existing, replacement) -> replacement // 如果有重复的 key，保留后面的值
+                )
+            ));
+        ArrayList<SubConstructionUser> userAddList = new ArrayList<>();
+        List<SubConstructionUserFile> fileAddList =  new ArrayList<>();
+        for (User user : userList) {
+            String phone = user.getPhone();
+            if(StrUtil.isBlank(phone)){
+                continue;
+            }
+
+            //获取sysUserId
+            Long sysUserId;
+            SysUserVo sysUserVo = userService.selectUserByPhonenumber(phone);
+            if (sysUserVo == null) {
+                SysUser sysUser = new SysUser();
+                sysUser.setUserName(user.getPhone());
+                sysUser.setNickName(user.getUserName());
+                sysUser.setPhonenumber(phone);
+                sysUser.setSex(user.getSex());
+                sysUser.setUserType(UserType.APP_USER.getUserType());
+                sysUserId = userService.save(sysUser);
+            } else {
+                sysUserId = sysUserVo.getUserId();
+            }
+            Map<String, String> map = oldFileMap.get(user.getId());
+            //创建施工人员
+            SubConstructionUser constructUser = createConstructUser(user, sysUserId,map);
+            userAddList.add(constructUser);
+            //处理附件
+            List<SubConstructionUserFile> subConstructionUserFiles = handle4to10ConstructionUser(map, sysUserId, user.getId());
+            fileAddList.addAll(subConstructionUserFiles);
+        }
+        constructionUserService.saveBatch(userAddList);
+        constructionUserFileService.saveBatch(fileAddList);
+
+    }
+
+
+    public SubConstructionUser createConstructUser(User user, Long sysUserId,Map<String, String> fileMap) {
+        SubConstructionUser constructionUser = baseInfo(user, sysUserId);
+        //处理人脸照
+        Long facePicId = handleSinglePhotoLimit(constructionUser.getFacePic());
+        constructionUser.setFacePic(facePicId == null ? null : facePicId.toString());
+        //处理身份证照,银行卡，特殊工作证
+        handle0to3ConstructionUser(fileMap, constructionUser);
+        return constructionUser;
+    }
+
+    public SubConstructionUser baseInfo(User user, Long sysUserId) {
+        SubConstructionUser constructionUser = new SubConstructionUser();
+        constructionUser.setFacePic(user.getPacePhoto());
+        constructionUser.setSysUserId(sysUserId);
+        constructionUser.setUserName(user.getUserName());
+        constructionUser.setTeamName(user.getTeamName());
+        constructionUser.setStatus(user.getStatus());
+        constructionUser.setPhone(user.getPhone());
+        constructionUser.setSex(user.getSex());
+        constructionUser.setNation(user.getSfzNation());
+        constructionUser.setSfzStart(stringToLocalDate(user.getSfzStart(), "yyyy-MM-dd"));
+        constructionUser.setSfzEnd(stringToLocalDate(user.getSfzEnd(), "yyyy-MM-dd"));
+        constructionUser.setSfzSite(user.getSfzSite());
+        constructionUser.setSfzBirth(stringToLocalDate(user.getSfzBirth(), "yyyy-MM-dd"));
+        constructionUser.setNativePlace(user.getNativePlace());
+        constructionUser.setYhkNumber(user.getYhkNumber());
+        constructionUser.setYhkOpeningBank(user.getYhkOpeningBank());
+        constructionUser.setYhkCardholder(user.getYhkCardholder());
+        constructionUser.setTypeOfWork(user.getTypeOfWork());
+        constructionUser.setWageMeasureUnit(user.getSalary().toString());
+        constructionUser.setClock("0");
+        constructionUser.setEntryDate(stringToDate(user.getEntryDate(), "yyyy-MM-dd HH:mm:ss"));
+        constructionUser.setLeaveDate(stringToDate(user.getLeaveDate(), "yyyy-MM-dd HH:mm:ss"));
+        constructionUser.setSalary(user.getSalary());
+        constructionUser.setUserRole("1");
+        constructionUser.setExitStatus(user.getProjectRecord());
+        constructionUser.setRemark("GoLand");
+        constructionUser.setGoId(user.getId());
+        constructionUser.setGoOpenid(user.getOpenid());
+        constructionUser.setProjectId(projectIdAndTeamId(1, user.getProjectId().toString()));
+        constructionUser.setTeamId(projectIdAndTeamId(2, user.getTeamId().toString()));
+
+        constructionUser.setCreateBy(1L);
+        constructionUser.setUpdateBy(1L);
+        return constructionUser;
+    }
+
+    public Long projectIdAndTeamId(Integer types, String id) {
+        Long val = 0L; // 假设val是long类型
+
+        switch (types) {
+            case 1: // 项目
+                long targetID = switch (id) {
+                    case "60" -> 1897160897167638529L;
+                    case "59" -> 1897161054676336641L;
+                    default ->
+                        // 处理未匹配的情况，这里可以根据需要设置默认值
+                        0;
+                };
+                val = targetID==0?null:targetID;
+                break;
+
+            case 2: // 班组
+                long targetID2 = switch (id) {
+                    case "135" -> 1966464033343221761L; // 二工区六朋班组
+                    case "131" -> 1966464063785480193L; // 二工区平武
+                    case "130" -> 1966464107406241794L; // 四工区甫必班组
+                    case "125" -> 1966464157486231554L; // 四工区福绿班组
+                    case "124" -> 1966469101257846785L; // 送出线路班组
+                    case "120" -> 1966469224557801474L; // 五工区班组
+                    case "118" -> 1966469254266056706L; // 三工区班组
+                    case "117" -> 1966469284351799298L; // 二工区外苏班组
+                    case "116" -> 1966469322419302402L; // 一工区班组
+                    case "115" -> 1967121995904598018L; // 贵州兰亭电力建设(集团)有限公司（升压站场平）
+                    case "114" -> 1967122038313205761L; // 源梦诚标段三
+                    case "113" -> 1967122072953962498L; // 上海耀垦标段二
+                    default ->
+                        // 处理未匹配的情况
+                        0;
+                };
+                val = targetID2==0?null:targetID2;
+                break;
+
+            default:
+                // 处理其他types值的情况
+                break;
+        }
+        return val;
+    }
+
+    private LocalDate stringToLocalDate(String date, String format){
+        if (date == null || date.isEmpty()) {
+            return null;
+        }
+        if ("长期".equals(date)) {
+            return LocalDate.of(9999, 12, 31);
+        }
+        SimpleDateFormat sdf = new SimpleDateFormat(format);
+        try {
+            Date parsedDate = sdf.parse(date);
+            return parsedDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDate();
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    private Date stringToDate(String date, String format){
+        if (date == null || date.isEmpty()) {
+            return null;
+        }
+        if ("长期".equals(date)) {
+            return Date.from(LocalDate.of(9999, 12, 31).atStartOfDay(ZoneId.systemDefault()).toInstant());
+        }
+        SimpleDateFormat sdf = new SimpleDateFormat(format);
+        try {
+            return sdf.parse(date);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+
+    private void handle0to3ConstructionUser(Map<String, String> map, SubConstructionUser constructionUser) {
+        String s = map.get("0");
+        if (s != null) {
+            Long l = handleSinglePhotoLimit(s);
+            if (l != null) {
+                constructionUser.setSfzFrontPic(l.toString());
+            }
+        }
+
+        String s1 = map.get("1");
+        if (s1 != null) {
+            Long l = handleSinglePhotoLimit(s1);
+            if (l != null) {
+                constructionUser.setSfzBackPic(l.toString());
+            }
+        }
+        String s2 = map.get("2");
+        if (s2 != null) {
+            Long l = handleSinglePhotoLimit(s2);
+            if (l != null) {
+                constructionUser.setYhkPic(l.toString());
+            }
+        }
+        String s3 = map.get("3");
+        if (s3 != null) {
+            Long l = handleSinglePhotoLimit(s3);
+            if (l != null) {
+                constructionUser.setSpecialWorkPic(l.toString());
+            }
+        }
+    }
+
+    private static List<String> types = Arrays.asList("0", "1", "2", "3");
+    private List<SubConstructionUserFile> handle4to10ConstructionUser(Map<String, String> map,Long sysUserId,Long goId) {
+        List<SubConstructionUserFile> fileAddList =  new ArrayList<>();
+        for(String type : map.keySet()){
+            if(types.contains(type)){
+                continue;
+            }
+            SubConstructionUserFile userFile =  new SubConstructionUserFile();
+            userFile.setUserId(sysUserId);
+            userFile.setFileType(type);
+            userFile.setRemark("GoLand");
+            userFile.setGoId(goId);
+
+            String s = map.get(type);
+            if (s != null) {
+                String[] split = s.split(",");
+                List<Long> path = new ArrayList<>();
+                for (String s1 : split) {
+                    Long l = handleSinglePhotoLimit(s1);
+                    if (l != null) {
+                        path.add(l);
+                    }
+                }
+                if (CollectionUtil.isNotEmpty(path)) {
+                    userFile.setPath(path.stream().map(String::valueOf).collect(Collectors.joining(",")));
+                }
+            }
+            fileAddList.add(userFile);
+        }
+        return fileAddList;
+    }
+
+
+    // 添加信号量控制并发数，避免过多线程竞争资源
+    private final Semaphore semaphore = new Semaphore(20); // 限制最多20个线程同时执行
+
+    private Long handleSinglePhotoLimit(String facePicRelativePath) {
+        if (facePicRelativePath == null || facePicRelativePath.isEmpty()) {
+            return null;
+        }
+
+        try {
+            // 获取信号量许可，控制并发数
+            semaphore.acquire();
+
+            // 1. 规范化路径（原逻辑不变）
+            String normalizedPath = facePicRelativePath.replace("\\", "/");
+            String filename = Paths.get(normalizedPath).getFileName().toString();
+
+            // 2. 解析文件类型（优化：优先从HTTP响应头获取，其次用文件名）
+            String contentType = "application/octet-stream"; // 默认二进制类型
+
+            // 3. 多URL重试（增加重试次数和超时时间）
+            for (int retry = 0; retry < 3; retry++) { // 增加重试次数到3次
+                for (String baseUrl : BASE_URLS) {
+                    String fullUrl = baseUrl + normalizedPath;
+                    try {
+                        // 构建带超时的HTTP请求
+                        HttpRequest request = HttpRequest.newBuilder()
+                            .uri(URI.create(fullUrl))
+                            .GET()
+                            .timeout(Duration.ofSeconds(30)) // 增加超时时间到30秒
+                            .build();
+
+                        // 发送请求并处理响应
+                        HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
+                        if (response.statusCode() == 200) {
+                            try (InputStream inputStream = response.body()) {
+                                // 从响应头获取真实Content-Type（比文件名解析更准确）
+                                contentType = response.headers().firstValue("Content-Type").orElse(contentType);
+                                long contentLength = response.headers().firstValueAsLong("Content-Length").orElse(-1);
+
+                                // 上传OSS（此时流已通过try-with-resources自动关闭）
+                                SysOssVo ossVo = ossService.upload(inputStream, filename, contentType, contentLength);
+                                return ossVo.getOssId(); // 成功则跳出所有循环
+                            }
+                        }
+                    } catch (Exception e) {
+                        log.warn("尝试URL失败（重试{}次）: {}", retry + 1, fullUrl, e);
+                        // 在重试前等待一段时间
+                        if (retry < 2) { // 不是最后一次重试才等待
+                            try {
+                                Thread.sleep(500 * (retry + 1)); // 递增等待时间
+                            } catch (InterruptedException ie) {
+                                Thread.currentThread().interrupt();
+                                break;
+                            }
+                        }
+                        continue; // 重试下一个URL
+                    }
+                }
+            }
+
+            // 所有URL和重试都失败
+            log.error("照片处理失败，所有URL重试完毕，relativePath={}", normalizedPath);
+            return null;
+
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            log.error("获取信号量被中断，relativePath={}", facePicRelativePath, e);
+            return null;
+        } catch (Exception e) {
+            log.error("处理照片时发生未知错误，relativePath={}", facePicRelativePath, e);
+            return null;
+        } finally {
+            // 释放信号量许可
+            semaphore.release();
+        }
+    }
 }