diff --git a/xinnengyuan/ruoyi-admin/src/main/resources/application-dev.yml b/xinnengyuan/ruoyi-admin/src/main/resources/application-dev.yml index b9dbd7ab..285d71fa 100644 --- a/xinnengyuan/ruoyi-admin/src/main/resources/application-dev.yml +++ b/xinnengyuan/ruoyi-admin/src/main/resources/application-dev.yml @@ -60,6 +60,13 @@ spring: url: jdbc:mysql://192.168.110.2:13386/zmkgdev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true username: zmkgdev password: JhYxREf25AXdy3h8 + slave1: + lazy: true + type: ${spring.datasource.type} + driverClassName: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://192.168.110.2:13386/zmkgprod?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true + username: zmkgprod + password: MaY8nehwWkJriWPm # oracle: # type: ${spring.datasource.type} # driverClassName: oracle.jdbc.OracleDriver diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUser.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUser.java index 2048866f..84883881 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUser.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUser.java @@ -200,4 +200,8 @@ public class SubConstructionUser extends BaseEntity { * 首次入职时间 */ private LocalDate firstDate; + + private Long goId; + + private String goOpenid; } diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUserFile.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUserFile.java index f6f920d0..3c56ce1a 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUserFile.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/contractor/domain/SubConstructionUserFile.java @@ -57,4 +57,6 @@ public class SubConstructionUserFile implements Serializable { */ private String remark; + private Long goId; + } diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java index 426133e3..3b65c8a4 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/controller/TransferDataController.java @@ -203,5 +203,9 @@ public class TransferDataController { } + @GetMapping("/userTransfer") + private void userTransfer() { + transferDataService.userTransfer(); + } } diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/User.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/User.java new file mode 100644 index 00000000..52972eec --- /dev/null +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/domain/User.java @@ -0,0 +1,174 @@ +package org.dromara.transferData.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import java.math.BigDecimal; +import java.util.Date; + +/** + * 用户信息实体类 + * 对应数据库表结构 + */ +@Data +public class User { + + /** + * 主键ID + */ + private Long id; + + /** + * 微信id + */ + private String openid; + + + /** + * 班组id + */ + private Long teamId; + + private String teamName; + + /** + * 人脸照 + */ + private String pacePhoto; + + /** + * 人员姓名 + */ + private String userName; + + /** + * 项目id + */ + private Long projectId; + + /** + * 状态（0在职 1离职）-字典position_status + */ + private String status = "0"; + + + /** + * 创建者 + */ + private String createBy = "1"; + + /** + * 更新者 + */ + private String updateBy = "1"; + + /** + * 创建时间 + */ + private Date createdAt; + + /** + * 更新时间 + */ + private Date updatedAt; + + /** + * 电话 + */ + private String phone; + + /** + * 1:男,2女,3保密 + */ + private String sex; + + /** + * 身份证民族 + */ + private String sfzNation; + + /** + * 身份证号码 + */ + private String sfzNumber; + + /** + * 身份证有效开始期 + */ + private String sfzStart; + + /** + * 身份证有效结束期 + */ + private String sfzEnd; + + /** + * 身份证地址 + */ + private String sfzSite; + + /** + * 身份证出生日期 + */ + private String sfzBirth; + + /** + * 籍贯 + */ + private String nativePlace; + + /** + * 银行卡号 + */ + private String yhkNumber; + + /** + * 开户行 + */ + private String yhkOpeningBank; + + /** + * 持卡人 + */ + private String yhkCardholder; + + /** + * 工种（字典） + */ + private String typeOfWork; + + /** + * 打卡(1启用打卡 2禁止打卡) + */ + private String clock = "2"; + + /** + * 劳务公司id + */ + private Long labourserviceId; + + /** + * 入场时间 + */ + private String entryDate; + + /** + * 离场时间 + */ + private String leaveDate; + + /** + * 薪水（此字段为0表示此字段无效） + */ + private BigDecimal salary = new BigDecimal("0.00"); + + /** + * 是否开启项目备案（小程序）1开启 2不开启 + */ + private String projectRecord = "2"; + + +} + diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java index 82b0c40a..72dba6c9 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/mapper/TransferDataMapper.java @@ -16,6 +16,10 @@ public interface TransferDataMapper { @Select("select * from bus_attendance where project_id in (59,60)") List getData(); + @DS("slave1") + @Select("select * from bus_construction_user where project_id in (59,60) and deleted_at is null") + List getUserList(); + @Select("select id,sys_user_id,project_id,user_name from sub_construction_user_copy1 where go_openid = #{openId}") ConstructionUserCopy getConstructionUserCopy(@Param("openId") String openId); @@ -38,7 +42,8 @@ public interface TransferDataMapper { int saveBatchCopy(@Param("list") List busAttendanceList); - @Select("select id,go_id,sfz_front_pic,sfz_back_pic,yhk_pic,special_work_pic from sub_construction_user_copy1") + @Select("select id,go_id,sfz_front_pic,sfz_back_pic,yhk_pic,special_work_pic from sub_construction_user " + + "where go_id between 107 and 893 order by go_id") List getConstructionUserList(); diff --git a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java index 1b8cb69e..ff099144 100644 --- a/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java +++ b/xinnengyuan/ruoyi-modules/ruoyi-system/src/main/java/org/dromara/transferData/service/TransferDataService.java @@ -11,6 +11,7 @@ import jakarta.activation.MimetypesFileTypeMap; import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.SerializationUtils; +import org.dromara.common.core.enums.UserType; import org.dromara.common.core.service.OssService; import org.dromara.contractor.domain.SubConstructionUser; import org.dromara.contractor.domain.SubConstructionUserFile; @@ -19,10 +20,14 @@ import org.dromara.contractor.service.ISubConstructionUserService; import org.dromara.project.domain.BusAttendance; import org.dromara.project.service.IBusAttendanceRuleService; import org.dromara.project.service.IBusAttendanceService; +import org.dromara.system.domain.SysUser; import org.dromara.system.domain.vo.SysOssVo; +import org.dromara.system.domain.vo.SysUserVo; import org.dromara.system.service.ISysOssService; +import org.dromara.system.service.ISysUserService; import org.dromara.transferData.domain.ConstructionUserNew; import org.dromara.transferData.domain.OldFile; +import org.dromara.transferData.domain.User; import org.dromara.transferData.domain.UserFile; import org.dromara.transferData.mapper.TransferDataMapper; import org.springframework.beans.factory.annotation.Qualifier; @@ -36,12 +41,14 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.Paths; +import java.text.SimpleDateFormat; import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import java.util.concurrent.Semaphore; import java.util.stream.Collectors; import static kotlin.reflect.jvm.internal.impl.builtins.StandardNames.FqNames.list; @@ -70,6 +77,8 @@ public class TransferDataService { private Executor attendanceAsyncExecutor; @Resource private TransferDataMapper transferDataMapper; + @Resource + private ISysUserService userService; // 两个候选基础URL @@ -371,13 +380,13 @@ public class TransferDataService { Map map = resultMap1.get(goId); if (map != null) { handle0to3(map, constructionUser); - handle4to10(map,goId); + handle4to10(map, goId); } Map map1 = resultMap.get(goId); if (map1 != null) { handle0to3(map1, constructionUser); - handle4to10(map1,goId); + handle4to10(map1, goId); } } @@ -414,11 +423,13 @@ public class TransferDataService { constructionUser.setSpecialWorkPic(l.toString()); } } + SubConstructionUser constructionUser1 = BeanUtil.copyProperties(constructionUser, SubConstructionUser.class); + log.info("开始修改数据，id：{}", constructionUser1.getId()); constructionUserService.updateById(constructionUser1); } - private void handle4to10(Map map,Long goId){ + private void handle4to10(Map map, Long goId) { List list1 = constructionUserFileService.list(Wrappers.lambdaQuery() .eq(SubConstructionUserFile::getUserId, goId) ); @@ -438,6 +449,357 @@ public class TransferDataService { } } } +// log.info("开始修改附件，数量：{}", list1.size()); constructionUserFileService.updateBatchById(list1); } + + + public void userTransfer() { + List userList = transferDataMapper.getUserList(); + log.info("开始处理用户数据，数量：{}", userList.size()); + + // 设置每批处理的数据量 + int batchSize = 300; + // 计算总批次数 + int totalBatches = (int) Math.ceil((double) userList.size() / batchSize); + + // 将数据分批 + List> batches = new ArrayList<>(); + for (int i = 0; i < totalBatches; i++) { + int startIndex = i * batchSize; + int endIndex = Math.min((i + 1) * batchSize, userList.size()); + batches.add(userList.subList(startIndex, endIndex)); + } + + // 使用并行流处理每批数据 + batches.parallelStream().forEach(batch -> { + try { + log.info("开始处理批次，数据量：{}", batch.size()); + handleUser(batch); + log.info("批次处理完成"); + } catch (Exception e) { + log.error("处理批次时发生错误，数据量：{}", batch.size(), e); + } + }); + + log.info("所有用户数据处理完成，总批次数：{}", totalBatches); + } + + + public void handleUser(List userList) { + //查询附件 + List list1 = userList.stream().map(User::getId).toList(); + List oldFileList = transferDataMapper.getOldFileList1(list1); + + // 将 oldFileList1 转换为 Map> 结构 + Map> oldFileMap = oldFileList.stream() + .collect(Collectors.groupingBy( + OldFile::getUserId, + Collectors.toMap( + OldFile::getUserImgType, + OldFile::getPath, + (existing, replacement) -> replacement // 如果有重复的 key，保留后面的值 + ) + )); + ArrayList userAddList = new ArrayList<>(); + List fileAddList = new ArrayList<>(); + for (User user : userList) { + String phone = user.getPhone(); + if(StrUtil.isBlank(phone)){ + continue; + } + + //获取sysUserId + Long sysUserId; + SysUserVo sysUserVo = userService.selectUserByPhonenumber(phone); + if (sysUserVo == null) { + SysUser sysUser = new SysUser(); + sysUser.setUserName(user.getPhone()); + sysUser.setNickName(user.getUserName()); + sysUser.setPhonenumber(phone); + sysUser.setSex(user.getSex()); + sysUser.setUserType(UserType.APP_USER.getUserType()); + sysUserId = userService.save(sysUser); + } else { + sysUserId = sysUserVo.getUserId(); + } + Map map = oldFileMap.get(user.getId()); + //创建施工人员 + SubConstructionUser constructUser = createConstructUser(user, sysUserId,map); + userAddList.add(constructUser); + //处理附件 + List subConstructionUserFiles = handle4to10ConstructionUser(map, sysUserId, user.getId()); + fileAddList.addAll(subConstructionUserFiles); + } + constructionUserService.saveBatch(userAddList); + constructionUserFileService.saveBatch(fileAddList); + + } + + + public SubConstructionUser createConstructUser(User user, Long sysUserId,Map fileMap) { + SubConstructionUser constructionUser = baseInfo(user, sysUserId); + //处理人脸照 + Long facePicId = handleSinglePhotoLimit(constructionUser.getFacePic()); + constructionUser.setFacePic(facePicId == null ? null : facePicId.toString()); + //处理身份证照,银行卡，特殊工作证 + handle0to3ConstructionUser(fileMap, constructionUser); + return constructionUser; + } + + public SubConstructionUser baseInfo(User user, Long sysUserId) { + SubConstructionUser constructionUser = new SubConstructionUser(); + constructionUser.setFacePic(user.getPacePhoto()); + constructionUser.setSysUserId(sysUserId); + constructionUser.setUserName(user.getUserName()); + constructionUser.setTeamName(user.getTeamName()); + constructionUser.setStatus(user.getStatus()); + constructionUser.setPhone(user.getPhone()); + constructionUser.setSex(user.getSex()); + constructionUser.setNation(user.getSfzNation()); + constructionUser.setSfzStart(stringToLocalDate(user.getSfzStart(), "yyyy-MM-dd")); + constructionUser.setSfzEnd(stringToLocalDate(user.getSfzEnd(), "yyyy-MM-dd")); + constructionUser.setSfzSite(user.getSfzSite()); + constructionUser.setSfzBirth(stringToLocalDate(user.getSfzBirth(), "yyyy-MM-dd")); + constructionUser.setNativePlace(user.getNativePlace()); + constructionUser.setYhkNumber(user.getYhkNumber()); + constructionUser.setYhkOpeningBank(user.getYhkOpeningBank()); + constructionUser.setYhkCardholder(user.getYhkCardholder()); + constructionUser.setTypeOfWork(user.getTypeOfWork()); + constructionUser.setWageMeasureUnit(user.getSalary().toString()); + constructionUser.setClock("0"); + constructionUser.setEntryDate(stringToDate(user.getEntryDate(), "yyyy-MM-dd HH:mm:ss")); + constructionUser.setLeaveDate(stringToDate(user.getLeaveDate(), "yyyy-MM-dd HH:mm:ss")); + constructionUser.setSalary(user.getSalary()); + constructionUser.setUserRole("1"); + constructionUser.setExitStatus(user.getProjectRecord()); + constructionUser.setRemark("GoLand"); + constructionUser.setGoId(user.getId()); + constructionUser.setGoOpenid(user.getOpenid()); + constructionUser.setProjectId(projectIdAndTeamId(1, user.getProjectId().toString())); + constructionUser.setTeamId(projectIdAndTeamId(2, user.getTeamId().toString())); + + constructionUser.setCreateBy(1L); + constructionUser.setUpdateBy(1L); + return constructionUser; + } + + public Long projectIdAndTeamId(Integer types, String id) { + Long val = 0L; // 假设val是long类型 + + switch (types) { + case 1: // 项目 + long targetID = switch (id) { + case "60" -> 1897160897167638529L; + case "59" -> 1897161054676336641L; + default -> + // 处理未匹配的情况，这里可以根据需要设置默认值 + 0; + }; + val = targetID==0?null:targetID; + break; + + case 2: // 班组 + long targetID2 = switch (id) { + case "135" -> 1966464033343221761L; // 二工区六朋班组 + case "131" -> 1966464063785480193L; // 二工区平武 + case "130" -> 1966464107406241794L; // 四工区甫必班组 + case "125" -> 1966464157486231554L; // 四工区福绿班组 + case "124" -> 1966469101257846785L; // 送出线路班组 + case "120" -> 1966469224557801474L; // 五工区班组 + case "118" -> 1966469254266056706L; // 三工区班组 + case "117" -> 1966469284351799298L; // 二工区外苏班组 + case "116" -> 1966469322419302402L; // 一工区班组 + case "115" -> 1967121995904598018L; // 贵州兰亭电力建设(集团)有限公司（升压站场平） + case "114" -> 1967122038313205761L; // 源梦诚标段三 + case "113" -> 1967122072953962498L; // 上海耀垦标段二 + default -> + // 处理未匹配的情况 + 0; + }; + val = targetID2==0?null:targetID2; + break; + + default: + // 处理其他types值的情况 + break; + } + return val; + } + + private LocalDate stringToLocalDate(String date, String format){ + if (date == null || date.isEmpty()) { + return null; + } + if ("长期".equals(date)) { + return LocalDate.of(9999, 12, 31); + } + SimpleDateFormat sdf = new SimpleDateFormat(format); + try { + Date parsedDate = sdf.parse(date); + return parsedDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + private Date stringToDate(String date, String format){ + if (date == null || date.isEmpty()) { + return null; + } + if ("长期".equals(date)) { + return Date.from(LocalDate.of(9999, 12, 31).atStartOfDay(ZoneId.systemDefault()).toInstant()); + } + SimpleDateFormat sdf = new SimpleDateFormat(format); + try { + return sdf.parse(date); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + + private void handle0to3ConstructionUser(Map map, SubConstructionUser constructionUser) { + String s = map.get("0"); + if (s != null) { + Long l = handleSinglePhotoLimit(s); + if (l != null) { + constructionUser.setSfzFrontPic(l.toString()); + } + } + + String s1 = map.get("1"); + if (s1 != null) { + Long l = handleSinglePhotoLimit(s1); + if (l != null) { + constructionUser.setSfzBackPic(l.toString()); + } + } + String s2 = map.get("2"); + if (s2 != null) { + Long l = handleSinglePhotoLimit(s2); + if (l != null) { + constructionUser.setYhkPic(l.toString()); + } + } + String s3 = map.get("3"); + if (s3 != null) { + Long l = handleSinglePhotoLimit(s3); + if (l != null) { + constructionUser.setSpecialWorkPic(l.toString()); + } + } + } + + private static List types = Arrays.asList("0", "1", "2", "3"); + private List handle4to10ConstructionUser(Map map,Long sysUserId,Long goId) { + List fileAddList = new ArrayList<>(); + for(String type : map.keySet()){ + if(types.contains(type)){ + continue; + } + SubConstructionUserFile userFile = new SubConstructionUserFile(); + userFile.setUserId(sysUserId); + userFile.setFileType(type); + userFile.setRemark("GoLand"); + userFile.setGoId(goId); + + String s = map.get(type); + if (s != null) { + String[] split = s.split(","); + List path = new ArrayList<>(); + for (String s1 : split) { + Long l = handleSinglePhotoLimit(s1); + if (l != null) { + path.add(l); + } + } + if (CollectionUtil.isNotEmpty(path)) { + userFile.setPath(path.stream().map(String::valueOf).collect(Collectors.joining(","))); + } + } + fileAddList.add(userFile); + } + return fileAddList; + } + + + // 添加信号量控制并发数，避免过多线程竞争资源 + private final Semaphore semaphore = new Semaphore(20); // 限制最多20个线程同时执行 + + private Long handleSinglePhotoLimit(String facePicRelativePath) { + if (facePicRelativePath == null || facePicRelativePath.isEmpty()) { + return null; + } + + try { + // 获取信号量许可，控制并发数 + semaphore.acquire(); + + // 1. 规范化路径（原逻辑不变） + String normalizedPath = facePicRelativePath.replace("\\", "/"); + String filename = Paths.get(normalizedPath).getFileName().toString(); + + // 2. 解析文件类型（优化：优先从HTTP响应头获取，其次用文件名） + String contentType = "application/octet-stream"; // 默认二进制类型 + + // 3. 多URL重试（增加重试次数和超时时间） + for (int retry = 0; retry < 3; retry++) { // 增加重试次数到3次 + for (String baseUrl : BASE_URLS) { + String fullUrl = baseUrl + normalizedPath; + try { + // 构建带超时的HTTP请求 + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(fullUrl)) + .GET() + .timeout(Duration.ofSeconds(30)) // 增加超时时间到30秒 + .build(); + + // 发送请求并处理响应 + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() == 200) { + try (InputStream inputStream = response.body()) { + // 从响应头获取真实Content-Type（比文件名解析更准确） + contentType = response.headers().firstValue("Content-Type").orElse(contentType); + long contentLength = response.headers().firstValueAsLong("Content-Length").orElse(-1); + + // 上传OSS（此时流已通过try-with-resources自动关闭） + SysOssVo ossVo = ossService.upload(inputStream, filename, contentType, contentLength); + return ossVo.getOssId(); // 成功则跳出所有循环 + } + } + } catch (Exception e) { + log.warn("尝试URL失败（重试{}次）: {}", retry + 1, fullUrl, e); + // 在重试前等待一段时间 + if (retry < 2) { // 不是最后一次重试才等待 + try { + Thread.sleep(500 * (retry + 1)); // 递增等待时间 + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + continue; // 重试下一个URL + } + } + } + + // 所有URL和重试都失败 + log.error("照片处理失败，所有URL重试完毕，relativePath={}", normalizedPath); + return null; + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("获取信号量被中断，relativePath={}", facePicRelativePath, e); + return null; + } catch (Exception e) { + log.error("处理照片时发生未知错误，relativePath={}", facePicRelativePath, e); + return null; + } finally { + // 释放信号量许可 + semaphore.release(); + } + } }