数据转移

This commit is contained in:
zt
2025-09-29 09:54:34 +08:00
parent 39038889b6
commit 2c554321bb
5 changed files with 139 additions and 32 deletions

View File

@ -57,9 +57,9 @@ spring:
lazy: true lazy: true
type: ${spring.datasource.type} type: ${spring.datasource.type}
driverClassName: com.mysql.cj.jdbc.Driver driverClassName: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://192.168.110.2:13386/zmkgdev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true url: jdbc:mysql://192.168.110.2:13386/zmkgc?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
username: zmkgdev username: zmkgc
password: JhYxREf25AXdy3h8 password: nWKDKRNRT48tFBdh
slave1: slave1:
lazy: true lazy: true
type: ${spring.datasource.type} type: ${spring.datasource.type}
@ -67,6 +67,13 @@ spring:
url: jdbc:mysql://192.168.110.2:13386/zmkgprod?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true url: jdbc:mysql://192.168.110.2:13386/zmkgprod?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
username: zmkgprod username: zmkgprod
password: MaY8nehwWkJriWPm password: MaY8nehwWkJriWPm
slave2:
lazy: true
type: ${spring.datasource.type}
driverClassName: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://192.168.110.2:13386/zmkgdev?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&useSSL=true&serverTimezone=GMT%2B8&autoReconnect=true&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&nullCatalogMeansCurrent=true
username: zmkgdev
password: JhYxREf25AXdy3h8
# oracle: # oracle:
# type: ${spring.datasource.type} # type: ${spring.datasource.type}
# driverClassName: oracle.jdbc.OracleDriver # driverClassName: oracle.jdbc.OracleDriver

View File

@ -13,7 +13,7 @@ import java.util.List;
* @author lilemy * @author lilemy
* @date 2025-09-10 19:32 * @date 2025-09-10 19:32
*/ */
@DS("slave") @DS("slave2")
@Mapper @Mapper
public interface ProjectBigScreenMapper { public interface ProjectBigScreenMapper {

View File

@ -7,6 +7,7 @@ import lombok.extern.slf4j.Slf4j;
import org.dromara.contractor.domain.SubConstructionUser; import org.dromara.contractor.domain.SubConstructionUser;
import org.dromara.contractor.service.ISubConstructionUserService; import org.dromara.contractor.service.ISubConstructionUserService;
import org.dromara.project.domain.BusAttendanceRule; import org.dromara.project.domain.BusAttendanceRule;
import org.dromara.project.service.IBusAttendanceService;
import org.dromara.system.domain.vo.SysOssVo; import org.dromara.system.domain.vo.SysOssVo;
import org.dromara.transferData.domain.ConstructionUserCopy; import org.dromara.transferData.domain.ConstructionUserCopy;
import org.dromara.transferData.domain.OldAttendance; import org.dromara.transferData.domain.OldAttendance;
@ -48,7 +49,12 @@ public class TransferDataController {
@Resource @Resource
private TransferDataService transferDataService; private TransferDataService transferDataService;
@Resource
private IBusAttendanceService attendanceService;
private static final String slave = "zmkgc";
private static final String slave1 = "zmkgprod";
// 两个候选基础URL // 两个候选基础URL
@ -59,10 +65,16 @@ public class TransferDataController {
@RequestMapping("/transferAttendance") @RequestMapping("/transferAttendance")
public void transferAttendance() { public void transferAttendance(String name) {
List<BusAttendance> arrs = new ArrayList<>(); List<BusAttendance> arrs = new ArrayList<>();
List<OldAttendance> data;
List<OldAttendance> data = transferDataMapper.getData(); if(slave.equals(name)){
data = transferDataMapper.getData0();
}else if(slave1.equals(name)) {
data = transferDataMapper.getData();
}else {
return;
}
List<BusAttendanceRule> list = attendanceRuleService.list(Wrappers.<BusAttendanceRule>lambdaQuery() List<BusAttendanceRule> list = attendanceRuleService.list(Wrappers.<BusAttendanceRule>lambdaQuery()
.in(BusAttendanceRule::getProjectId, Arrays.asList(1897160897167638529L, 1897161054676336641L))); .in(BusAttendanceRule::getProjectId, Arrays.asList(1897160897167638529L, 1897161054676336641L)));
@ -75,16 +87,16 @@ public class TransferDataController {
continue; continue;
} }
LocalDate clockDate = LocalDate.parse(oldAttendance.getPrintingDate()); LocalDate clockDate = LocalDate.parse(oldAttendance.getPrintingDate());
// //判定是否重 //判定是否重
// List<BusAttendance> list = attendanceService.list(Wrappers.lambdaQuery(BusAttendance.class) List<BusAttendance> list1 = attendanceService.list(Wrappers.lambdaQuery(BusAttendance.class)
// .eq(BusAttendance::getUserId, constructionUserCopy.getSysUserId()) .eq(BusAttendance::getUserId, constructionUserCopy.getSysUserId())
// .eq(BusAttendance::getProjectId, constructionUserCopy.getProjectId()) .eq(BusAttendance::getProjectId, constructionUserCopy.getProjectId())
// .eq(BusAttendance::getClockDate, clockDate) .eq(BusAttendance::getClockDate, clockDate)
// .eq(BusAttendance::getClockType, oldAttendance.getCommuter()) .eq(BusAttendance::getClockType, oldAttendance.getCommuter())
// ); );
// if(CollectionUtil.isNotEmpty(list)){ if(CollectionUtil.isNotEmpty(list1)){
// continue; continue;
// } }
BusAttendance busAttendance = new BusAttendance(); BusAttendance busAttendance = new BusAttendance();
@ -162,10 +174,10 @@ public class TransferDataController {
// attendanceService.saveBatch(arrs); // attendanceService.saveBatch(arrs);
// transferDataMapper.saveBatchCopy(arrs); // transferDataMapper.saveBatchCopy(arrs);
//更新施工人员图片 // //更新施工人员图片
transferDataService.renYuan(); // transferDataService.renYuan();
//更新附件图片 // //更新附件图片
transferDataService.renYuanFuJiang(); // transferDataService.renYuanFuJiang();
} }
@ -204,8 +216,8 @@ public class TransferDataController {
@GetMapping("/userTransfer") @GetMapping("/userTransfer")
private void userTransfer() { private void userTransfer(String name) {
transferDataService.userTransfer(); transferDataService.userTransfer(name);
} }

View File

@ -18,10 +18,19 @@ public interface TransferDataMapper {
@Select("select * from bus_attendance where project_id in (59,60)") @Select("select * from bus_attendance where project_id in (59,60)")
List<OldAttendance> getData(); List<OldAttendance> getData();
@DS("slave")
@Select("select * from bus_attendance where project_id in (59,60)")
List<OldAttendance> getData0();
@DS("slave1") @DS("slave1")
@Select("select * from bus_construction_user where project_id in (59,60) and deleted_at is null") @Select("select * from bus_construction_user where project_id in (59,60) and deleted_at is null")
List<User> getUserList(); List<User> getUserList();
@DS("slave")
@Select("select * from bus_construction_user where project_id in (59,60) and deleted_at is null")
List<User> getUserList0();
@Select("select id,sys_user_id,project_id,user_name from sub_construction_user where go_openid = #{openId}") @Select("select id,sys_user_id,project_id,user_name from sub_construction_user where go_openid = #{openId}")
ConstructionUserCopy getConstructionUserCopy(@Param("openId") String openId); ConstructionUserCopy getConstructionUserCopy(@Param("openId") String openId);
@ -60,6 +69,16 @@ public interface TransferDataMapper {
List<OldFile> getOldFileList1(List<Long> userIds); List<OldFile> getOldFileList1(List<Long> userIds);
@DS("slave") @DS("slave")
@Select("<script>" +
"SELECT * FROM bus_construction_user_file " +
"WHERE user_id IN " +
"<foreach collection='userIds' item='userId' open='(' separator=',' close=')'>" +
"#{userId}" +
"</foreach>" +
"</script>")
List<OldFile> getOldFileList0(List<Long> userIds);
@DS("slave2")
@Select("<script>" + @Select("<script>" +
"SELECT * FROM bus_construction_user_file " + "SELECT * FROM bus_construction_user_file " +
"WHERE user_id IN " + "WHERE user_id IN " +

View File

@ -19,8 +19,12 @@ import org.dromara.contractor.domain.SubConstructionUserFile;
import org.dromara.contractor.service.ISubConstructionUserFileService; import org.dromara.contractor.service.ISubConstructionUserFileService;
import org.dromara.contractor.service.ISubConstructionUserService; import org.dromara.contractor.service.ISubConstructionUserService;
import org.dromara.project.domain.BusAttendance; import org.dromara.project.domain.BusAttendance;
import org.dromara.project.domain.BusProjectTeamMember;
import org.dromara.project.domain.BusUserProjectRelevancy;
import org.dromara.project.service.IBusAttendanceRuleService; import org.dromara.project.service.IBusAttendanceRuleService;
import org.dromara.project.service.IBusAttendanceService; import org.dromara.project.service.IBusAttendanceService;
import org.dromara.project.service.IBusProjectTeamMemberService;
import org.dromara.project.service.IBusUserProjectRelevancyService;
import org.dromara.system.domain.SysUser; import org.dromara.system.domain.SysUser;
import org.dromara.system.domain.vo.SysOssVo; import org.dromara.system.domain.vo.SysOssVo;
import org.dromara.system.domain.vo.SysUserVo; import org.dromara.system.domain.vo.SysUserVo;
@ -83,6 +87,14 @@ public class TransferDataService {
private ISysUserService userService; private ISysUserService userService;
@Resource @Resource
private IdCardEncryptorUtil idCardEncryptorUtil; private IdCardEncryptorUtil idCardEncryptorUtil;
@Resource
private IBusProjectTeamMemberService projectTeamMemberService;
@Resource
private IBusUserProjectRelevancyService relevancyService;
private static final String slave = "zmkgc";
private static final String slave1 = "zmkgprod";
// 两个候选基础URL // 两个候选基础URL
@ -91,6 +103,7 @@ public class TransferDataService {
"http://192.168.110.2:7363" "http://192.168.110.2:7363"
}; };
/** /**
* 异步处理照片并批量保存(修复线程安全 + 性能优化) * 异步处理照片并批量保存(修复线程安全 + 性能优化)
* *
@ -461,8 +474,15 @@ public class TransferDataService {
} }
public void userTransfer() { public void userTransfer(String name) {
List<User> userList = transferDataMapper.getUserList(); List<User> userList;
if(slave.equals(name)){
userList = transferDataMapper.getUserList0();
}else if(slave1.equals(name)) {
userList = transferDataMapper.getUserList();
}else {
return;
}
log.info("开始处理用户数据,数量:{}", userList.size()); log.info("开始处理用户数据,数量:{}", userList.size());
// 设置每批处理的数据量 // 设置每批处理的数据量
@ -482,7 +502,7 @@ public class TransferDataService {
batches.parallelStream().forEach(batch -> { batches.parallelStream().forEach(batch -> {
try { try {
log.info("开始处理批次,数据量:{}", batch.size()); log.info("开始处理批次,数据量:{}", batch.size());
handleUser(batch); handleUser(batch,name);
log.info("批次处理完成"); log.info("批次处理完成");
} catch (Exception e) { } catch (Exception e) {
log.error("处理批次时发生错误,数据量:{}", batch.size(), e); log.error("处理批次时发生错误,数据量:{}", batch.size(), e);
@ -493,10 +513,17 @@ public class TransferDataService {
} }
public void handleUser(List<User> userList) { public void handleUser(List<User> userList,String name) {
//查询附件 //查询附件
List<Long> list1 = userList.stream().map(User::getId).toList(); List<Long> list1 = userList.stream().map(User::getId).toList();
List<OldFile> oldFileList = transferDataMapper.getOldFileList1(list1); List<OldFile> oldFileList;
if(slave.equals(name)){
oldFileList = transferDataMapper.getOldFileList0(list1);
}else if(slave1.equals(name)) {
oldFileList = transferDataMapper.getOldFileList1(list1);
}else {
oldFileList = new ArrayList<>();
}
// 将 oldFileList1 转换为 Map<Long, Map<String, String>> 结构 // 将 oldFileList1 转换为 Map<Long, Map<String, String>> 结构
Map<Long, Map<String, String>> oldFileMap = oldFileList.stream() Map<Long, Map<String, String>> oldFileMap = oldFileList.stream()
@ -510,12 +537,26 @@ public class TransferDataService {
)); ));
ArrayList<SubConstructionUser> userAddList = new ArrayList<>(); ArrayList<SubConstructionUser> userAddList = new ArrayList<>();
List<SubConstructionUserFile> fileAddList = new ArrayList<>(); List<SubConstructionUserFile> fileAddList = new ArrayList<>();
//人员与班组关联
List<BusProjectTeamMember> teamMemberAddList = new ArrayList<>();
//人员与项目关联
List<BusUserProjectRelevancy> relevancyAddList = new ArrayList<>();
for (User user : userList) { for (User user : userList) {
String phone = user.getPhone(); String phone = user.getPhone();
if(StrUtil.isBlank(phone)){ if(StrUtil.isBlank(phone)){
continue; continue;
} }
List<SubConstructionUser> list2 = constructionUserService.list(Wrappers.<SubConstructionUser>lambdaQuery()
.eq(SubConstructionUser::getPhone, user.getPhone())
);
if(CollectionUtil.isNotEmpty(list2)){
continue;
}
//获取sysUserId //获取sysUserId
Long sysUserId; Long sysUserId;
SysUserVo sysUserVo = userService.selectUserByPhonenumber(phone); SysUserVo sysUserVo = userService.selectUserByPhonenumber(phone);
@ -531,16 +572,44 @@ public class TransferDataService {
sysUserId = sysUserVo.getUserId(); sysUserId = sysUserVo.getUserId();
} }
Map<String, String> map = oldFileMap.get(user.getId()); Map<String, String> map = oldFileMap.get(user.getId());
//创建施工人员 //创建施工人员
SubConstructionUser constructUser = createConstructUser(user, sysUserId,map); SubConstructionUser constructUser = createConstructUser(user, sysUserId,map);
userAddList.add(constructUser); userAddList.add(constructUser);
//处理附件 //处理附件
List<SubConstructionUserFile> subConstructionUserFiles = handle4to10ConstructionUser(map, sysUserId, user.getId()); List<SubConstructionUserFile> subConstructionUserFiles = handle4to10ConstructionUser(map, sysUserId, user.getId());
fileAddList.addAll(subConstructionUserFiles); fileAddList.addAll(subConstructionUserFiles);
}
constructionUserService.saveBatch(userAddList);
constructionUserFileService.saveBatch(fileAddList);
//项目关联
if(constructUser.getProjectId() != null){
BusUserProjectRelevancy relevancy = new BusUserProjectRelevancy();
relevancy.setUserId(constructUser.getSysUserId());
relevancy.setProjectId(constructUser.getProjectId());
relevancyAddList.add(relevancy);
}
//班组关联
if(constructUser.getTeamId() != null){
BusProjectTeamMember teamMember = new BusProjectTeamMember();
teamMember.setTeamId(constructUser.getTeamId());
teamMember.setMemberId(constructUser.getSysUserId());
teamMember.setPostId("0");
teamMember.setProjectId(constructUser.getProjectId());
teamMemberAddList.add(teamMember);
}
}
if(CollectionUtil.isNotEmpty(userAddList)){
constructionUserService.saveBatch(userAddList);
}
if(CollectionUtil.isNotEmpty(fileAddList)){
constructionUserFileService.saveBatch(fileAddList);
}
if(CollectionUtil.isNotEmpty(teamMemberAddList)){
projectTeamMemberService.saveBatch(teamMemberAddList);
}
if(CollectionUtil.isNotEmpty(relevancyAddList)){
relevancyService.saveBatch(relevancyAddList);
}
} }
@ -581,7 +650,7 @@ public class TransferDataService {
constructionUser.setEntryDate(stringToDate(user.getEntryDate(), "yyyy-MM-dd HH:mm:ss")); constructionUser.setEntryDate(stringToDate(user.getEntryDate(), "yyyy-MM-dd HH:mm:ss"));
constructionUser.setLeaveDate(stringToDate(user.getLeaveDate(), "yyyy-MM-dd HH:mm:ss")); constructionUser.setLeaveDate(stringToDate(user.getLeaveDate(), "yyyy-MM-dd HH:mm:ss"));
constructionUser.setSalary(user.getSalary()); constructionUser.setSalary(user.getSalary());
constructionUser.setUserRole("1"); constructionUser.setUserRole("0");
constructionUser.setExitStatus(user.getProjectRecord()); constructionUser.setExitStatus(user.getProjectRecord());
constructionUser.setRemark("GoLand"); constructionUser.setRemark("GoLand");
constructionUser.setGoId(user.getId()); constructionUser.setGoId(user.getId());