|
|
|
@ -3,17 +3,21 @@ package cn.jyjz.xiaoyao.ocr.controller;
|
|
|
|
|
import cn.hutool.core.date.DateField;
|
|
|
|
|
import cn.hutool.core.date.DateTime;
|
|
|
|
|
import cn.hutool.core.date.DateUtil;
|
|
|
|
|
import cn.jyjz.xiaoyao.admin.dataDao.DepartmentMybatisDao;
|
|
|
|
|
import cn.jyjz.xiaoyao.admin.dataobject.Department;
|
|
|
|
|
import cn.jyjz.xiaoyao.common.base.util.StringUtils;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.thread.ImageDownloadMonitoring;
|
|
|
|
|
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
|
|
|
|
|
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
|
|
|
|
|
import com.google.common.collect.Maps;
|
|
|
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
|
import org.apache.commons.collections4.CollectionUtils;
|
|
|
|
|
import org.apache.commons.compress.utils.Lists;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
|
|
|
import org.springframework.web.bind.annotation.GetMapping;
|
|
|
|
|
import org.springframework.web.bind.annotation.RequestMapping;
|
|
|
|
|
import org.springframework.web.bind.annotation.RestController;
|
|
|
|
@ -21,7 +25,11 @@ import org.springframework.web.bind.annotation.RestController;
|
|
|
|
|
import javax.annotation.Resource;
|
|
|
|
|
import javax.servlet.http.HttpServletRequest;
|
|
|
|
|
import java.time.Instant;
|
|
|
|
|
import java.util.*;
|
|
|
|
|
import java.util.ArrayList;
|
|
|
|
|
import java.util.Date;
|
|
|
|
|
import java.util.HashMap;
|
|
|
|
|
import java.util.List;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* TODO
|
|
|
|
@ -41,71 +49,90 @@ public class ApiTestController {
|
|
|
|
|
@Autowired
|
|
|
|
|
private PrevailCloudApi prevailCloudApi;
|
|
|
|
|
|
|
|
|
|
@Resource
|
|
|
|
|
private DepartmentMybatisDao departmentMybatisDao;
|
|
|
|
|
|
|
|
|
|
@GetMapping("psTest")
|
|
|
|
|
public String psTset(HttpServletRequest request){
|
|
|
|
|
public String psTset(HttpServletRequest request) {
|
|
|
|
|
String beginStr = request.getParameter("beginStr");
|
|
|
|
|
String endStr = request.getParameter("endStr");
|
|
|
|
|
String pageSize = request.getParameter("pageSize");
|
|
|
|
|
String pageNo = request.getParameter("pageNo");
|
|
|
|
|
String tenantNoStr = request.getParameter("tenantNo");
|
|
|
|
|
if(StringUtils.isBlank(tenantNoStr)){
|
|
|
|
|
if (StringUtils.isBlank(tenantNoStr)) {
|
|
|
|
|
return "租户不能为空";
|
|
|
|
|
}
|
|
|
|
|
String accountNoStr = request.getParameter("accountNo");
|
|
|
|
|
if(StringUtils.isBlank(accountNoStr)){
|
|
|
|
|
if (StringUtils.isBlank(accountNoStr)) {
|
|
|
|
|
return "业务主体不能为空";
|
|
|
|
|
}
|
|
|
|
|
Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
|
|
|
|
|
Long accountNo = Long.parseLong(request.getParameter("accountNo"));
|
|
|
|
|
// Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
|
|
|
|
|
// Long accountNo = Long.parseLong(request.getParameter("accountNo"));
|
|
|
|
|
|
|
|
|
|
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
|
|
|
|
|
DateTime dateTime = DateUtil.parse(beginStr+" 00:00:00","yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
// DateTime dateTime = DateUtil.parse("2024-03-31 07:00:000","yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
Instant binstant = dateTime.toInstant();
|
|
|
|
|
DateTime dateTime1 = DateUtil.parse(endStr+" 00:00:00","yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00","yyyy-MM-dd HH:mm:ss");
|
|
|
|
|
Instant einstant = dateTime1.toInstant();
|
|
|
|
|
pictureSourceParameter.setStartTime(Date.from(binstant));
|
|
|
|
|
pictureSourceParameter.setEndTime(Date.from(einstant));
|
|
|
|
|
pictureSourceParameter.setPageNo(Integer.parseInt(pageNo));
|
|
|
|
|
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
|
|
|
|
|
pictureSourceParameter.setTenantNo(tenantNo);
|
|
|
|
|
pictureSourceParameter.setAccountNo(accountNo);
|
|
|
|
|
try {
|
|
|
|
|
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
|
|
|
|
|
if(apiPage==null){
|
|
|
|
|
return "获取数据为空!";
|
|
|
|
|
|
|
|
|
|
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
|
|
|
|
|
if (CollectionUtils.isEmpty(departments)) {
|
|
|
|
|
return "业务主体数据为空";
|
|
|
|
|
}
|
|
|
|
|
List<PictureSourceResult> listResult= apiPage.getRecords();
|
|
|
|
|
int count = apiPage.getTotal();
|
|
|
|
|
int localCount = listResult.size();
|
|
|
|
|
int size = 0;
|
|
|
|
|
for (PictureSourceResult pictureSourceResult : listResult) {
|
|
|
|
|
|
|
|
|
|
Map<Long, Department> departmentMap = Maps.newHashMapWithExpectedSize(departments.size());
|
|
|
|
|
for (Department department : departments) {
|
|
|
|
|
if (department.getDlevel() == 0) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
departmentMap.put(department.getId(), department);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (departmentMap.isEmpty()) {
|
|
|
|
|
return "业务主体数据为空";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
List<PictureSourceResult> totalList = Lists.newArrayList();
|
|
|
|
|
for (Department department : departments) {
|
|
|
|
|
if (!departmentMap.containsKey(department.getParentid())) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
|
|
|
|
|
Long accountNo = Long.valueOf(department.getDeptno());
|
|
|
|
|
|
|
|
|
|
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, Integer.parseInt(pageNo), Integer.parseInt(pageSize), tenantNo, accountNo);
|
|
|
|
|
if (CollectionUtils.isEmpty(resultList)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
totalList.addAll(resultList);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (PictureSourceResult pictureSourceResult : totalList) {
|
|
|
|
|
//1.如果不存在图片默认一个默认图片,方便走测试。
|
|
|
|
|
if(pictureSourceResult.getLivePhoto()==null){
|
|
|
|
|
List<Map<String,Object>> livePhoto = new ArrayList<>();
|
|
|
|
|
Map<String,Object> map = new HashMap<>();
|
|
|
|
|
map.put("imgUrl","http://47.93.59.251/api/image/default.png");
|
|
|
|
|
map.put("imgName","default.png");
|
|
|
|
|
if (pictureSourceResult.getLivePhoto() == null) {
|
|
|
|
|
List<Map<String, Object>> livePhoto = new ArrayList<>();
|
|
|
|
|
Map<String, Object> map = new HashMap<>();
|
|
|
|
|
map.put("imgUrl", "http://47.93.59.251/api/image/default.png");
|
|
|
|
|
map.put("imgName", "default.png");
|
|
|
|
|
livePhoto.add(map);
|
|
|
|
|
pictureSourceResult.setLivePhoto(livePhoto);
|
|
|
|
|
// continue;
|
|
|
|
|
}
|
|
|
|
|
size++;
|
|
|
|
|
//将可以处理数据放入处理队列中
|
|
|
|
|
TaskQueue.pictureDisposePushData(pictureSourceResult);
|
|
|
|
|
}
|
|
|
|
|
return "当前区间内图片总数:"+count+"条,本次获取:"+localCount+"条,可处理数据:"+size+"条";
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return "处理失败";
|
|
|
|
|
return "处理成功";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@GetMapping("pullAccountInfo")
|
|
|
|
|
public String pullAccountInfo(HttpServletRequest request){
|
|
|
|
|
public String pullAccountInfo(HttpServletRequest request) {
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
prevailCloudApi.pullAccountInfo(1714548920583360512L);
|
|
|
|
@ -118,7 +145,7 @@ public class ApiTestController {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@GetMapping("psTest01")
|
|
|
|
|
public String psTset01(HttpServletRequest request){
|
|
|
|
|
public String psTset01(HttpServletRequest request) {
|
|
|
|
|
String beginStr = request.getParameter("beginStr");
|
|
|
|
|
String endStr = request.getParameter("endStr");
|
|
|
|
|
String pageSize = request.getParameter("pageSize");
|
|
|
|
@ -126,8 +153,6 @@ public class ApiTestController {
|
|
|
|
|
|
|
|
|
|
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
|
|
|
|
|
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
|
|
|
|
|
pictureSourceParameter.setTenantNo(1714548920583360512L);
|
|
|
|
|
pictureSourceParameter.setAccountNo(1714559323824623616L);
|
|
|
|
|
|
|
|
|
|
List<String> notAddedToQueueTaskIds = new ArrayList<>();
|
|
|
|
|
|
|
|
|
@ -135,6 +160,24 @@ public class ApiTestController {
|
|
|
|
|
@Override
|
|
|
|
|
public void run() {
|
|
|
|
|
try {
|
|
|
|
|
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
|
|
|
|
|
if (CollectionUtils.isEmpty(departments)) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Map<Long, Department> departmentMap = Maps.newHashMapWithExpectedSize(departments.size());
|
|
|
|
|
for (Department department : departments) {
|
|
|
|
|
if (department.getDlevel() == 0) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
departmentMap.put(department.getId(), department);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (departmentMap.isEmpty()) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int totalCount = 0;
|
|
|
|
|
int totalLocalCount = 0;
|
|
|
|
|
int totalSize = 0;
|
|
|
|
@ -149,17 +192,29 @@ public class ApiTestController {
|
|
|
|
|
// 设置当天的开始和结束时间
|
|
|
|
|
Instant startInstant = startDate.toInstant();
|
|
|
|
|
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
|
|
|
|
|
pictureSourceParameter.setStartTime(Date.from(startInstant));
|
|
|
|
|
pictureSourceParameter.setEndTime(Date.from(endInstant));
|
|
|
|
|
pictureSourceParameter.setPageNo(currentPage);
|
|
|
|
|
|
|
|
|
|
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
|
|
|
|
|
List<PictureSourceResult> listResult = apiPage.getRecords();
|
|
|
|
|
int count = apiPage.getTotal();
|
|
|
|
|
int localCount = listResult.size();
|
|
|
|
|
List<PictureSourceResult> totalList = Lists.newArrayList();
|
|
|
|
|
for (Department department : departments) {
|
|
|
|
|
if (!departmentMap.containsKey(department.getParentid())) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
|
|
|
|
|
Long accountNo = Long.valueOf(department.getDeptno());
|
|
|
|
|
|
|
|
|
|
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, startInstant, endInstant, currentPage, tenantNo, accountNo);
|
|
|
|
|
if (CollectionUtils.isEmpty(resultList)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
totalList.addAll(resultList);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int localCount = totalList.size();
|
|
|
|
|
int size = 0;
|
|
|
|
|
|
|
|
|
|
for (PictureSourceResult pictureSourceResult : listResult) {
|
|
|
|
|
for (PictureSourceResult pictureSourceResult : totalList) {
|
|
|
|
|
int retryCount = 0;
|
|
|
|
|
boolean addedToQueue = false;
|
|
|
|
|
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
|
|
|
|
@ -173,7 +228,7 @@ public class ApiTestController {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (!addedToQueue) {
|
|
|
|
|
log.error("无法将数据添加到队列,开始时间:{},任务ID:{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
|
|
|
|
|
log.error("无法将数据添加到队列,开始时间:{},任务ID:{}", startDate.toString("yyyy-MM-dd HH:mm:ss"), pictureSourceResult.getTaskId().toString());
|
|
|
|
|
notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -183,12 +238,9 @@ public class ApiTestController {
|
|
|
|
|
totalSize += size;
|
|
|
|
|
|
|
|
|
|
if (!isCountAdded) {
|
|
|
|
|
totalCount += count;
|
|
|
|
|
isCountAdded = true; // 标记当天的统计信息已经被累加过了
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 输出当前日期范围内的统计信息
|
|
|
|
|
log.error("{}图片总数:{}条,本次获取:{}条,可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
|
|
|
|
|
// 准备处理下一页的数据
|
|
|
|
|
currentPage++;
|
|
|
|
|
// 检查是否有下一页
|
|
|
|
@ -199,8 +251,8 @@ public class ApiTestController {
|
|
|
|
|
// 准备处理下一天的数据
|
|
|
|
|
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
|
|
|
|
|
}
|
|
|
|
|
log.error("未加入队列的任务ID列表:{}",notAddedToQueueTaskIds);
|
|
|
|
|
log.error("当前区间内图片总数:{}条,本次获取:{}条,可处理数据:{}条",totalCount,totalLocalCount,totalSize);
|
|
|
|
|
log.error("未加入队列的任务ID列表:{}", notAddedToQueueTaskIds);
|
|
|
|
|
log.error("当前区间内图片总数:{}条,本次获取:{}条,可处理数据:{}条", totalCount, totalLocalCount, totalSize);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
@ -226,5 +278,45 @@ public class ApiTestController {
|
|
|
|
|
}
|
|
|
|
|
return message;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, Instant startInstant, Instant endInstant, Integer currentPage, Long tenantNo, Long accountNo) {
|
|
|
|
|
|
|
|
|
|
pictureSourceParameter.setStartTime(Date.from(startInstant));
|
|
|
|
|
pictureSourceParameter.setEndTime(Date.from(endInstant));
|
|
|
|
|
pictureSourceParameter.setPageNo(currentPage);
|
|
|
|
|
pictureSourceParameter.setTenantNo(tenantNo);
|
|
|
|
|
pictureSourceParameter.setAccountNo(accountNo);
|
|
|
|
|
|
|
|
|
|
ApiPage<PictureSourceResult> apiPage = null;
|
|
|
|
|
try {
|
|
|
|
|
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
|
|
|
|
|
//创建查询参数对象
|
|
|
|
|
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
|
|
|
|
|
Instant binstant = dateTime.toInstant();
|
|
|
|
|
Instant einstant = dateTime1.toInstant();
|
|
|
|
|
pictureSourceParameter.setStartTime(Date.from(binstant));
|
|
|
|
|
pictureSourceParameter.setEndTime(Date.from(einstant));
|
|
|
|
|
pictureSourceParameter.setPageNo(pageNo);
|
|
|
|
|
pictureSourceParameter.setPageSize(pageSize);
|
|
|
|
|
pictureSourceParameter.setTenantNo(tenantNo);
|
|
|
|
|
pictureSourceParameter.setAccountNo(accountNo);
|
|
|
|
|
|
|
|
|
|
ApiPage<PictureSourceResult> apiPage = null;
|
|
|
|
|
try {
|
|
|
|
|
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
|
e.printStackTrace();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|