guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 定时拉取任务

Browse Source
feature/schedule
Vincent 1 year ago
parent 199dd11d34
commit ed5ca80adb
2 changed files with 221 additions and 85 deletions
Show Stats Download Patch File Download Diff File

170
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -3,17 +3,21 @@ package cn.jyjz.xiaoyao.ocr.controller;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.jyjz.xiaoyao.admin.dataDao.DepartmentMybatisDao;
import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.common.base.util.StringUtils;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.ImageDownloadMonitoring;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.utils.Lists;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ -21,7 +25,11 @@ import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import java.time.Instant;
import java.util.*;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* TODO
@ -41,6 +49,9 @@ public class ApiTestController {
@Autowired
private PrevailCloudApi prevailCloudApi;
@Resource
private DepartmentMybatisDao departmentMybatisDao;
@GetMapping("psTest")
public String psTset(HttpServletRequest request) {
String beginStr = request.getParameter("beginStr");
@ -55,8 +66,8 @@ public class ApiTestController {
if (StringUtils.isBlank(accountNoStr)) {
return "业务主体不能为空";
}
Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
Long accountNo = Long.parseLong(request.getParameter("accountNo"));
// Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
// Long accountNo = Long.parseLong(request.getParameter("accountNo"));
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
@ -65,22 +76,43 @@ public class ApiTestController {
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00","yyyy-MM-dd HH:mm:ss");
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(Integer.parseInt(pageNo));
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
try {
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
if(apiPage==null){
return "获取数据为空！";
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(departments)) {
return "业务主体数据为空";
}
List<PictureSourceResult> listResult= apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
Map<Long, Department> departmentMap = Maps.newHashMapWithExpectedSize(departments.size());
for (Department department : departments) {
if (department.getDlevel() == 0) {
continue;
}
departmentMap.put(department.getId(), department);
}
if (departmentMap.isEmpty()) {
return "业务主体数据为空";
}
List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) {
continue;
}
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, Integer.parseInt(pageNo), Integer.parseInt(pageSize), tenantNo, accountNo);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
totalList.addAll(resultList);
}
for (PictureSourceResult pictureSourceResult : totalList) {
//1.如果不存在图片默认一个默认图片，方便走测试。
if (pictureSourceResult.getLivePhoto() == null) {
List<Map<String, Object>> livePhoto = new ArrayList<>();
@ -91,16 +123,11 @@ public class ApiTestController {
pictureSourceResult.setLivePhoto(livePhoto);
// continue;
}
size++;
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult);
}
return "当前区间内图片总数:"+count+"条，本次获取："+localCount+"条，可处理数据:"+size+"条";
} catch (Exception e) {
e.printStackTrace();
}
return "处理失败";
return "处理成功";
}
@ -126,8 +153,6 @@ public class ApiTestController {
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
pictureSourceParameter.setTenantNo(1714548920583360512L);
pictureSourceParameter.setAccountNo(1714559323824623616L);
List<String> notAddedToQueueTaskIds = new ArrayList<>();
@ -135,6 +160,24 @@ public class ApiTestController {
@Override
public void run() {
try {
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(departments)) {
return;
}
Map<Long, Department> departmentMap = Maps.newHashMapWithExpectedSize(departments.size());
for (Department department : departments) {
if (department.getDlevel() == 0) {
continue;
}
departmentMap.put(department.getId(), department);
}
if (departmentMap.isEmpty()) {
return;
}
int totalCount = 0;
int totalLocalCount = 0;
int totalSize = 0;
@ -149,17 +192,29 @@ public class ApiTestController {
// 设置当天的开始和结束时间
Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) {
continue;
}
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, startInstant, endInstant, currentPage, tenantNo, accountNo);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
totalList.addAll(resultList);
}
int localCount = totalList.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
for (PictureSourceResult pictureSourceResult : totalList) {
int retryCount = 0;
boolean addedToQueue = false;
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
@ -183,12 +238,9 @@ public class ApiTestController {
totalSize += size;
if (!isCountAdded) {
totalCount += count;
isCountAdded = true; // 标记当天的统计信息已经被累加过了
}
// 输出当前日期范围内的统计信息
log.error("{}图片总数:{}条，本次获取：{}条，可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
@ -226,5 +278,45 @@ public class ApiTestController {
}
return message;
}
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, Instant startInstant, Instant endInstant, Integer currentPage, Long tenantNo, Long accountNo) {
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

86
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/timerJob/PullPictureDataJob.java
Unescape View File

@ -1,21 +1,28 @@
package cn.jyjz.xiaoyao.ocr.timerJob;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.jyjz.xiaoyao.admin.dataDao.DepartmentMybatisDao;
import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import java.time.Instant;
import java.util.Date;
import java.util.List;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.compress.utils.Lists;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
@Component
public class PullPictureDataJob {
@ -23,13 +30,14 @@ public class PullPictureDataJob {
@Autowired
private PrevailCloudApi prevailCloudApi;
@Resource
private DepartmentMybatisDao departmentMybatisDao;
// @Scheduled(cron = "0 5 0 * * ?") // 每天晚上0点五分执行一次
public void pullPicture() {
//配置查询参数信息
Integer pageNo = 1;
Integer pageSize = 20;
Long tenantNo = 1714548920583360512L;
Long accountNo = 1714559587755397120L;
// 设置前一天日期，0点到23点59分59秒
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar calendar = Calendar.getInstance();
@ -44,6 +52,50 @@ public class PullPictureDataJob {
// DateTime dateTime = DateUtil.parse("2024-03-31 07:00:000", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00", "yyyy-MM-dd HH:mm:ss");
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(departments)) {
return;
}
Map<Long, Department> departmentMap = Maps.newHashMapWithExpectedSize(departments.size());
for (Department department : departments) {
if (department.getDlevel() == 0) {
continue;
}
departmentMap.put(department.getId(), department);
}
List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) {
continue;
}
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, pageNo, pageSize, tenantNo, accountNo);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
totalList.addAll(resultList);
}
int count = totalList.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : totalList) {
if (pictureSourceResult.getLivePhoto() != null) {
size++;
// 将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult);
}
}
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
}
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
@ -55,21 +107,13 @@ public class PullPictureDataJob {
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
if (pictureSourceResult.getLivePhoto() != null) {
size++;
// 将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult);
}
}
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

Write Preview
Loading…
Cancel
Save