guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 多线程情况下对准备处理的数据进行查重

Browse Source
pull/139/head
Vincent 1 year ago
parent fcee0dabb2
commit e6c94f4a09
8 changed files with 127 additions and 155 deletions
Show Stats Download Patch File Download Diff File

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/PrevailCloudApi.java
Unescape View File

@ -3,6 +3,9 @@ package cn.jyjz.xiaoyao.ocr.api;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import org.apache.commons.compress.utils.Lists;
import java.util.List;
/**
* 无量云三方api接口
@ -33,4 +36,6 @@ public interface PrevailCloudApi {
*/
boolean startImageDownloadMonitoring(String localImagePath) throws Exception;
List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter);
}

24
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/impl/PrevailCloudApiImpl.java
Unescape View File

@ -6,7 +6,11 @@ import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.*;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiHelper;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.api.utils.RequestData;
import cn.jyjz.xiaoyao.ocr.api.utils.ResultData;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
@ -16,16 +20,13 @@ import cn.jyjz.xiaoyao.ocr.util.httputil.HttpMethod;
import cn.jyjz.xiaoyao.ocr.util.httputil.HttpParamers;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.annotation.JsonFormat;
import liquibase.pro.packaged.S;
import lombok.SneakyThrows;
import org.apache.commons.collections.SequencedHashMap;
import org.apache.commons.compress.utils.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -157,7 +158,8 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
String responseJsonStr = HttpClient.doPost(url, httpParamers, null, apiConfig.getConnectTimeout(), apiConfig.getReadTimeout());
System.out.println(responseJsonStr);
ResultData<String> resultData = JSONUtil.toBean(responseJsonStr, ResultData.class);
if(resultData.getStatus()==100){;
if (resultData.getStatus() == 100) {
;
//解析基础数据
String data = ApiHelper.decryptResponse(apiConfig.getAccessCode(), resultData);
ApiPage<String> apiPage = JSONUtil.toBean(data, ApiPage.class);
@ -218,5 +220,15 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
return true;
}
@Override
public List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter) {
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = this.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/utils/ApiConfig.java
Unescape View File

@ -48,6 +48,11 @@ public class ApiConfig {
private String localImagePath;
/**
* 默认图片
*/
private String defaultPictureImg;
/**
* 检查配置文件是否 正确注入
* @return

81
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -10,7 +10,6 @@ import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
@ -66,16 +65,6 @@ public class ApiTestController {
if (StringUtils.isBlank(accountNoStr)) {
return "业务主体不能为空";
}
// Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
// Long accountNo = Long.parseLong(request.getParameter("accountNo"));
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime = DateUtil.parse("2024-03-31 07:00:000","yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00","yyyy-MM-dd HH:mm:ss");
Instant einstant = dateTime1.toInstant();
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(departments)) {
@ -95,6 +84,16 @@ public class ApiTestController {
return "业务主体数据为空";
}
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(Integer.parseInt(pageNo));
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) {
@ -103,8 +102,10 @@ public class ApiTestController {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, Integer.parseInt(pageNo), Integer.parseInt(pageSize), tenantNo, accountNo);
List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
@ -117,11 +118,10 @@ public class ApiTestController {
if (pictureSourceResult.getLivePhoto() == null) {
List<Map<String, Object>> livePhoto = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
map.put("imgUrl", "http://47.93.59.251/api/image/default.png");
map.put("imgUrl", StringUtils.isEmpty(apiConfig.getDefaultPictureImg()) ? "http://47.93.59.251/api/image/default.png" : apiConfig.getDefaultPictureImg());
map.put("imgName", "default.png");
livePhoto.add(map);
pictureSourceResult.setLivePhoto(livePhoto);
// continue;
}
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult);
@ -151,8 +151,6 @@ public class ApiTestController {
String pageSize = request.getParameter("pageSize");
String pageNo = request.getParameter("pageNo");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
List<String> notAddedToQueueTaskIds = new ArrayList<>();
@ -183,6 +181,10 @@ public class ApiTestController {
int totalSize = 0;
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true;
@ -201,8 +203,13 @@ public class ApiTestController {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, startInstant, endInstant, currentPage, tenantNo, accountNo);
List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
@ -278,45 +285,5 @@ public class ApiTestController {
}
return message;
}
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, Instant startInstant, Instant endInstant, Integer currentPage, Long tenantNo, Long accountNo) {
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrFrontConfServiceImpl.java
Unescape View File

@ -33,6 +33,7 @@ public class OcrFrontConfServiceImpl extends BaseServiceImpl<OcrFrontConfMapper,
apiConfig.setConnectTimeout(frontConf.getWlyConnectTimeout());
apiConfig.setReadTimeout(frontConf.getWlyReadTimeout());
apiConfig.setLocalImagePath(frontConf.getWlyLocalImagePath());
apiConfig.setDefaultPictureImg(frontConf.getDefaultPictureImg());
}
}
}

23
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskQueue.java
Unescape View File

@ -2,9 +2,9 @@ package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
/**
@ -15,7 +15,6 @@ import java.util.concurrent.LinkedBlockingQueue;
* @date 2024/03/14 3:08
*/
public class TaskQueue {
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
/**
* 无量云接口数据处理 任务队列
@ -27,17 +26,25 @@ public class TaskQueue {
*/
public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
public static volatile Set<String> resultSet = Sets.newConcurrentHashSet();
/**
* 无量云接口数据处理 push 队列值
* @return
*
* @param pictureSourceResult //三方拉取 获取到的图片对象
* @return
*/
public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult) {
try {
Long taskId = pictureSourceResult.getTaskId();
if (resultSet.contains(String.valueOf(taskId))) {
return true;
}
resultSet.add(String.valueOf(taskId));
pictureDisposeQueue.put(pictureSourceResult);
} catch (InterruptedException e) {
logger.error("任务队列添加异常:{}",e.getMessage());
return false;
}
return true;
@ -45,6 +52,7 @@ public class TaskQueue {
/**
* 无量云接口数据处理 拉取队列值
*
* @return
*/
public static PictureSourceResult pictureDisposePullData() {
@ -54,14 +62,14 @@ public class TaskQueue {
/**
* 图片缓存本地队列 push 队列值
* @return
*
* @param pictureImgToLocalEntity //图片下载对象
* @return
*/
public static boolean pictureImgToLocalPushData(PictureImgToLocalEntity pictureImgToLocalEntity) {
try {
pictureImgToLocalQueue.put(pictureImgToLocalEntity);
} catch (InterruptedException e) {
logger.error("图片下载任务队列添加异常:{}",e.getMessage());
return false;
}
return true;
@ -69,6 +77,7 @@ public class TaskQueue {
/**
* 无量云接口图片数据处理 拉取队列值
*
* @return
*/
public static PictureImgToLocalEntity pictureImgToLocalPullData() {

32
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/timerJob/PictureSourceTimerJob.java
Unescape View File

@ -7,7 +7,6 @@ import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
@ -56,11 +55,8 @@ public class PictureSourceTimerJob {
//组装拉取参数
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
// pictureSourceParameter.setStartTime();
// pictureSourceParameter.setEndTime();
pictureSourceParameter.setPageNo(1);
pictureSourceParameter.setPageSize(200);
// pictureSourceParameter.setAccountNo();
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_MONTH, -1);
@ -69,6 +65,11 @@ public class PictureSourceTimerJob {
String endDateStr = dateFormat.format(yesterday).split(" ")[0] + " 23:59:59"; // 结束时间为昨天的23点59分59秒
DateTime dateTime = DateUtil.parse(startDateStr, "yyyy-MM-dd HH:mm:ss");
DateTime dateTime1 = DateUtil.parse(endDateStr, "yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
//1.拉取数据
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
@ -93,8 +94,10 @@ public class PictureSourceTimerJob {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, dateTime, dateTime1, tenantNo, accountNo);
List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
@ -114,23 +117,4 @@ public class PictureSourceTimerJob {
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
}
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, DateTime dateTime, DateTime dateTime1, Long tenantNo, Long accountNo) {
//创建查询参数对象
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

37
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/timerJob/PullPictureDataJob.java
Unescape View File

@ -7,7 +7,6 @@ import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
@ -66,6 +65,16 @@ public class PullPictureDataJob {
departmentMap.put(department.getId(), department);
}
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) {
@ -74,8 +83,10 @@ public class PullPictureDataJob {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, pageNo, pageSize, tenantNo, accountNo);
List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) {
continue;
}
@ -94,26 +105,4 @@ public class PullPictureDataJob {
}
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
}
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
}

Write Preview
Loading…
Cancel
Save