guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Labels Milestones
New Pull Request

feat: 多线程情况下对准备处理的数据进行查重 #139

Merged
yaoshuli merged 2 commits from feature/schedulePullData into test 1 year ago
Conversation 0 Commits 2 Files Changed 10
10 changed files with 142 additions and 155 deletions
Show Stats Download Patch File Download Diff File

10
htmlweb/web/backstage/oa/ocr/frontconf/list.html
Unescape View File

@ -128,6 +128,16 @@
</div> </div>
</div> </div>
</div> </div>
<div class="row">
<div class="col-sm-12">
<div class="form-group">
<label class="col-sm-3 control-label is-required"> 默认任务图片:</label>
<div class="col-sm-9">
<input name="defaultPictureImg" id="defaultPictureImg" class="form-control" placeholder="默认任务图片" class="form-control" type="text" value="" >
</div>
</div>
</div>
</div>
<div class="row"> <div class="row">
<div class="form-group"> <div class="form-group">
<div class="col-sm-offset-5"> <div class="col-sm-offset-5">

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/PrevailCloudApi.java
Unescape View File

@ -3,6 +3,9 @@ package cn.jyjz.xiaoyao.ocr.api;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage; import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import org.apache.commons.compress.utils.Lists;
import java.util.List;
/** /**
* 无量云三方api接口 * 无量云三方api接口
@ -33,4 +36,6 @@ public interface PrevailCloudApi {
*/ */
boolean startImageDownloadMonitoring(String localImagePath) throws Exception; boolean startImageDownloadMonitoring(String localImagePath) throws Exception;
List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter);
} }

76
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/impl/PrevailCloudApiImpl.java
Unescape View File

@ -6,7 +6,11 @@ import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi; import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.*; import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiHelper;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.api.utils.RequestData;
import cn.jyjz.xiaoyao.ocr.api.utils.ResultData;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture; import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService; import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
@ -16,16 +20,13 @@ import cn.jyjz.xiaoyao.ocr.util.httputil.HttpMethod;
import cn.jyjz.xiaoyao.ocr.util.httputil.HttpParamers; import cn.jyjz.xiaoyao.ocr.util.httputil.HttpParamers;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.annotation.JsonFormat;
import liquibase.pro.packaged.S;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.apache.commons.collections.SequencedHashMap; import org.apache.commons.compress.utils.Lists;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.math.BigDecimal;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -73,17 +74,17 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
@Override @Override
public ApiPage<PictureSourceResult> pullPictureSource(PictureSourceParameter pictureSourceParameter) throws Exception { public ApiPage<PictureSourceResult> pullPictureSource(PictureSourceParameter pictureSourceParameter) throws Exception {
//1.检查api配置是否正常 //1.检查api配置是否正常
if(!apiConfig.checkConfig()){ if (!apiConfig.checkConfig()) {
return null; return null;
} }
//2.对象转map //2.对象转map
Map<String, Object> queryParam = BeanUtil.beanToMap(pictureSourceParameter); Map<String, Object> queryParam = BeanUtil.beanToMap(pictureSourceParameter);
Map<String,String> queryParamStr = new HashMap<>(); Map<String, String> queryParamStr = new HashMap<>();
//map值转string 注意对日期类型进行操作,加密包要求值全部为string //map值转string 注意对日期类型进行操作,加密包要求值全部为string
for (String key : queryParam.keySet()) { for (String key : queryParam.keySet()) {
if(queryParam.get(key)!=null){ if (queryParam.get(key) != null) {
queryParamStr.put(key,queryParam.get(key).toString()); queryParamStr.put(key, queryParam.get(key).toString());
} }
} }
@ -92,36 +93,36 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
String tenantNo = pictureSourceParameter.getTenantNo().toString(); String tenantNo = pictureSourceParameter.getTenantNo().toString();
String json = JSONObject.toJSONString(queryParam); String json = JSONObject.toJSONString(queryParam);
System.out.println(json); System.out.println(json);
RequestData requestData = ApiHelper.buildRequest(tenantNo, apiConfig.getAccessKey(), apiConfig.getAccessCode(),json); RequestData requestData = ApiHelper.buildRequest(tenantNo, apiConfig.getAccessKey(), apiConfig.getAccessCode(), json);
//组装请求参数 //组装请求参数
String url = apiConfig.getInterfaceDomain()+pictureSourceUrl; String url = apiConfig.getInterfaceDomain() + pictureSourceUrl;
String requestBodyJson = JSONObject.toJSONString(requestData); String requestBodyJson = JSONObject.toJSONString(requestData);
// String requestBodyJson = "{\"accessKey\":\"7390F0221A1A73D8E13F8C8BB96F33B0\",\"data\":\"gzUtN5Mt24_fbDDMK3wLop8Tgu68xLCVy3Wzp9BcXnkxnCiPg_D2TNEXnQ68-Y-GhGYYcD7Iwg8juZTx1m5v4NkDvKvEwdvjQvZnMDTRYbqtHrwgVLAvUGotIMTckDY67yo2Ku52vWedjYpzzDseHwiIGVcIl-u1CljEbyjLzJzIf6oXfD7qKjOYnxKjfjp11k0IlvsPdS-mgNXTHcBei5jOFrYmZrE2y4v9bI5n1QE\",\"signature\":\"0bmIbhhybDcGeyX4ao5gSPZQeaY8G2JRh3nRt7NC7qw\",\"tenantCode\":\"1673196576094806016\",\"timestamp\":1711971774448}"; // String requestBodyJson = "{\"accessKey\":\"7390F0221A1A73D8E13F8C8BB96F33B0\",\"data\":\"gzUtN5Mt24_fbDDMK3wLop8Tgu68xLCVy3Wzp9BcXnkxnCiPg_D2TNEXnQ68-Y-GhGYYcD7Iwg8juZTx1m5v4NkDvKvEwdvjQvZnMDTRYbqtHrwgVLAvUGotIMTckDY67yo2Ku52vWedjYpzzDseHwiIGVcIl-u1CljEbyjLzJzIf6oXfD7qKjOYnxKjfjp11k0IlvsPdS-mgNXTHcBei5jOFrYmZrE2y4v9bI5n1QE\",\"signature\":\"0bmIbhhybDcGeyX4ao5gSPZQeaY8G2JRh3nRt7NC7qw\",\"tenantCode\":\"1673196576094806016\",\"timestamp\":1711971774448}";
//入参 //入参
System.out.println("入参数据:"+requestBodyJson); System.out.println("入参数据:" + requestBodyJson);
HttpParamers httpParamers = new HttpParamers(HttpMethod.POST); HttpParamers httpParamers = new HttpParamers(HttpMethod.POST);
httpParamers.setJsonParamer(requestBodyJson); httpParamers.setJsonParamer(requestBodyJson);
//发起请求 //发起请求
String responseJsonStr = HttpClient.doPost(url,httpParamers,null,apiConfig.getConnectTimeout(),apiConfig.getReadTimeout()); String responseJsonStr = HttpClient.doPost(url, httpParamers, null, apiConfig.getConnectTimeout(), apiConfig.getReadTimeout());
System.out.println(responseJsonStr); System.out.println(responseJsonStr);
ResultData<String> resultData =JSONUtil.toBean(responseJsonStr,ResultData.class); ResultData<String> resultData = JSONUtil.toBean(responseJsonStr, ResultData.class);
//解密请求数据 //解密请求数据
if(resultData.getStatus()==100){ if (resultData.getStatus() == 100) {
//解析基础数据 //解析基础数据
String data = ApiHelper.decryptResponse(apiConfig.getAccessCode(), resultData); String data = ApiHelper.decryptResponse(apiConfig.getAccessCode(), resultData);
ApiPage<String> apiPage = JSONUtil.toBean(data,ApiPage.class); ApiPage<String> apiPage = JSONUtil.toBean(data, ApiPage.class);
//解析数据结果为List //解析数据结果为List
List<PictureSourceResult> listResult = JSONUtil.toList(JSONObject.toJSONString(apiPage.getRecords()),PictureSourceResult.class); List<PictureSourceResult> listResult = JSONUtil.toList(JSONObject.toJSONString(apiPage.getRecords()), PictureSourceResult.class);
ApiPage<PictureSourceResult> resultApiPage = new ApiPage<>(); ApiPage<PictureSourceResult> resultApiPage = new ApiPage<>();
resultApiPage.setPageNo(apiPage.getPageNo()); resultApiPage.setPageNo(apiPage.getPageNo());
resultApiPage.setPageSize(apiPage.getPageSize()); resultApiPage.setPageSize(apiPage.getPageSize());
resultApiPage.setTotal(apiPage.getTotal()); resultApiPage.setTotal(apiPage.getTotal());
resultApiPage.setRecords(listResult); resultApiPage.setRecords(listResult);
return resultApiPage; return resultApiPage;
}else{ } else {
logger.debug("请求图片接口数据失败,参数:{},返回值:{}",json,responseJsonStr); logger.debug("请求图片接口数据失败,参数:{},返回值:{}", json, responseJsonStr);
} }
return null; return null;
@ -131,36 +132,37 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
public void sendFlowTaskResult(String tenantId, String data) throws Exception { public void sendFlowTaskResult(String tenantId, String data) throws Exception {
ResultData<String> stringResultData = ApiHelper.buildResponse(apiConfig.getAccessCode(), data); ResultData<String> stringResultData = ApiHelper.buildResponse(apiConfig.getAccessCode(), data);
// RequestData requestData = ApiHelper.buildRequest(tenantId, apiConfig.getAccessKey(), apiConfig.getAccessCode(),data);//组装请求参数 // RequestData requestData = ApiHelper.buildRequest(tenantId, apiConfig.getAccessKey(), apiConfig.getAccessCode(),data);//组装请求参数
String url = apiConfig.getInterfaceDomain()+sendResultUrl; String url = apiConfig.getInterfaceDomain() + sendResultUrl;
String requestBodyJson = JSONObject.toJSONString(stringResultData); String requestBodyJson = JSONObject.toJSONString(stringResultData);
//入参 //入参
System.out.println("入参数据:"+requestBodyJson); System.out.println("入参数据:" + requestBodyJson);
HttpParamers httpParamers = new HttpParamers(HttpMethod.POST); HttpParamers httpParamers = new HttpParamers(HttpMethod.POST);
httpParamers.setJsonParamer(requestBodyJson); httpParamers.setJsonParamer(requestBodyJson);
//发起请求 //发起请求
String responseJsonStr = HttpClient.doPost(url,httpParamers,null,apiConfig.getConnectTimeout(),apiConfig.getReadTimeout()); String responseJsonStr = HttpClient.doPost(url, httpParamers, null, apiConfig.getConnectTimeout(), apiConfig.getReadTimeout());
System.out.println(responseJsonStr); System.out.println(responseJsonStr);
} }
@Override @Override
public void pullAccountInfo(long tenantNo) throws Exception { public void pullAccountInfo(long tenantNo) throws Exception {
Map<String, Object> queryParam = new HashMap<>(); Map<String, Object> queryParam = new HashMap<>();
queryParam.put("tenantNo",tenantNo); queryParam.put("tenantNo", tenantNo);
String json = JSONObject.toJSONString(queryParam); String json = JSONObject.toJSONString(queryParam);
RequestData requestData = ApiHelper.buildRequest(tenantNo+"", apiConfig.getAccessKey(), apiConfig.getAccessCode(),json); RequestData requestData = ApiHelper.buildRequest(tenantNo + "", apiConfig.getAccessKey(), apiConfig.getAccessCode(), json);
String url = apiConfig.getInterfaceDomain()+tenantSourceUrl; String url = apiConfig.getInterfaceDomain() + tenantSourceUrl;
String requestBodyJson = JSONObject.toJSONString(requestData); String requestBodyJson = JSONObject.toJSONString(requestData);
//入参 //入参
HttpParamers httpParamers = new HttpParamers(HttpMethod.POST); HttpParamers httpParamers = new HttpParamers(HttpMethod.POST);
httpParamers.setJsonParamer(requestBodyJson); httpParamers.setJsonParamer(requestBodyJson);
//发起请求 //发起请求
String responseJsonStr = HttpClient.doPost(url,httpParamers,null,apiConfig.getConnectTimeout(),apiConfig.getReadTimeout()); String responseJsonStr = HttpClient.doPost(url, httpParamers, null, apiConfig.getConnectTimeout(), apiConfig.getReadTimeout());
System.out.println(responseJsonStr); System.out.println(responseJsonStr);
ResultData<String> resultData =JSONUtil.toBean(responseJsonStr,ResultData.class); ResultData<String> resultData = JSONUtil.toBean(responseJsonStr, ResultData.class);
if(resultData.getStatus()==100){; if (resultData.getStatus() == 100) {
;
//解析基础数据 //解析基础数据
String data = ApiHelper.decryptResponse(apiConfig.getAccessCode(), resultData); String data = ApiHelper.decryptResponse(apiConfig.getAccessCode(), resultData);
ApiPage<String> apiPage = JSONUtil.toBean(data,ApiPage.class); ApiPage<String> apiPage = JSONUtil.toBean(data, ApiPage.class);
System.out.println(apiPage); System.out.println(apiPage);
//解析数据结果为List //解析数据结果为List
// List<PictureSourceResult> listResult = JSONUtil.toList(JSONObject.toJSONString(apiPage.getRecords()),PictureSourceResult.class); // List<PictureSourceResult> listResult = JSONUtil.toList(JSONObject.toJSONString(apiPage.getRecords()),PictureSourceResult.class);
@ -191,7 +193,7 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl); ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
List<OcrPicture> ocrPictureList = new ArrayList<>(); List<OcrPicture> ocrPictureList = new ArrayList<>();
ocrPictureList = ocrPictureService.list(ocrPictureLambdaQueryWrapper); ocrPictureList = ocrPictureService.list(ocrPictureLambdaQueryWrapper);
logger.debug("待执行下载图片数量:{}",ocrPictureList.size()); logger.debug("待执行下载图片数量:{}", ocrPictureList.size());
for (OcrPicture pictureOne : ocrPictureList) { for (OcrPicture pictureOne : ocrPictureList) {
// 创建 PictureImgToLocalEntity 对象并赋值 // 创建 PictureImgToLocalEntity 对象并赋值
PictureImgToLocalEntity entity = new PictureImgToLocalEntity(); PictureImgToLocalEntity entity = new PictureImgToLocalEntity();
@ -204,9 +206,9 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
// 将对象添加到下载图片队列中 // 将对象添加到下载图片队列中
boolean isAddedToQueue = false; boolean isAddedToQueue = false;
while (!isAddedToQueue) { while (!isAddedToQueue) {
isAddedToQueue =TaskQueue.pictureImgToLocalPushData(entity); isAddedToQueue = TaskQueue.pictureImgToLocalPushData(entity);
if (!isAddedToQueue) { if (!isAddedToQueue) {
logger.info("图片下载监测-添加队列失败，等待五秒后继续进行添加,图片id:{}",entity.getPictureId()); logger.info("图片下载监测-添加队列失败，等待五秒后继续进行添加,图片id:{}", entity.getPictureId());
Thread.sleep(5000); Thread.sleep(5000);
} }
} }
@ -218,5 +220,15 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
return true; return true;
} }
@Override
public List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter) {
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = this.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
} }

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/api/utils/ApiConfig.java
Unescape View File

@ -48,6 +48,11 @@ public class ApiConfig {
private String localImagePath; private String localImagePath;
/**
* 默认图片
*/
private String defaultPictureImg;
/** /**
* 检查配置文件是否 正确注入 * 检查配置文件是否 正确注入
* @return * @return

81
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -10,7 +10,6 @@ import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig; import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -66,16 +65,6 @@ public class ApiTestController {
if (StringUtils.isBlank(accountNoStr)) { if (StringUtils.isBlank(accountNoStr)) {
return "业务主体不能为空"; return "业务主体不能为空";
} }
// Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
// Long accountNo = Long.parseLong(request.getParameter("accountNo"));
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime = DateUtil.parse("2024-03-31 07:00:000","yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00","yyyy-MM-dd HH:mm:ss");
Instant einstant = dateTime1.toInstant();
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>()); List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(departments)) { if (CollectionUtils.isEmpty(departments)) {
@ -95,6 +84,16 @@ public class ApiTestController {
return "业务主体数据为空"; return "业务主体数据为空";
} }
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(Integer.parseInt(pageNo));
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
List<PictureSourceResult> totalList = Lists.newArrayList(); List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) { for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) { if (!departmentMap.containsKey(department.getParentid())) {
@ -103,8 +102,10 @@ public class ApiTestController {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno()); Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno()); Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, Integer.parseInt(pageNo), Integer.parseInt(pageSize), tenantNo, accountNo); List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) { if (CollectionUtils.isEmpty(resultList)) {
continue; continue;
} }
@ -117,11 +118,10 @@ public class ApiTestController {
if (pictureSourceResult.getLivePhoto() == null) { if (pictureSourceResult.getLivePhoto() == null) {
List<Map<String, Object>> livePhoto = new ArrayList<>(); List<Map<String, Object>> livePhoto = new ArrayList<>();
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("imgUrl", "http://47.93.59.251/api/image/default.png"); map.put("imgUrl", StringUtils.isEmpty(apiConfig.getDefaultPictureImg()) ? "http://47.93.59.251/api/image/default.png" : apiConfig.getDefaultPictureImg());
map.put("imgName", "default.png"); map.put("imgName", "default.png");
livePhoto.add(map); livePhoto.add(map);
pictureSourceResult.setLivePhoto(livePhoto); pictureSourceResult.setLivePhoto(livePhoto);
// continue;
} }
//将可以处理数据放入处理队列中 //将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult); TaskQueue.pictureDisposePushData(pictureSourceResult);
@ -151,8 +151,6 @@ public class ApiTestController {
String pageSize = request.getParameter("pageSize"); String pageSize = request.getParameter("pageSize");
String pageNo = request.getParameter("pageNo"); String pageNo = request.getParameter("pageNo");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
List<String> notAddedToQueueTaskIds = new ArrayList<>(); List<String> notAddedToQueueTaskIds = new ArrayList<>();
@ -183,6 +181,10 @@ public class ApiTestController {
int totalSize = 0; int totalSize = 0;
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss"); DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss"); DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) { while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo); int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true; boolean hasNextPage = true;
@ -201,8 +203,13 @@ public class ApiTestController {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno()); Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno()); Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, startInstant, endInstant, currentPage, tenantNo, accountNo); List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) { if (CollectionUtils.isEmpty(resultList)) {
continue; continue;
} }
@ -278,45 +285,5 @@ public class ApiTestController {
} }
return message; return message;
} }
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, Instant startInstant, Instant endInstant, Integer currentPage, Long tenantNo, Long accountNo) {
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
} }

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrFrontConf.java
Unescape View File

@ -58,4 +58,9 @@ public class OcrFrontConf implements java.io.Serializable {
* 拉取租户信息 * 拉取租户信息
*/ */
private String tenantSourceUrl; private String tenantSourceUrl;
/**
* 默认图片
*/
private String defaultPictureImg;
} }

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrFrontConfServiceImpl.java
Unescape View File

@ -33,6 +33,7 @@ public class OcrFrontConfServiceImpl extends BaseServiceImpl<OcrFrontConfMapper,
apiConfig.setConnectTimeout(frontConf.getWlyConnectTimeout()); apiConfig.setConnectTimeout(frontConf.getWlyConnectTimeout());
apiConfig.setReadTimeout(frontConf.getWlyReadTimeout()); apiConfig.setReadTimeout(frontConf.getWlyReadTimeout());
apiConfig.setLocalImagePath(frontConf.getWlyLocalImagePath()); apiConfig.setLocalImagePath(frontConf.getWlyLocalImagePath());
apiConfig.setDefaultPictureImg(frontConf.getDefaultPictureImg());
} }
} }
} }

31
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskQueue.java
Unescape View File

@ -2,9 +2,9 @@ package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import org.slf4j.Logger; import com.google.common.collect.Sets;
import org.slf4j.LoggerFactory;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
/** /**
@ -15,7 +15,6 @@ import java.util.concurrent.LinkedBlockingQueue;
* @date 2024/03/14 3:08 * @date 2024/03/14 3:08
*/ */
public class TaskQueue { public class TaskQueue {
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
/** /**
* 无量云接口数据处理 任务队列 * 无量云接口数据处理 任务队列
@ -27,17 +26,25 @@ public class TaskQueue {
*/ */
public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue(); public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
public static volatile Set<String> resultSet = Sets.newConcurrentHashSet();
/** /**
* 无量云接口数据处理 push 队列值 * 无量云接口数据处理 push 队列值
* @return *
* @param pictureSourceResult //三方拉取 获取到的图片对象 * @param pictureSourceResult //三方拉取 获取到的图片对象
* @return
*/ */
public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult){ public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult) {
try { try {
Long taskId = pictureSourceResult.getTaskId();
if (resultSet.contains(String.valueOf(taskId))) {
return true;
}
resultSet.add(String.valueOf(taskId));
pictureDisposeQueue.put(pictureSourceResult); pictureDisposeQueue.put(pictureSourceResult);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error("任务队列添加异常:{}",e.getMessage());
return false; return false;
} }
return true; return true;
@ -45,23 +52,24 @@ public class TaskQueue {
/** /**
* 无量云接口数据处理 拉取队列值 * 无量云接口数据处理 拉取队列值
*
* @return * @return
*/ */
public static PictureSourceResult pictureDisposePullData(){ public static PictureSourceResult pictureDisposePullData() {
return (PictureSourceResult) pictureDisposeQueue.poll(); return (PictureSourceResult) pictureDisposeQueue.poll();
} }
/** /**
* 图片缓存本地队列 push 队列值 * 图片缓存本地队列 push 队列值
* @return *
* @param pictureImgToLocalEntity //图片下载对象 * @param pictureImgToLocalEntity //图片下载对象
* @return
*/ */
public static boolean pictureImgToLocalPushData(PictureImgToLocalEntity pictureImgToLocalEntity){ public static boolean pictureImgToLocalPushData(PictureImgToLocalEntity pictureImgToLocalEntity) {
try { try {
pictureImgToLocalQueue.put(pictureImgToLocalEntity); pictureImgToLocalQueue.put(pictureImgToLocalEntity);
} catch (InterruptedException e) { } catch (InterruptedException e) {
logger.error("图片下载任务队列添加异常:{}",e.getMessage());
return false; return false;
} }
return true; return true;
@ -69,9 +77,10 @@ public class TaskQueue {
/** /**
* 无量云接口图片数据处理 拉取队列值 * 无量云接口图片数据处理 拉取队列值
*
* @return * @return
*/ */
public static PictureImgToLocalEntity pictureImgToLocalPullData(){ public static PictureImgToLocalEntity pictureImgToLocalPullData() {
return (PictureImgToLocalEntity) pictureImgToLocalQueue.poll(); return (PictureImgToLocalEntity) pictureImgToLocalQueue.poll();
} }
} }

32
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/timerJob/PictureSourceTimerJob.java
Unescape View File

@ -7,7 +7,6 @@ import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi; import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -56,11 +55,8 @@ public class PictureSourceTimerJob {
//组装拉取参数 //组装拉取参数
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter(); PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
// pictureSourceParameter.setStartTime();
// pictureSourceParameter.setEndTime();
pictureSourceParameter.setPageNo(1); pictureSourceParameter.setPageNo(1);
pictureSourceParameter.setPageSize(200); pictureSourceParameter.setPageSize(200);
// pictureSourceParameter.setAccountNo();
Calendar calendar = Calendar.getInstance(); Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_MONTH, -1); calendar.add(Calendar.DAY_OF_MONTH, -1);
@ -69,6 +65,11 @@ public class PictureSourceTimerJob {
String endDateStr = dateFormat.format(yesterday).split(" ")[0] + " 23:59:59"; // 结束时间为昨天的23点59分59秒 String endDateStr = dateFormat.format(yesterday).split(" ")[0] + " 23:59:59"; // 结束时间为昨天的23点59分59秒
DateTime dateTime = DateUtil.parse(startDateStr, "yyyy-MM-dd HH:mm:ss"); DateTime dateTime = DateUtil.parse(startDateStr, "yyyy-MM-dd HH:mm:ss");
DateTime dateTime1 = DateUtil.parse(endDateStr, "yyyy-MM-dd HH:mm:ss"); DateTime dateTime1 = DateUtil.parse(endDateStr, "yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
//1.拉取数据 //1.拉取数据
List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>()); List<Department> departments = departmentMybatisDao.selectList(new QueryWrapper<>());
@ -93,8 +94,10 @@ public class PictureSourceTimerJob {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno()); Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno()); Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(pictureSourceParameter, dateTime, dateTime1, tenantNo, accountNo); List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) { if (CollectionUtils.isEmpty(resultList)) {
continue; continue;
} }
@ -114,23 +117,4 @@ public class PictureSourceTimerJob {
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条"); System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
} }
private List<PictureSourceResult> getPictureSourceResultApiPage(PictureSourceParameter pictureSourceParameter, DateTime dateTime, DateTime dateTime1, Long tenantNo, Long accountNo) {
//创建查询参数对象
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
} }

37
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/timerJob/PullPictureDataJob.java
Unescape View File

@ -7,7 +7,6 @@ import cn.jyjz.xiaoyao.admin.dataobject.Department;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi; import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
@ -66,6 +65,16 @@ public class PullPictureDataJob {
departmentMap.put(department.getId(), department); departmentMap.put(department.getId(), department);
} }
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
List<PictureSourceResult> totalList = Lists.newArrayList(); List<PictureSourceResult> totalList = Lists.newArrayList();
for (Department department : departments) { for (Department department : departments) {
if (!departmentMap.containsKey(department.getParentid())) { if (!departmentMap.containsKey(department.getParentid())) {
@ -74,8 +83,10 @@ public class PullPictureDataJob {
Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno()); Long tenantNo = Long.valueOf(departmentMap.get(department.getParentid()).getDeptno());
Long accountNo = Long.valueOf(department.getDeptno()); Long accountNo = Long.valueOf(department.getDeptno());
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
List<PictureSourceResult> resultList = getPictureSourceResultApiPage(dateTime, dateTime1, pageNo, pageSize, tenantNo, accountNo); List<PictureSourceResult> resultList = prevailCloudApi.getPictureSourceResultApiPage(pictureSourceParameter);
if (CollectionUtils.isEmpty(resultList)) { if (CollectionUtils.isEmpty(resultList)) {
continue; continue;
} }
@ -94,26 +105,4 @@ public class PullPictureDataJob {
} }
System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条"); System.out.println("当前区间内图片总数:" + count + "条，可处理数据:" + size + "条");
} }
private List<PictureSourceResult> getPictureSourceResultApiPage(DateTime dateTime, DateTime dateTime1, Integer pageNo, Integer pageSize, Long tenantNo, Long accountNo) {
//创建查询参数对象
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
Instant binstant = dateTime.toInstant();
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
pictureSourceParameter.setEndTime(Date.from(einstant));
pictureSourceParameter.setPageNo(pageNo);
pictureSourceParameter.setPageSize(pageSize);
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
ApiPage<PictureSourceResult> apiPage = null;
try {
apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
} catch (Exception e) {
e.printStackTrace();
}
return apiPage == null ? Lists.newArrayList() : apiPage.getRecords();
}
} }

Write Preview
Loading…
Cancel
Save