Compare commits

..

1 Commits

@ -124,9 +124,6 @@ public class SearchQuery implements java.io.Serializable{
public void addGreaterEqual(String field,Date value){
addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
}
public void addGreaterEqual(String field,Long value){
addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
}
public void addLess(String field,Integer value){
addQuery(field,value, SearchQueryConstant.SEARCH_LESS);
}
@ -158,10 +155,6 @@ public class SearchQuery implements java.io.Serializable{
addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
}
public void addLessEqual(String field,Long value){
addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
}
public void addRulesOp(String field,String rulesOp){
SearchQueryForm searchQueryForm = new SearchQueryForm();
if(query.get(field)!=null){

@ -60,7 +60,7 @@ public class JwtAuthenticationFilter extends OncePerRequestFilter {
String[] excludespath = systemopenurlService.getOpenUrl();
//不需要过滤的url
String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs","/modeler","/app"};
String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs"};
String[] urls = ArrayUtils.addAll(excludespath, excludesOther);

@ -1317,14 +1317,14 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
query.put("searchassignee", user.getLoginname());
query.putAll(searchQuery);
if (!StringUtils.isBlank(userSearchId)) {
OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId);
SearchQuery additionalQuery = new SearchQuery();
for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) {
this.formattingSearchChild1(additionalQuery, usersearchchild);
}
query.putAll(additionalQuery.getQuery());
}
// if (!StringUtils.isBlank(userSearchId)) {
// OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId);
// SearchQuery additionalQuery = new SearchQuery();
// for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) {
// this.formattingSearchChild1(additionalQuery, usersearchchild);
// }
// query.putAll(additionalQuery.getQuery());
// }
Long count = ocrTaskchildPictureService.listmytaskCount(query);
@ -1953,134 +1953,132 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
return queryWrapper;
}
private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
switch (usersearchchild.getSearchRelationType()) {
case "where":
// 添加 "where" 查询条件
this.formattingSearchType1(searchQuery, usersearchchild);
break;
case "and":
// 添加 "and" 查询条件
searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and");
SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild);
searchQuery.setSearchQuery(andQuery);
break;
case "or":
// 添加 "or" 查询条件
searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or");
SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild);
searchQuery.setSearchQuery(orQuery);
break;
default:
break;
}
return searchQuery;
}
private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
//查询类型 相等true 不相等fase
boolean queryType = "eq".equals(usersearchchild.getSearchtype());
// 1.获取检查数据,确定数据类型
String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue());
List<String> arrayData = null; //集合型数据
List<Integer> sectionInts = null; //纯数字区间型数据
String[] sections = null; //字符串区间型数据
DateTime beginTime = null; //日期时间区间型数据
DateTime endTime = null; //日期时间区间型数据
// 2.根据数据类型,解构数据
switch (valueType) {
case "array":
//处理多文件
String[] datas = usersearchchild.getSearchvalue().split(",");
arrayData = Arrays.asList(datas);
break;
case "section":
//格式化数据,仅在做区间的时候去除空格,其他地方不进行空格去除.
sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
//校验是否纯数字,纯数字的话要转成int,否则比对时会有问题
sectionInts = new ArrayList<>();
for (String data : sections) {
if (data.matches("\\d+")) {
sectionInts.add(Integer.parseInt(data));
}
}
break;
case "time":
//格式化时间数据
String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
beginTime = DateUtil.parse(times[0]+" 00:00:00", "yyyy/MM/dd HH:mm:ss");
endTime = DateUtil.parse(times[1]+" 23:59:59", "yyyy/MM/dd HH:mm:ss");
break;
default:
}
// 3.根据查询类型 组装查询条件 相等
switch (valueType) {
case "array":
//集合数据
//相等
if (queryType) {
searchQuery.addIn(usersearchchild.getSearchfield(), arrayData);
break;
}
//不相等
searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData);
break;
case "section":
//数字区间查询
if (sectionInts != null && sectionInts.size() == 2) {
if (queryType) {
//相等
searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1));
break;
}
//不相等
searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
break;
}
//字符串区间查询
if (queryType) {
//相等
searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]);
searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]);
break;
}
//不相等
searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]);
searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]);
break;
case "time":
//时间区间数据
//相等
if (queryType) {
searchQuery.addGreaterEqual(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
searchQuery.addLessEqual(usersearchchild.getSearchfield()+"End", endTime.getTime());
break;
}
//不相等
searchQuery.addRulesOp(usersearchchild.getSearchfield()+"Start","or");
searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime());
break;
default:
if (queryType) {
searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
break;
}
searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
}
return searchQuery;
}
// private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
// switch (usersearchchild.getSearchRelationType()) {
// case "where":
// // 添加 "where" 查询条件
// this.formattingSearchType1(searchQuery, usersearchchild);
// break;
// case "and":
// // 添加 "and" 查询条件
// searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and");
// SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild);
// searchQuery.setSearchQuery(andQuery);
// break;
// case "or":
// // 添加 "or" 查询条件
// searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or");
// SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild);
// searchQuery.setSearchQuery(orQuery);
// break;
// default:
// break;
// }
// return searchQuery;
// }
//
// private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
// //查询类型 相等true 不相等fase
// boolean queryType = "eq".equals(usersearchchild.getSearchtype());
//
// // 1.获取检查数据,确定数据类型
// String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue());
// List<String> arrayData = null; //集合型数据
// List<Integer> sectionInts = null; //纯数字区间型数据
// String[] sections = null; //字符串区间型数据
// DateTime beginTime = null; //日期时间区间型数据
// DateTime endTime = null; //日期时间区间型数据
//
// // 2.根据数据类型,解构数据
// switch (valueType) {
// case "array":
// //处理多文件
// String[] datas = usersearchchild.getSearchvalue().split(",");
// arrayData = Arrays.asList(datas);
// break;
// case "section":
// //格式化数据,仅在做区间的时候去除空格,其他地方不进行空格去除.
// sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
// //校验是否纯数字,纯数字的话要转成int,否则比对时会有问题
// sectionInts = new ArrayList<>();
// for (String data : sections) {
// if (data.matches("\\d+")) {
// sectionInts.add(Integer.parseInt(data));
// }
// }
// break;
// case "time":
// //格式化时间数据
// String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
// beginTime = DateUtil.parse(times[0], "yyyy/MM/dd");
// endTime = DateUtil.parse(times[1], "yyyy/MM/dd");
// break;
// default:
// }
//
// // 3.根据查询类型 组装查询条件 相等
// switch (valueType) {
// case "array":
// //集合数据
// //相等
// if (queryType) {
// searchQuery.addIn(usersearchchild.getSearchfield(), arrayData);
// break;
// }
// //不相等
// searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData);
// break;
//
// case "section":
// //数字区间查询
// if (sectionInts != null && sectionInts.size() == 2) {
// if (queryType) {
// //相等
// searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0));
// searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1));
// break;
// }
// //不相等
// searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
// searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
// break;
// }
//
// //字符串区间查询
// if (queryType) {
// //相等
// searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]);
// searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]);
// break;
// }
// //不相等
// searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]);
// searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]);
// break;
//
//
// case "time":
// //时间区间数据
// //相等
// if (queryType) {
// searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), beginTime);
// searchQuery.addLessEqual(usersearchchild.getSearchfield(), endTime);
// break;
// }
// //不相等
// searchQuery.addLess(usersearchchild.getSearchfield(), beginTime);
// searchQuery.addGreater(usersearchchild.getSearchfield(), endTime);
// break;
//
// default:
// if (queryType) {
// searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
// break;
// }
// searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
// }
// return searchQuery;
// }
}

@ -1,6 +1,5 @@
package cn.jyjz.xiaoyao.ocr.controller;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
@ -8,14 +7,12 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -26,7 +23,6 @@ import java.util.List;
* @version 1.0
* @date 2024/3/14 19:47
*/
@Slf4j
@RestController
@RequestMapping("/test/api")
public class ApiTestController {
@ -75,98 +71,4 @@ public class ApiTestController {
return "处理失败";
}
@GetMapping("psTest01")
public String psTset01(HttpServletRequest request){
String beginStr = request.getParameter("beginStr");
String endStr = request.getParameter("endStr");
String pageSize = request.getParameter("pageSize");
String pageNo = request.getParameter("pageNo");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
pictureSourceParameter.setTenantNo(1714548920583360512L);
pictureSourceParameter.setAccountNo(1714887674875514880L);
List<String> notAddedToQueueTaskIds = new ArrayList<>();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
int totalCount = 0;
int totalLocalCount = 0;
int totalSize = 0;
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true;
boolean isCountAdded = false;
while (hasNextPage) {
// 设置当天的开始和结束时间
Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
int retryCount = 0;
boolean addedToQueue = false;
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
size++;
// 尝试将数据加入队列
while (!addedToQueue && retryCount < 5) { // 最多重试5次
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult);
if (!addedToQueue) {
Thread.sleep(60000);
retryCount++;
}
}
if (!addedToQueue) {
log.error("无法将数据添加到队列,开始时间:{}任务ID{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
}
}
}
totalLocalCount += localCount;
totalSize += size;
if (!isCountAdded) {
totalCount += count;
isCountAdded = true; // 标记当天的统计信息已经被累加过了
}
// 输出当前日期范围内的统计信息
log.error("{}图片总数:{}条,本次获取:{}条,可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
}
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
}
log.error("未加入队列的任务ID列表{}",notAddedToQueueTaskIds);
log.error("当前区间内图片总数:{}条,本次获取:{}条,可处理数据:{}条",totalCount,totalLocalCount,totalSize);
} catch (Exception e) {
e.printStackTrace();
}
}
});
thread.start();
return "开启入库";
}
}

@ -81,16 +81,10 @@ public class OcrCheckDuplicateController {
}
//调用查询参数解析方法解析为queryWrapper
queryWrapper = SearchParaFormatting.searchParaParse(queryWrapper, paraMap);
//提报人模糊搜索
String upUserName = req.getParameter("upUserName");
if(StringUtils.isNotBlank(upUserName)){
queryWrapper.like("upName",upUserName);
}
queryWrapper.eq("tenant_id", tenantId);
queryWrapper.eq("suspiciousfile", "0");
queryWrapper.orderBy(true,true,"submit_date_timestamp");
//本地图片地址,不能为空,为空时查重会有问题
queryWrapper.isNotNull("local_pictrue_url");
List<OcrPicture> pageList = ocrPictureService.listPage(queryWrapper);
if (null != pageList && !pageList.isEmpty()) {
if (pageList.size() == 1) {
@ -120,9 +114,6 @@ public class OcrCheckDuplicateController {
OcrCheckDuplicate dto = this.ocrCheckDuplicateService.getCheckDuplicateStatus(checkDuplicateNo);
if (null != dto) {
if(dto.getStatus()==2){
return ResultVoUtil.success("查重任务执行完毕",dto);
}
return ResultVoUtil.success(dto);
} else {
return ResultVoUtil.error();

@ -311,8 +311,6 @@ public class OcrPictureController extends BaseController {
queryWrapper = SearchParaFormatting.searchParaParse(queryWrapper, paraMap);
queryWrapper.eq("tenant_id", tenantId);
//本地图片地址,不能为空,为空时查重会有问题
queryWrapper.isNotNull("local_pictrue_url");
Page<OcrPicture> page = new Page<OcrPicture>(pageNo, pageSize);
IPage<OcrPicture> pageList = ocrPictureService.page(page, queryWrapper);

@ -65,10 +65,9 @@ public class OcrTaskchildDubiousFileController {
@ApiOperation(value="可疑文件列表查询", notes="可疑文件列表查询")
@RequestMapping(value = "/dubiousfilelist", method = RequestMethod.GET)
public ResultVo<Page<OcrTaskChildPictureVo>> dubiousfilelist(HttpServletRequest request,
@RequestParam(value = "pageNo", required = false, defaultValue = "0") Integer pageNo,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
@RequestParam(value = "orderByUptime",required = false) String orderByUptime,
@RequestParam(value = "orderBysimilarity",required = false) String orderBysimilarity) {
@RequestParam(value = "pageNo", required = false, defaultValue = "0") Integer pageNo,
@RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize,
@RequestParam(value = "orderbyname", required = true, defaultValue = "asc") String orderbyname) {
String tenantId = request.getHeader("X-Tenant-Id");
@ -98,7 +97,7 @@ public class OcrTaskchildDubiousFileController {
//
// //IPage<OcrPicture> page = ocrpictureservice.page(iPage, wrapper);
Page<OcrTaskChildPictureVo> page = ocrTaskchildDubiousFileService.dubiousfilelist(pageNo,pageSize,orderByUptime,orderBysimilarity,userToken,tenantId);
Page<OcrTaskChildPictureVo> page = ocrTaskchildDubiousFileService.dubiousfilelist(pageNo,pageSize,orderbyname,userToken,tenantId);
return ResultVoUtil.success(page);
}
@ -131,11 +130,9 @@ public class OcrTaskchildDubiousFileController {
dubiouswrapper.eq("userid",userToken.getUserid()).eq("picture_id",childpictureid);
List<OcrTaskchildDubiousFile> dubious = ocrTaskchildDubiousFileService.list(dubiouswrapper);
if(dubious != null){
for (OcrTaskchildDubiousFile ocrTaskchildDubiousFile : dubious) {
fileList.add(ocrTaskchildDubiousFile);
}
OcrTaskchildDubiousFile one = ocrTaskchildDubiousFileService.getOne(dubiouswrapper);
if(one != null){
fileList.add(one);
}
}

@ -41,8 +41,7 @@ public class OcrCheckDuplicateResult implements Serializable {
@ApiModelProperty(value = "图片id")
private Long pictureId;
@ApiModelProperty(value = "当前图片与任务包其他图片比对结果 (废弃,入库导致数据量过大)")
@TableField(exist = false)
@ApiModelProperty(value = "当前图片与任务包其他图片比对结果")
private String compareResultJson;
@ApiModelProperty(value = "查重任务id")

@ -311,10 +311,6 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(value = "similarity_score")
private Integer similarityscore;
@ApiModelProperty(value = "下载到本地,失败次数")
@TableField(value = "download_error_count")
private Integer downloadErrorCount;
@ApiModelProperty(value = "图片查重的比对图片集合")
@TableField(exist = false)
private List<OcrPicture> listCom;
@ -371,9 +367,6 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(exist = false)
private String taskId;
//保存当前登录用户的数据权限范围的搜索条件
@TableField(exist = false)
@JsonIgnore

@ -1,23 +0,0 @@
package cn.jyjz.xiaoyao.ocr.dto;
import lombok.Data;
/**
* ClassName ResultDto$.java
* Description:
* Author scl.
* Date 2024/4/7 21:40
*/
@Data
public class ResultDto {
private Integer status;
private String message;
private Long timestamp;
private String signature;
private String data;
}

@ -17,6 +17,6 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface IOcrTaskchildDubiousFileService extends IService<OcrTaskchildDubiousFile> {
Page<OcrTaskChildPictureVo> dubiousfilelist(Integer pageNo, Integer pageSize,String orderByUptime,String orderBysimilarity,UserToken userToken, String tenantId);
Page<OcrTaskChildPictureVo> dubiousfilelist(Integer pageNo, Integer pageSize, String orderbyname, UserToken userToken, String tenantId);
}

@ -61,8 +61,6 @@ public interface OcrPictureService extends BaseService<OcrPicture> {
*/
int savePictures(List<OcrPicture> ocrPictureList);
void savePicturesInfo(OcrPicture ocrPicture);
ResultVo createPackageTask(List<OcrPicture> pageList, String tenantId, UserToken userToken, String searchMonth, HttpServletRequest req, String buessinessno, Map<String, Object> paramMap);
Map<String,String> getOcrPictureClassifyAndHash(String img);

@ -106,7 +106,6 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
//异步调用查重任务
checkDuplicateService.comparePictureResultSave(ocrPictureList, sysUser, id);
checkDuplicate.setStatus(2);
checkDuplicate.setCompletionTime(System.currentTimeMillis());
} catch (Exception e) {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
@ -116,7 +115,6 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
logger.error("异步执行图片比对异常:", e);
logger.error("异步执行图片比对异常:", e.getMessage());
checkDuplicate.setStatus(3);
checkDuplicate.setCompletionTime(System.currentTimeMillis());
} finally {
checkDuplicateService.updateById(checkDuplicate);
}
@ -178,8 +176,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
* 6.ocr_picturesimilarity_score
*/
@Override
// @Transactional(rollbackFor = Exception.class)
public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId){
@Transactional(rollbackFor = Exception.class)
public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId) throws Exception {
List<OcrCheckDuplicateResult> saveList = new ArrayList<>();
//需要创建任务的图片集合
@ -198,10 +196,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) {
String url = ocrPicture.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
// if(StringUtils.isBlank(ocrPicture.getImgurl())){
// }
// url = ocrPicture.getImgurl();
continue;
if(StringUtils.isBlank(ocrPicture.getImgurl())){
continue;
}
url = ocrPicture.getImgurl();
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
@ -228,11 +226,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
logger.info("图片没有hash值进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext));
String url = ocrPictureNext.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
// if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
// continue;
// }
// url = ocrPictureNext.getImgurl();
continue;
if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
continue;
}
url = ocrPictureNext.getImgurl();
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
if (map != null) {
@ -291,58 +288,33 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if(similarityGroupOrderMap.get(ocrPicture.getId().toString())!=null){
ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString()));
}
//1.3开启入库操作,不能进行批量入库,会导致内存被拉崩
saveList.add(ocrCheckDuplicateResult);
List<OcrPicture> pictures = new ArrayList<>();
pictures.add(ocrPicture);
try {
this.saveCheckDuplicateData(checkDuplicateId,saveList,pictures,ocrPictureList.get(0).getTenantId().toString(),sysUser);
} catch (Exception e) {
logger.error("比对结果入库异常! error:{}",e.getMessage());
}
saveList = new ArrayList<>();
}
}
}
/**
*
* @return
*/
@Transactional(rollbackFor = Exception.class)
public boolean saveCheckDuplicateData(Long checkDuplicateId,List<OcrCheckDuplicateResult> saveList,List<OcrPicture> ocrPictureList, String tenantId,UserToken sysUser) throws Exception {
//1.批量入库
logger.debug("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
logger.error("入库比对结果失败,{}",checkDuplicateId);
throw new Exception("批量入库比对结果失败!");
}
//2.开始更新图片最高阈值
logger.debug("查重任务:{},更新最高阈值",checkDuplicateId);
b = this.updatePictureSimilarityScore(saveList);
if (!b) {
logger.error("更新最高阈值,{}",checkDuplicateId);
throw new Exception("更新最高阈值失败!");
}
//3.开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
b = this.updateCompareToPictureDuplicateHis(saveList);
if (!b) {
logger.error("更新图片历史比对结果失败,{}",checkDuplicateId);
throw new Exception("更新图片历史比对结果失败!");
}
//4.开始更新 图片历史比对数据结果
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
return true;
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
throw new Exception("批量入库比对结果失败!");
}
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
}
@Override
public Boolean removeCheckDuplicate(String checkDuplicateNo, String tenantId) {
QueryWrapper queryWrapper = new QueryWrapper();
@ -506,7 +478,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
Page<OcrPicture> pageVO = new Page<OcrPicture>(pageNo, pageSize);
pageVO.setRecords(collect);
pageVO.setTotal(ocrCheckDuplicateResults.size());
pageVO.setTotal(records.size());
return pageVO;
}

@ -26,7 +26,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import liquibase.pro.packaged.E;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -39,9 +38,7 @@ import javax.imageio.ImageIO;
import javax.servlet.http.HttpServletRequest;
import java.awt.color.ColorSpace;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.net.URL;
import java.net.URLConnection;
@ -668,7 +665,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
}
@Override
public int savePictures(List<OcrPicture> ocrPictureList){
public int savePictures(List<OcrPicture> ocrPictureList) {
//1.字典数据 解析入库
boolean b = dataDictionaryUtil.pictureDataDictionarySaveArray(ocrPictureList);
@ -676,26 +673,16 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
//2.调整图片地址,补齐域名
for (OcrPicture picture : ocrPictureList) {
picture.setCreateTime(System.currentTimeMillis()); //获取创建时间戳
// picture.setImgurl(interfaceDomain + picture.getImgurl());
}
try {
boolean saveBatch = false;
synchronized(lock) {
// 同步的代码块
saveBatch = this.saveBatch(ocrPictureList);
}
//3.添加图片信息
if (saveBatch) {
// this.savePicturesInfo(ocrPictureList);
this.convertImage(ocrPictureList);
return ocrPictureList.size();
}
log.error("存储图片失败:{}",JSONObject.toJSON(ocrPictureList));
}catch (Exception e){
log.error("存储图片异常:{}",e.getMessage());
boolean saveBatch = this.saveBatch(ocrPictureList);
//3.添加图片信息
if (saveBatch) {
this.savePicturesInfo(ocrPictureList);
this.convertImage(ocrPictureList);
return ocrPictureList.size();
}
return 0;
}
@ -703,7 +690,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
List<OcrPictureInfo> list = new ArrayList<>();
for (OcrPicture ocrPicture : ocrPictureList) {
if (StringUtils.isNotBlank(ocrPicture.getImgurl())) {
OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl(),true);
OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl());
if (ObjectUtil.isNotEmpty(imageInfo)) {
imageInfo.setPictureId(ocrPicture.getId());
imageInfo.setImgName(ocrPicture.getImgname());
@ -714,49 +701,18 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
ocrPictureInfoService.saveBatch(list);
}
/**
*
* @param ocrPicture
*/
@Override
public void savePicturesInfo(OcrPicture ocrPicture) {
boolean isConnection = true;
String url = ocrPicture.getImgurl();
if(!StringUtils.isEmpty(ocrPicture.getLocalpictrueurl())){
url = ocrPicture.getLocalpictrueurl();
isConnection = false;
}
OcrPictureInfo imageInfo = getImageInfo(url,isConnection);
imageInfo.setPictureId(ocrPicture.getId());
imageInfo.setImgName(ocrPicture.getImgname());
ocrPictureInfoService.save(imageInfo);
}
/**
*
* @param imageUrl
* @param isConnection url
* @return
*/
private OcrPictureInfo getImageInfo(String imageUrl,boolean isConnection) {
private OcrPictureInfo getImageInfo(String imageUrl) {
try {
BufferedImage image = null;
Integer imageSize = 0;
if(isConnection){
// 创建URL对象
URL url = new URL(imageUrl);
// 打开连接
URLConnection urlConnection = url.openConnection();
// 获取图片大小
imageSize = urlConnection.getContentLength();
// 使用ImageIO读取图片
image= ImageIO.read(url);
}else{
File file = new File(imageUrl);
image = ImageIO.read(file);
imageSize =(int)file.length();
}
// 创建URL对象
URL url = new URL(imageUrl);
// 打开连接
URLConnection urlConnection = url.openConnection();
// 获取图片大小
Integer imageSize = urlConnection.getContentLength();
// 使用ImageIO读取图片
BufferedImage image = ImageIO.read(url);
if (image != null) {
OcrPictureInfo info = new OcrPictureInfo();
@ -795,14 +751,12 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
log.info("Cannot read image from URL.");
return null;
}
} catch (Exception e) {
} catch (IOException e) {
log.error("Error: " + e.getMessage());
}
return null;
}
private void convertImage(List<OcrPicture> list) {
for (OcrPicture ocrPicture : list) {
PictureImgToLocalEntity entity = new PictureImgToLocalEntity();
@ -810,22 +764,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
entity.setImgUrl(ocrPicture.getImgurl());
String imgurl = ocrPicture.getImgurl();
entity.setLocalPath(localImagePath + imgurl.substring(imgurl.lastIndexOf("/")));
boolean addedToQueue = false;
int count = 1;
while (!addedToQueue){
try {
addedToQueue = TaskQueue.pictureImgToLocalPushData(entity);
if(addedToQueue) {
break;
}
Thread.sleep(5000);
} catch (InterruptedException e) {
log.error("放入图片下载队列,等待休眠异常:{}",e.getMessage());
}
count++;
}
TaskQueue.pictureImgToLocalPushData(entity);
}
}

@ -2,7 +2,6 @@ package cn.jyjz.xiaoyao.ocr.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.jyjz.flowable.common.ProcessConstant;
import cn.jyjz.xiaoyao.common.base.util.StringUtils;
import cn.jyjz.xiaoyao.common.base.vo.ResultVoUtil;
import cn.jyjz.xiaoyao.common.base.vo.UserToken;
import cn.jyjz.xiaoyao.ocr.dataDao.OcrTaskchildDubiousFileMapper;
@ -48,14 +47,13 @@ public class OcrTaskchildDubiousFileServiceImpl extends ServiceImpl<OcrTaskchild
*
* @param pageNo
* @param pageSize
* @param orderByUptime
* @param orderBysimilarity
* @param orderbyname
* @param userToken
* @param tenantId
* @return
*/
@Override
public Page<OcrTaskChildPictureVo> dubiousfilelist(Integer pageNo, Integer pageSize,String orderByUptime,String orderBysimilarity,UserToken userToken, String tenantId) {
public Page<OcrTaskChildPictureVo> dubiousfilelist(Integer pageNo, Integer pageSize, String orderbyname, UserToken userToken, String tenantId) {
Page<OcrTaskChildPictureVo> objectPage = new Page<>(Integer.valueOf(pageNo), Integer.valueOf(pageSize));
@ -76,41 +74,13 @@ public class OcrTaskchildDubiousFileServiceImpl extends ServiceImpl<OcrTaskchild
List<OcrTaskChildPictureVo> taskChildPictures = ocrTaskchildPictureService.getDubiousfileList(new ArrayList<>(pictureIdList), userNodeType);
if (CollUtil.isNotEmpty(taskChildPictures)) {
Comparator<OcrTaskChildPictureVo> comparator = null;
// 判断 orderByUptime 是否不为空
if (orderByUptime != null && StringUtils.isNotEmpty(orderByUptime)) {
if ("asc".equals(orderByUptime)) {
comparator = Comparator.comparing(OcrTaskChildPictureVo::getSubmitDateTimestamp);
} else if ("desc".equals(orderByUptime)) {
comparator = Comparator.comparing(OcrTaskChildPictureVo::getSubmitDateTimestamp).reversed();
}
}
// 判断 orderBysimilarity 是否不为空
if (orderBysimilarity != null && StringUtils.isNotEmpty(orderBysimilarity)) {
Comparator<OcrTaskChildPictureVo> similarityComparator = null;
if ("asc".equals(orderBysimilarity)) {
similarityComparator = Comparator.comparing(OcrTaskChildPictureVo::getSimilarityScore);
} else if ("desc".equals(orderBysimilarity)) {
similarityComparator = Comparator.comparing(OcrTaskChildPictureVo::getSimilarityScore).reversed();
}
// 如果 orderByUptime 和 orderBysimilarity 都不为空,则先按照 orderBysimilarity 排序
if (comparator != null) {
comparator = comparator.thenComparing(similarityComparator);
} else {
comparator = similarityComparator;
}
}
// 进行排序
if (comparator != null) {
taskChildPictures.sort(comparator);
//按时间字段排序
if ("desc".equals(orderbyname)) {
taskChildPictures.sort(Comparator.comparing(OcrTaskChildPictureVo::getSubmitDateTimestamp).reversed());
} else {
taskChildPictures.sort(Comparator.comparing(OcrTaskChildPictureVo::getSubmitDateTimestamp));
}
List<OcrTaskChildPictureVo> records = taskChildPictures.stream().skip((pageNo - 1) * pageSize).limit(pageSize).collect(Collectors.toList());
objectPage.setRecords(records);
objectPage.setTotal(taskChildPictures.size());

@ -2,7 +2,6 @@ package cn.jyjz.xiaoyao.ocr.service.impl;
import cn.hutool.core.map.MapBuilder;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.http.HttpUtil;
import cn.jyjz.flowable.common.ProcessConstant;
import cn.jyjz.flowable.common.SystemConstantsOa;
import cn.jyjz.flowable.domain.vo.FlowModelVO;
@ -23,7 +22,6 @@ import cn.jyjz.xiaoyao.ocr.dataDao.OcrTaskchildPictureMybatisDao;
import cn.jyjz.xiaoyao.ocr.dataobject.*;
import cn.jyjz.xiaoyao.ocr.dto.ApproveNodeDto;
import cn.jyjz.xiaoyao.ocr.dto.FlowResultDto;
import cn.jyjz.xiaoyao.ocr.dto.ResultDto;
import cn.jyjz.xiaoyao.ocr.dto.UserapproveDto;
import cn.jyjz.xiaoyao.ocr.service.IOcrPictureDuplicateHisService;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureInfoService;
@ -31,7 +29,6 @@ import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrTaskchildPictureService;
import cn.jyjz.xiaoyao.ocr.vo.OcrTaskChildPictureVo;
import cn.jyjz.xiaoyao.ocr.vo.PictureDetailVo;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@ -45,8 +42,6 @@ import org.flowable.bpmn.model.FlowElement;
import org.flowable.bpmn.model.UserTask;
import org.flowable.engine.history.HistoricProcessInstance;
import org.flowable.task.api.Task;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.mock.web.MockHttpServletRequest;
import org.springframework.stereotype.Service;
import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseServiceImpl;
@ -65,9 +60,6 @@ import java.util.stream.Collectors;
@Service("ocrTaskchildPictureService")
@Slf4j
public class OcrTaskchildPictureServiceImpl extends BaseServiceImpl<OcrTaskchildPictureMybatisDao, OcrTaskchildPicture> implements OcrTaskchildPictureService {
@Value("${ocr.api.wly.data-url}")
private String dataUrl;
@Resource
private OcrTaskchildPictureMybatisDao ocrtaskchildpicturemybatisdao;
@ -683,21 +675,9 @@ public class OcrTaskchildPictureServiceImpl extends BaseServiceImpl<OcrTaskchild
String userNodeType = ProcessConstant.userNodeType.getOrDefault(userToken.getNodeType(), "一级审批");
JSONObject jsonObject = JSONObject.parseObject(duplicateHis.getCheckDuplicateResultHisJson());
Set<String> pictureIds = jsonObject.keySet();
List<OcrTaskChildPictureVo> taskChildPicturesOld = this.getPictureHistoryList(new ArrayList<>(pictureIds), userNodeType);
List<OcrTaskChildPictureVo> taskChildPictures = new ArrayList<>();
for (OcrTaskChildPictureVo taskChildPicture : taskChildPicturesOld) {
if(taskChildPicture.getPictureId()!=null){
taskChildPictures.add(taskChildPicture);
}
}
List<OcrTaskChildPictureVo> taskChildPictures = this.getPictureHistoryList(new ArrayList<>(pictureIds), userNodeType);
for (OcrTaskChildPictureVo taskChildPicture : taskChildPictures) {
if(jsonObject.getString(taskChildPicture.getPictureId())!=null){
taskChildPicture.setSimilarityScore(Integer.parseInt(jsonObject.getString(taskChildPicture.getPictureId().toString())));
}else{
taskChildPicture.setSimilarityScore(1);
}
taskChildPicture.setSimilarityScore(Integer.parseInt(jsonObject.getString(taskChildPicture.getPictureId().toString())));
}
if (orderValue.equals("similarityScore")) {
if ("desc".equals(orderType)) {
@ -735,8 +715,8 @@ public class OcrTaskchildPictureServiceImpl extends BaseServiceImpl<OcrTaskchild
flowResultDto.setAccountNo(Long.parseLong(picture.getPictureid()));
flowResultDto.setTaskNo(Long.parseLong(picture.getRemark()));
flowResultDto.setPictureMatchDegree(BigDecimal.valueOf(picture.getSimilarityscore()));
flowResultDto.setIsPictureRepeat(picture.getField14()==null?0:Integer.valueOf(picture.getField14()));
flowResultDto.setIsPictureRight(picture.getIztrueorfalse()==null?0:picture.getIztrueorfalse());
flowResultDto.setIsPictureRepeat(Integer.valueOf(picture.getField14()));
flowResultDto.setIsPictureRight(picture.getIztrueorfalse());
Integer resultStatus = Objects.equals(taskchildPicture.getStates(), SystemConstantsOa.OA_STATUS_TYPE_END) ? SystemConstantsOa.OCR_STATUS_SUCCESS : SystemConstantsOa.OCR_STATUS_FAILURE;
flowResultDto.setApproveResult(resultStatus);
flowResultDto.setApproveRemark(flowApprove.getComment());
@ -749,15 +729,8 @@ public class OcrTaskchildPictureServiceImpl extends BaseServiceImpl<OcrTaskchild
}
}
flowResultDto.setApproveDetailList(nodeList);
ResultDto resultDto = new ResultDto();
resultDto.setStatus(HttpStatus.OK.value());
resultDto.setTimestamp(System.currentTimeMillis());
resultDto.setData(JSON.toJSONString(flowResultDto));
System.out.println(JSON.toJSONString(resultDto));
String post = HttpUtil.post(dataUrl, JSONObject.toJSONString(resultDto));
System.out.println(post);
}
}
@Override
public List<OcrTaskChildPictureVo> getDubiousfileList(List<String> pictureIds, String taskname) {

@ -1,79 +0,0 @@
package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
@Slf4j
public class ImageDownloadMonitoring {
@Autowired
private OcrPictureService ocrPictureService;
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
/**
*
*
* @return
*/
public boolean startImageDownloadMonitoring(String localImagePath) {
Runnable runnable = new Runnable() {
@SneakyThrows
@Override
public void run() {
OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService");
// while (true) {
logger.debug("图片下载监测中...");
// try {
// 查询数据库中local_picture_url字段为null的图片数据
LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>();
ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
List<OcrPicture> ocrPictureList = new ArrayList<>();
ocrPictureList = ocrPictureService.list(ocrPictureLambdaQueryWrapper);
logger.debug("待执行下载图片数量:{}",ocrPictureList.size());
for (OcrPicture pictureOne : ocrPictureList) {
// 创建 PictureImgToLocalEntity 对象并赋值
PictureImgToLocalEntity entity = new PictureImgToLocalEntity();
entity.setPictureId(pictureOne.getId());
entity.setImgUrl(pictureOne.getImgurl());
String imgurl = pictureOne.getImgurl();
entity.setLocalPath(localImagePath + imgurl.substring(imgurl.lastIndexOf("/")));
//本地调试下载地址
// entity.setLocalPath("D:\\server\\data\\images" + imgurl.substring(imgurl.lastIndexOf("/")));
// 将对象添加到下载图片队列中
boolean isAddedToQueue = false;
while (!isAddedToQueue) {
isAddedToQueue =TaskQueue.pictureImgToLocalPushData(entity);
if (!isAddedToQueue) {
logger.info("图片下载监测-添加队列失败,等待五秒后继续进行添加,图片id:{}",entity.getPictureId());
Thread.sleep(5000);
}
}
}
// // 等待一段时间再继续检查
// Thread.sleep(60000);
// } catch (Exception e) {
// logger.error("图片下载监测出错{}", e.getMessage());
// Thread.sleep(5000);
// }
// }
}
};
Thread thread = new Thread(runnable);
thread.start();
return true;
}
}

@ -1,6 +1,5 @@
package cn.jyjz.xiaoyao.ocr.thread;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
@ -13,16 +12,10 @@ import org.springframework.stereotype.Component;
*/
@Component
public class TaskRunner implements CommandLineRunner {
@Value("${ocr.api.wly.localImagePath}")
private String localImagePath;
@Override
public void run(String... args) throws Exception {
TaskThreadPool taskThreadPool = new TaskThreadPool();
taskThreadPool.startPictureDisposePull();
taskThreadPool.startPictureImgToLocalPull();
// ImageDownloadMonitoring imageDownloadMonitoring = new ImageDownloadMonitoring();
// imageDownloadMonitoring.startImageDownloadMonitoring(localImagePath);
}
}

@ -4,7 +4,6 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
import liquibase.pro.packaged.E;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -22,7 +21,6 @@ public class TaskThreadPool {
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
ExecutorService threadPool = null;
ExecutorService threadPoolImg = null;
public TaskThreadPool() {
if (threadPool == null) {
@ -34,15 +32,6 @@ public class TaskThreadPool {
new LinkedBlockingDeque<>(),
Executors.defaultThreadFactory(),
new ThreadPoolExecutor.DiscardOldestPolicy());
threadPoolImg = new ThreadPoolExecutor(
10,
40,
3,
TimeUnit.SECONDS,
new LinkedBlockingDeque<>(),
Executors.defaultThreadFactory(),
new ThreadPoolExecutor.DiscardOldestPolicy());
}
}
@ -56,8 +45,8 @@ public class TaskThreadPool {
@SneakyThrows
@Override
public void run() {
logger.debug("无量云接口数据处理消费线程检测中...");
while (true) {
logger.debug("无量云接口数据处理消费线程检测中...,队列数量:{}",TaskQueue.pictureDisposeQueue.size());
PictureSourceResult pictureSourceResult = TaskQueue.pictureDisposePullData();
if (pictureSourceResult != null) {
threadPool.execute(new PictureDisposeTask(pictureSourceResult));
@ -82,12 +71,13 @@ public class TaskThreadPool {
@SneakyThrows
@Override
public void run() {
logger.debug("图片下载消费线程检测中...");
while (true) {
logger.debug("图片下载消费线程检测中...,队列数量:{}",TaskQueue.pictureImgToLocalQueue.size());
try {
PictureImgToLocalEntity pictureImgToLocalEntity = TaskQueue.pictureImgToLocalPullData();
if (pictureImgToLocalEntity != null) {
threadPoolImg.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
threadPool.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
} else {
Thread.sleep(5000);
}

@ -4,7 +4,6 @@ import cn.hutool.core.util.ObjectUtil;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
import cn.jyjz.xiaoyao.ocr.util.ImageUtils;
@ -81,22 +80,7 @@ public class PictureImgToLocalTask implements Runnable {
// picture.setPhotoDateTimestamp(timestamp);
// }
ocrPictureService.updateById(picture);
logger.info("图片本地化,回写数据完成!");
logger.info("开启获取图片信息操作!");
ocrPictureService.savePicturesInfo(picture);
logger.info("获取图片信息操作,结束!");
}
}else{
//判断是否是第一次失败!
if(picture.getDownloadErrorCount()==null){
picture.setDownloadErrorCount(0);
}
//判断如果没超过5次则放回队列进行重试
if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){
TaskQueue.pictureImgToLocalPushData(pictureImgToLocal);
picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1);
}
ocrPictureService.updateById(picture);
}
}

@ -5,8 +5,6 @@ import org.slf4j.LoggerFactory;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URL;
/**
@ -34,16 +32,8 @@ public class DownloadImgUtil {
while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n);
}
out.flush();
} catch (Exception e) {
logger.error("转储异常问题msg:{}",e.getMessage());
logger.error("转储异常问题json:{}",e.getStackTrace());
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串
logger.error("图片本地化,转储图片异常:{}",exceptionDetails);
logger.error(e.getMessage());
return false;
}
return true;

@ -236,7 +236,7 @@
<if test="pictureId != null and pictureId != ''">
and t1.picture_id = #{pictureId}
</if>
ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp asc
ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp DESC
LIMIT 1
</select>

@ -482,91 +482,6 @@
</trim>
</where>
</sql>
<sql id="Base_Where_Union_New">
<where>
<trim prefixOverrides="and|or">
<if test="izupuser != null">
${izupuser.rulesOp} tcp.FROMUSERID ${izupuser.dataOp} ${izupuser.likestar}
<!-- #{fromuserid.value}-->
<if test="izupuser.value instanceof java.util.List">
<foreach item="item" collection="izupuser.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izupuser.value instanceof java.util.List">
#{izupuser.value}
</if>
${izupuser.likeend}
</if>
<if test="izApprovalStatus != null">
${izApprovalStatus.rulesOp} tcp.FROMUSERID ${izApprovalStatus.dataOp} ${izApprovalStatus.likestar}
<!-- #{fromuserid.value}-->
<if test="izApprovalStatus.value instanceof java.util.List">
<foreach item="item" collection="izApprovalStatus.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izApprovalStatus.value instanceof java.util.List">
#{izApprovalStatus.value}
</if>
${izApprovalStatus.likeend}
</if>
<if test="iztaskrrom != null">
${iztaskrrom.rulesOp} tcp.FROMSOURCEID ${iztaskrrom.dataOp} ${iztaskrrom.likestar}
<!-- #{fromuserid.value}-->
<if test="iztaskrrom.value instanceof java.util.List">
<foreach item="item" collection="iztaskrrom.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!iztaskrrom.value instanceof java.util.List">
#{iztaskrrom.value}
</if>
${iztaskrrom.likeend}
</if>
<if test="izproject != null">
${izproject.rulesOp} tcp.FROMPROJECTID ${izproject.dataOp} ${izproject.likestar}
<!-- #{fromuserid.value}-->
<if test="izproject.value instanceof java.util.List">
<foreach item="item" collection="izproject.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izproject.value instanceof java.util.List">
#{izproject.value}
</if>
${izproject.likeend}
</if>
<!-- <if test="izuptime != null">-->
<!-- ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}-->
<!-- </if>-->
<!-- <if test="izuptimeStart != null">-->
<!-- and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'-->
<!-- </if>-->
<!-- <if test="izuptimeEnd != null">-->
<!-- and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'-->
<!-- </if>-->
<if test="izuptimeStart != null">
<!-- 如果 izuptimeStart.dataOp 的值为 "<",则使用 "or" 连接 -->
<choose>
<when test="izuptimeStart.rulesOp == 'or' || izuptimeEnd.rulesOp=='or'">
<if test="izuptimeEnd != null">
and (tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}' or tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}')
</if>
</when>
<otherwise>
<!-- 其他情况下仍然拼接 "and" 连接 -->
and tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}'
<if test="izuptimeEnd != null">
and tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}'
</if>
</otherwise>
</choose>
</if>
</trim>
</where>
</sql>
<select id="listFinal" resultType="cn.jyjz.xiaoyao.ocr.dataobject.OcrTaskchildPicture"
parameterType="java.util.Map">
select tcp.*, pic.field11 reporttime, pic.similarity_score similarityscore from (
@ -641,45 +556,42 @@
parameterType="java.util.Map">
select tcp.* from (
select tc.*,
uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS,
uf.TASKINDEX,
uf.TASKNO,
uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS,
uf.TASKINDEX,
uf.TASKNO,
op.field6 as fromCustomerLevel,
op.releaseArea as fromCity
from ocr_taskchild_picture tc
LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID
left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0
LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID
left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0
where tc.ASSIGNEE = #{searchassignee}
and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2
and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2
UNION
select tc.*,
uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS,
uf.TASKINDEX,
uf.TASKNO,
uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS,
uf.TASKINDEX,
uf.TASKNO,
op.field6 as fromCustomerLevel,
op.releaseArea as fromCity
from ocr_taskchild_picture tc,
OA_USERAPPROVE_T uf,
ocr_picture op
OA_USERAPPROVE_T uf,
ocr_picture op
where tc.ID = uf.FORMID
and tc.PICTUREID = op.ID
and op.suspiciousfile = 0
and tc.TENANTID = #{searchtenantid}
and tc.PICTUREID = op.ID
and op.suspiciousfile = 0
and tc.TENANTID = #{searchtenantid}
<if test="thisLoginUserid != null">
and uf.USERID = #{thisLoginUserid}
</if>
) as tcp
<include refid="Base_Where_union">
</include>
<include refid="Base_Where_Union_New">
</include>
<if test="page != null">
<if test="page.sortname != null">
order by tcp.${page.sortname} ${page.sortorder}
@ -692,22 +604,20 @@
select tc.*
from ocr_taskchild_picture tc
where tc.ASSIGNEE = #{searchassignee}
and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2
and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2
UNION
select tc.*
from ocr_taskchild_picture tc,
OA_USERAPPROVE_T uf
OA_USERAPPROVE_T uf
where tc.ID = uf.FORMID
and tc.TENANTID = #{searchtenantid}
and tc.TENANTID = #{searchtenantid}
<if test="thisLoginUserid != null">
and uf.USERID = #{thisLoginUserid}
</if>
) as tcp
<include refid="Base_Where_union">
</include>
<include refid="Base_Where_Union_New">
</include>
</select>
@ -874,7 +784,7 @@
and t1.picture_id != #{pictureId}
</if>
<if test="oderName == 'similarityScore'">
order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp asc
order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp ${oderType}
</if>
<if test="oderName == 'createdate'">
order by t2.submit_date_timestamp ${oderType}

@ -282,7 +282,6 @@ ocr:
connectTimeout: 5000 #请求超时时间 毫秒
readTimeout: 60000 #接收超时时间 毫秒
localImagePath: /server/data/images #图片本地存储地址
data-url: https://b-test.prevailcloud.com/pangu/jingwei/thirdclient/command/pb/approval/result
image:
similarityPath: http://47.93.59.251/ai/api/similarity/
classifyPath: http://47.93.59.251/api/classify/

@ -286,7 +286,6 @@ ocr:
connectTimeout: 5000 #请求超时时间 毫秒
readTimeout: 60000 #接收超时时间 毫秒
localImagePath: /server/data/images #图片本地存储地址
data-url: https://b-test.prevailcloud.com/pangu/jingwei/thirdclient/command/pb/approval/result
image:
similarityPath: http://127.0.0.1:8000/api/similarity/
classifyPath: http://127.0.0.1:8000/api/classify/

@ -1,3 +1,3 @@
spring:
profiles:
active: pro
active: dev

Loading…
Cancel
Save