From d96a911c3f5b627b7909482b40999d50e9cf5ea9 Mon Sep 17 00:00:00 2001 From: shuliYao <1397940314@qq.com> Date: Mon, 8 Apr 2024 21:36:42 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E5=8C=96=EF=BC=8C=E6=8B=89=E5=8F=96=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E5=A2=9E=E5=8A=A0=E9=87=8D=E8=AF=95=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3=E6=9F=A5=E9=87=8D1.6w=E6=9D=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AF=BC=E8=87=B4=E5=86=85=E5=AD=98=E8=A2=AB?= =?UTF-8?q?=E6=8B=89=E5=B4=A9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../xiaoyao/ocr/dataobject/OcrPicture.java | 7 +++ .../impl/OcrCheckDuplicateServiceImpl.java | 62 ++++++++++--------- .../ocr/thread/ImageDownloadMonitoring.java | 18 +++--- .../xiaoyao/ocr/thread/TaskThreadPool.java | 1 + .../thread/tasks/PictureImgToLocalTask.java | 38 ++++++++---- .../xiaoyao/ocr/util/DownloadImgUtil.java | 8 ++- 6 files changed, 80 insertions(+), 54 deletions(-) diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java index e266f691..0cb6d759 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java @@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable { @TableField(value = "similarity_score") private Integer similarityscore; + @ApiModelProperty(value = "下载到本地，失败次数") + @TableField(value = "download_error_count") + private Integer downloadErrorCount; + @ApiModelProperty(value = "图片查重的比对图片集合") @TableField(exist = false) private List listCom; @@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable { @TableField(exist = false) private String taskId; + + + //保存当前登录用户的数据权限范围的搜索条件 @TableField(exist = false) @JsonIgnore diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java index 27061db9..435774a8 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java @@ -198,10 +198,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl map = this.getOcrPictureClassifyAndHash(url); @@ -228,10 +228,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl map = this.getOcrPictureClassifyAndHash(url); if (map != null) { @@ -291,30 +292,31 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl(); + } + //结果执行 + logger.info("查重任务:{},开启创建任务",checkDuplicateId); + if (CollectionUtils.isNotEmpty(ocrPictureList)) { + ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); } - - } - logger.info("查重任务:{},开启创建任务",checkDuplicateId); - if (CollectionUtils.isNotEmpty(ocrPictureList)) { - ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); - } - logger.info("查重任务:{},结束创建任务",checkDuplicateId); - //批量入库 - logger.info("查重任务:{},开启批量入库任务",checkDuplicateId); - boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList); - if (!b) { - throw new Exception("批量入库比对结果失败!"); } - logger.info("查重任务:{},结束批量入库任务",checkDuplicateId); - //开始更新图片最高阈值 - logger.info("查重任务:{},更新最高阈值",checkDuplicateId); - this.updatePictureSimilarityScore(saveList); - logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId); - //开始更新 图片历史比对数据结果 - logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId); - this.updateCompareToPictureDuplicateHis(saveList); - logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId); + } @Override diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java index 55dba0da..2f5b3f8c 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java @@ -35,9 +35,9 @@ public class ImageDownloadMonitoring { @Override public void run() { OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService"); - while (true) { +// while (true) { logger.debug("图片下载监测中..."); - try { +// try { // 查询数据库中local_picture_url字段为null的图片数据 LambdaQueryWrapper ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>(); ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl); @@ -63,13 +63,13 @@ public class ImageDownloadMonitoring { } } } - // 等待一段时间再继续检查 - Thread.sleep(60000); - } catch (Exception e) { - logger.error("图片下载监测出错{}", e.getMessage()); - Thread.sleep(5000); - } - } +// // 等待一段时间再继续检查 +// Thread.sleep(60000); +// } catch (Exception e) { +// logger.error("图片下载监测出错{}", e.getMessage()); +// Thread.sleep(5000); +// } +// } } }; Thread thread = new Thread(runnable); diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java index 7b9c976f..40136cbb 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java @@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask; +import liquibase.pro.packaged.E; import lombok.SneakyThrows; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java index 98a71755..0e5ff181 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; import cn.jyjz.xiaoyao.common.base.util.SpringUtils; import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture; import cn.jyjz.xiaoyao.ocr.service.OcrPictureService; +import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil; import cn.jyjz.xiaoyao.ocr.util.ImageUtils; @@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable { if (ObjectUtil.isNotEmpty(picture)) { picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath()); picture.setIsdownload(1); - try { - picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182)); - picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl())); - } catch (IOException e) { - throw new RuntimeException(e); - } - Map ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl()); - if(ocrPictureClassifyAndHash!=null){ - picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); - picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); - } +// try { +// picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182)); +// picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl())); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// Map ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl()); +// if(ocrPictureClassifyAndHash!=null){ +// picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); +// picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); +// } // 获取拍照时间 // Long timestamp = ocrPictureService.getPhotoDateTime(picture); // if(timestamp != 0L){ // picture.setPhotoDateTimestamp(timestamp); // } - ocrPictureService.updateById(picture); +// ocrPictureService.updateById(picture); logger.info("图片本地化，回写数据完成!"); logger.info("开启获取图片信息操作!"); - ocrPictureService.savePicturesInfo(picture); +// ocrPictureService.savePicturesInfo(picture); logger.info("获取图片信息操作,结束!"); } + }else{ + //判断是否是第一次失败! + if(picture.getDownloadErrorCount()==null){ + picture.setDownloadErrorCount(0); + } + //判断如果没超过5次则放回队列进行重试 + if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){ + TaskQueue.pictureImgToLocalPushData(pictureImgToLocal); + picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1); + } + ocrPictureService.updateById(picture); } } diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java index f4db9494..3c06415e 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java @@ -34,12 +34,16 @@ public class DownloadImgUtil { while ((n = in.read(buffer)) != -1) { out.write(buffer, 0, n); } + out.flush(); } catch (Exception e) { + logger.error("转储异常问题msg:{}",e.getMessage()); + logger.error("转储异常问题json:{}",e.getStackTrace()); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); - String exceptionDetails = sw.toString(); // 包含堆栈跟踪信息的字符串 - logger.error("图片本地化，转储图片异常:",exceptionDetails); + String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串 + logger.error("图片本地化，转储图片异常:{}",exceptionDetails); + return false; } return true; From d76ba019c246d1135cac27e7adf263b117cff66d Mon Sep 17 00:00:00 2001 From: shuliYao <1397940314@qq.com> Date: Tue, 9 Apr 2024 10:55:56 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20=E8=A7=A3=E5=86=B3=E6=9F=A5=E9=87=8D?= =?UTF-8?q?=E5=85=A5=E5=BA=93=E6=95=B0=E6=8D=AE=E9=87=8F=E8=BF=87=E5=A4=A7?= =?UTF-8?q?=EF=BC=8C=E5=AF=BC=E8=87=B4=E4=BA=8B=E5=8A=A1=E5=B4=A9=E6=BA=83?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataobject/OcrCheckDuplicateResult.java | 3 +- .../impl/OcrCheckDuplicateServiceImpl.java | 68 +++++++++++++------ 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java index e67df8d6..0dba40bc 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java @@ -41,7 +41,8 @@ public class OcrCheckDuplicateResult implements Serializable { @ApiModelProperty(value = "图片id") private Long pictureId; - @ApiModelProperty(value = "当前图片与任务包其他图片比对结果") + @ApiModelProperty(value = "当前图片与任务包其他图片比对结果 (废弃，入库导致数据量过大)") + @TableField(exist = false) private String compareResultJson; @ApiModelProperty(value = "查重任务id") diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java index 435774a8..dfa6beef 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java @@ -178,8 +178,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl ocrPictureList, UserToken sysUser, Long checkDuplicateId) throws Exception { +// @Transactional(rollbackFor = Exception.class) + public void comparePictureResultSave(List ocrPictureList, UserToken sysUser, Long checkDuplicateId){ List saveList = new ArrayList<>(); //需要创建任务的图片集合 @@ -291,34 +291,58 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl pictures = new ArrayList<>(); + pictures.add(ocrPicture); + try { + this.saveCheckDuplicateData(checkDuplicateId,saveList,pictures,ocrPictureList.get(0).getTenantId().toString(),sysUser); + } catch (Exception e) { + logger.error("比对结果入库异常! error:{}",e.getMessage()); + } - logger.info("查重任务:{},结束批量入库任务",checkDuplicateId); - //开始更新图片最高阈值 - logger.info("查重任务:{},更新最高阈值",checkDuplicateId); - this.updatePictureSimilarityScore(saveList); - logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId); - //开始更新 图片历史比对数据结果 - logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId); - this.updateCompareToPictureDuplicateHis(saveList); - logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId); saveList = new ArrayList<>(); } - //结果执行 - logger.info("查重任务:{},开启创建任务",checkDuplicateId); - if (CollectionUtils.isNotEmpty(ocrPictureList)) { - ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); - } } } + /** + * 保存查重数据 + * @return + */ + @Transactional(rollbackFor = Exception.class) + public boolean saveCheckDuplicateData(Long checkDuplicateId,List saveList,List ocrPictureList, String tenantId,UserToken sysUser) throws Exception { + //1.批量入库 + logger.debug("查重任务:{},开启批量入库任务",checkDuplicateId); + boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList); + if (!b) { + logger.error("入库比对结果失败,{}",checkDuplicateId); + throw new Exception("批量入库比对结果失败!"); + } + //2.开始更新图片最高阈值 + logger.debug("查重任务:{},更新最高阈值",checkDuplicateId); + b = this.updatePictureSimilarityScore(saveList); + if (!b) { + logger.error("更新最高阈值,{}",checkDuplicateId); + throw new Exception("更新最高阈值失败!"); + } + //3.开始更新 图片历史比对数据结果 + logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId); + b = this.updateCompareToPictureDuplicateHis(saveList); + if (!b) { + logger.error("更新图片历史比对结果失败,{}",checkDuplicateId); + throw new Exception("更新图片历史比对结果失败!"); + } + //4.开始更新 图片历史比对数据结果 + logger.info("查重任务:{},开启创建任务",checkDuplicateId); + if (CollectionUtils.isNotEmpty(ocrPictureList)) { + ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); + } + return true; + } + + @Override public Boolean removeCheckDuplicate(String checkDuplicateNo, String tenantId) { QueryWrapper queryWrapper = new QueryWrapper();