Merge pull request 'fix/checkDuplicateBug' (#79) from fix/checkDuplicateBug into test

Reviewed-on: #79

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java

@@ -41,7 +41,8 @@ public class OcrCheckDuplicateResult implements Serializable {
     @ApiModelProperty(value = "图片id")
     private Long pictureId;
 
-    @ApiModelProperty(value = "当前图片与任务包其他图片比对结果")
+    @ApiModelProperty(value = "当前图片与任务包其他图片比对结果 (废弃，入库导致数据量过大)")
+    @TableField(exist = false)
     private String compareResultJson;
 
     @ApiModelProperty(value = "查重任务id")

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java

@@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
     @TableField(value = "similarity_score")
     private Integer similarityscore;
 
+    @ApiModelProperty(value = "下载到本地，失败次数")
+    @TableField(value = "download_error_count")
+    private Integer downloadErrorCount;
+
     @ApiModelProperty(value = "图片查重的比对图片集合")
     @TableField(exist = false)
     private List<OcrPicture> listCom;
@@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
     @TableField(exist = false)
     private String taskId;
 
+
+
+
     //保存当前登录用户的数据权限范围的搜索条件
     @TableField(exist = false)
     @JsonIgnore

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java

@@ -178,8 +178,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
      * 6.回写ocr_picture表，similarity_score字段，字段值是该图片所有历史比对的最高值，如果本次比对的最高阈值小于最高值，则不进行回写。
      */
     @Override
-    @Transactional(rollbackFor = Exception.class)
+//    @Transactional(rollbackFor = Exception.class)
-    public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId) throws Exception {
+    public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId){
 
         List<OcrCheckDuplicateResult> saveList = new ArrayList<>();
         //需要创建任务的图片集合
@@ -198,10 +198,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
                 if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) {
                     String url = ocrPicture.getLocalpictrueurl();
                     if(StringUtils.isBlank(url)){
-                        if(StringUtils.isBlank(ocrPicture.getImgurl())){
+//                        if(StringUtils.isBlank(ocrPicture.getImgurl())){
-                            continue;
+//                        }
-                        }
+//                        url = ocrPicture.getImgurl();
-                        url = ocrPicture.getImgurl();
+                        continue;
                     }
 
                     Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
@@ -228,10 +228,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
                         logger.info("图片没有hash值，进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext));
                         String url = ocrPictureNext.getLocalpictrueurl();
                         if(StringUtils.isBlank(url)){
-                            if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
+//                            if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
-                                continue;
+//                                continue;
-                            }
+//                            }
-                            url = ocrPictureNext.getImgurl();
+//                            url = ocrPictureNext.getImgurl();
+                            continue;
                         }
                         Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
                         if (map != null) {
@@ -290,33 +291,58 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
                 if(similarityGroupOrderMap.get(ocrPicture.getId().toString())!=null){
                     ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString()));
                 }
+                //1.3开启入库操作,不能进行批量入库，会导致内存被拉崩
                 saveList.add(ocrCheckDuplicateResult);
+                List<OcrPicture> pictures = new ArrayList<>();
+                pictures.add(ocrPicture);
+                try {
+                    this.saveCheckDuplicateData(checkDuplicateId,saveList,pictures,ocrPictureList.get(0).getTenantId().toString(),sysUser);
+                } catch (Exception e) {
+                    logger.error("比对结果入库异常! error:{}",e.getMessage());
 
+                }
+                saveList = new ArrayList<>();
             }
-
         }
-        logger.info("查重任务:{},开启创建任务",checkDuplicateId);
+
-        if (CollectionUtils.isNotEmpty(ocrPictureList)) {
+    }
-            ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
+
-        }
+    /**
-        logger.info("查重任务:{},结束创建任务",checkDuplicateId);
+     * 保存查重数据
-        //批量入库
+     * @return
-        logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
+     */
+    @Transactional(rollbackFor = Exception.class)
+    public boolean saveCheckDuplicateData(Long checkDuplicateId,List<OcrCheckDuplicateResult> saveList,List<OcrPicture> ocrPictureList, String tenantId,UserToken sysUser) throws Exception {
+        //1.批量入库
+        logger.debug("查重任务:{},开启批量入库任务",checkDuplicateId);
         boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
         if (!b) {
+            logger.error("入库比对结果失败,{}",checkDuplicateId);
             throw new Exception("批量入库比对结果失败!");
         }
-        logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
+        //2.开始更新图片最高阈值
-        //开始更新图片最高阈值
+        logger.debug("查重任务:{},更新最高阈值",checkDuplicateId);
-        logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
+        b = this.updatePictureSimilarityScore(saveList);
-        this.updatePictureSimilarityScore(saveList);
+        if (!b) {
-        logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
+            logger.error("更新最高阈值,{}",checkDuplicateId);
-        //开始更新 图片历史比对数据结果
+            throw new Exception("更新最高阈值失败!");
+        }
+        //3.开始更新 图片历史比对数据结果
         logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
-        this.updateCompareToPictureDuplicateHis(saveList);
+        b = this.updateCompareToPictureDuplicateHis(saveList);
-        logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
+        if (!b) {
+            logger.error("更新图片历史比对结果失败,{}",checkDuplicateId);
+            throw new Exception("更新图片历史比对结果失败!");
+        }
+        //4.开始更新 图片历史比对数据结果
+        logger.info("查重任务:{},开启创建任务",checkDuplicateId);
+        if (CollectionUtils.isNotEmpty(ocrPictureList)) {
+            ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
+        }
+        return true;
     }
 
+
     @Override
     public Boolean removeCheckDuplicate(String checkDuplicateNo, String tenantId) {
         QueryWrapper queryWrapper = new QueryWrapper();

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java

@@ -35,9 +35,9 @@ public class ImageDownloadMonitoring {
             @Override
             public void run() {
                 OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService");
-                while (true) {
+//                while (true) {
                     logger.debug("图片下载监测中...");
-                    try {
+//                    try {
                         // 查询数据库中local_picture_url字段为null的图片数据
                         LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>();
                         ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
@@ -63,13 +63,13 @@ public class ImageDownloadMonitoring {
                                 }
                             }
                         }
-                        // 等待一段时间再继续检查
+//                        // 等待一段时间再继续检查
-                        Thread.sleep(60000);
+//                        Thread.sleep(60000);
-                    } catch (Exception e) {
+//                    } catch (Exception e) {
-                        logger.error("图片下载监测出错{}", e.getMessage());
+//                        logger.error("图片下载监测出错{}", e.getMessage());
-                        Thread.sleep(5000);
+//                        Thread.sleep(5000);
-                    }
+//                    }
-                }
+//                }
             }
         };
         Thread thread = new Thread(runnable);

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java

@@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
 import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
 import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
 import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
+import liquibase.pro.packaged.E;
 import lombok.SneakyThrows;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java

@@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
 import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
 import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
 import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
+import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
 import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
 import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
 import cn.jyjz.xiaoyao.ocr.util.ImageUtils;
@@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable {
             if (ObjectUtil.isNotEmpty(picture)) {
                 picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath());
                 picture.setIsdownload(1);
-                try {
+//                try {
-                    picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
+//                    picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
-                    picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
+//                    picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
-                } catch (IOException e) {
+//                } catch (IOException e) {
-                    throw new RuntimeException(e);
+//                    throw new RuntimeException(e);
-                }
+//                }
-                Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
+//                Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
-                if(ocrPictureClassifyAndHash!=null){
+//                if(ocrPictureClassifyAndHash!=null){
-                    picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
+//                    picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
-                    picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
+//                    picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
-                }
+//                }
                 // 获取拍照时间
 //                Long timestamp = ocrPictureService.getPhotoDateTime(picture);
 //                if(timestamp != 0L){
 //                    picture.setPhotoDateTimestamp(timestamp);
 //                }
-                ocrPictureService.updateById(picture);
+//                ocrPictureService.updateById(picture);
                 logger.info("图片本地化，回写数据完成!");
                 logger.info("开启获取图片信息操作!");
-                ocrPictureService.savePicturesInfo(picture);
+//                ocrPictureService.savePicturesInfo(picture);
                 logger.info("获取图片信息操作,结束!");
             }
+        }else{
+            //判断是否是第一次失败!
+            if(picture.getDownloadErrorCount()==null){
+                picture.setDownloadErrorCount(0);
+            }
+            //判断如果没超过5次则放回队列进行重试
+            if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){
+                TaskQueue.pictureImgToLocalPushData(pictureImgToLocal);
+                picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1);
+            }
+            ocrPictureService.updateById(picture);
         }
     }
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java

@@ -34,12 +34,16 @@ public class DownloadImgUtil {
             while ((n = in.read(buffer)) != -1) {
                 out.write(buffer, 0, n);
             }
+            out.flush();
         } catch (Exception e) {
+            logger.error("转储异常问题msg:{}",e.getMessage());
+            logger.error("转储异常问题json:{}",e.getStackTrace());
             StringWriter sw = new StringWriter();
             PrintWriter pw = new PrintWriter(sw);
             e.printStackTrace(pw);
-            String exceptionDetails = sw.toString(); // 包含堆栈跟踪信息的字符串
+            String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串
-            logger.error("图片本地化，转储图片异常:",exceptionDetails);
+            logger.error("图片本地化，转储图片异常:{}",exceptionDetails);
+
             return false;
         }
         return true;