diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java index e67df8d6..0dba40bc 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java @@ -41,7 +41,8 @@ public class OcrCheckDuplicateResult implements Serializable { @ApiModelProperty(value = "图片id") private Long pictureId; - @ApiModelProperty(value = "当前图片与任务包其他图片比对结果") + @ApiModelProperty(value = "当前图片与任务包其他图片比对结果 (废弃，入库导致数据量过大)") + @TableField(exist = false) private String compareResultJson; @ApiModelProperty(value = "查重任务id") diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java index e266f691..0cb6d759 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java @@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable { @TableField(value = "similarity_score") private Integer similarityscore; + @ApiModelProperty(value = "下载到本地，失败次数") + @TableField(value = "download_error_count") + private Integer downloadErrorCount; + @ApiModelProperty(value = "图片查重的比对图片集合") @TableField(exist = false) private List listCom; @@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable { @TableField(exist = false) private String taskId; + + + //保存当前登录用户的数据权限范围的搜索条件 @TableField(exist = false) @JsonIgnore diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java index 27061db9..dfa6beef 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java @@ -178,8 +178,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl ocrPictureList, UserToken sysUser, Long checkDuplicateId) throws Exception { +// @Transactional(rollbackFor = Exception.class) + public void comparePictureResultSave(List ocrPictureList, UserToken sysUser, Long checkDuplicateId){ List saveList = new ArrayList<>(); //需要创建任务的图片集合 @@ -198,10 +198,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl map = this.getOcrPictureClassifyAndHash(url); @@ -228,10 +228,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl map = this.getOcrPictureClassifyAndHash(url); if (map != null) { @@ -290,33 +291,58 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl pictures = new ArrayList<>(); + pictures.add(ocrPicture); + try { + this.saveCheckDuplicateData(checkDuplicateId,saveList,pictures,ocrPictureList.get(0).getTenantId().toString(),sysUser); + } catch (Exception e) { + logger.error("比对结果入库异常! error:{}",e.getMessage()); + } + saveList = new ArrayList<>(); } - } - logger.info("查重任务:{},开启创建任务",checkDuplicateId); - if (CollectionUtils.isNotEmpty(ocrPictureList)) { - ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); - } - logger.info("查重任务:{},结束创建任务",checkDuplicateId); - //批量入库 - logger.info("查重任务:{},开启批量入库任务",checkDuplicateId); + + } + + /** + * 保存查重数据 + * @return + */ + @Transactional(rollbackFor = Exception.class) + public boolean saveCheckDuplicateData(Long checkDuplicateId,List saveList,List ocrPictureList, String tenantId,UserToken sysUser) throws Exception { + //1.批量入库 + logger.debug("查重任务:{},开启批量入库任务",checkDuplicateId); boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList); if (!b) { + logger.error("入库比对结果失败,{}",checkDuplicateId); throw new Exception("批量入库比对结果失败!"); } - logger.info("查重任务:{},结束批量入库任务",checkDuplicateId); - //开始更新图片最高阈值 - logger.info("查重任务:{},更新最高阈值",checkDuplicateId); - this.updatePictureSimilarityScore(saveList); - logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId); - //开始更新 图片历史比对数据结果 + //2.开始更新图片最高阈值 + logger.debug("查重任务:{},更新最高阈值",checkDuplicateId); + b = this.updatePictureSimilarityScore(saveList); + if (!b) { + logger.error("更新最高阈值,{}",checkDuplicateId); + throw new Exception("更新最高阈值失败!"); + } + //3.开始更新 图片历史比对数据结果 logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId); - this.updateCompareToPictureDuplicateHis(saveList); - logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId); + b = this.updateCompareToPictureDuplicateHis(saveList); + if (!b) { + logger.error("更新图片历史比对结果失败,{}",checkDuplicateId); + throw new Exception("更新图片历史比对结果失败!"); + } + //4.开始更新 图片历史比对数据结果 + logger.info("查重任务:{},开启创建任务",checkDuplicateId); + if (CollectionUtils.isNotEmpty(ocrPictureList)) { + ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser); + } + return true; } + @Override public Boolean removeCheckDuplicate(String checkDuplicateNo, String tenantId) { QueryWrapper queryWrapper = new QueryWrapper(); diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java index 55dba0da..2f5b3f8c 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java @@ -35,9 +35,9 @@ public class ImageDownloadMonitoring { @Override public void run() { OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService"); - while (true) { +// while (true) { logger.debug("图片下载监测中..."); - try { +// try { // 查询数据库中local_picture_url字段为null的图片数据 LambdaQueryWrapper ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>(); ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl); @@ -63,13 +63,13 @@ public class ImageDownloadMonitoring { } } } - // 等待一段时间再继续检查 - Thread.sleep(60000); - } catch (Exception e) { - logger.error("图片下载监测出错{}", e.getMessage()); - Thread.sleep(5000); - } - } +// // 等待一段时间再继续检查 +// Thread.sleep(60000); +// } catch (Exception e) { +// logger.error("图片下载监测出错{}", e.getMessage()); +// Thread.sleep(5000); +// } +// } } }; Thread thread = new Thread(runnable); diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java index 7b9c976f..40136cbb 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java @@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask; +import liquibase.pro.packaged.E; import lombok.SneakyThrows; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java index 98a71755..0e5ff181 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; import cn.jyjz.xiaoyao.common.base.util.SpringUtils; import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture; import cn.jyjz.xiaoyao.ocr.service.OcrPictureService; +import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil; import cn.jyjz.xiaoyao.ocr.util.ImageUtils; @@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable { if (ObjectUtil.isNotEmpty(picture)) { picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath()); picture.setIsdownload(1); - try { - picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182)); - picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl())); - } catch (IOException e) { - throw new RuntimeException(e); - } - Map ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl()); - if(ocrPictureClassifyAndHash!=null){ - picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); - picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); - } +// try { +// picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182)); +// picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl())); +// } catch (IOException e) { +// throw new RuntimeException(e); +// } +// Map ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl()); +// if(ocrPictureClassifyAndHash!=null){ +// picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); +// picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); +// } // 获取拍照时间 // Long timestamp = ocrPictureService.getPhotoDateTime(picture); // if(timestamp != 0L){ // picture.setPhotoDateTimestamp(timestamp); // } - ocrPictureService.updateById(picture); +// ocrPictureService.updateById(picture); logger.info("图片本地化，回写数据完成!"); logger.info("开启获取图片信息操作!"); - ocrPictureService.savePicturesInfo(picture); +// ocrPictureService.savePicturesInfo(picture); logger.info("获取图片信息操作,结束!"); } + }else{ + //判断是否是第一次失败! + if(picture.getDownloadErrorCount()==null){ + picture.setDownloadErrorCount(0); + } + //判断如果没超过5次则放回队列进行重试 + if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){ + TaskQueue.pictureImgToLocalPushData(pictureImgToLocal); + picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1); + } + ocrPictureService.updateById(picture); } } diff --git a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java index f4db9494..3c06415e 100644 --- a/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java +++ b/jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java @@ -34,12 +34,16 @@ public class DownloadImgUtil { while ((n = in.read(buffer)) != -1) { out.write(buffer, 0, n); } + out.flush(); } catch (Exception e) { + logger.error("转储异常问题msg:{}",e.getMessage()); + logger.error("转储异常问题json:{}",e.getStackTrace()); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); - String exceptionDetails = sw.toString(); // 包含堆栈跟踪信息的字符串 - logger.error("图片本地化，转储图片异常:",exceptionDetails); + String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串 + logger.error("图片本地化，转储图片异常:{}",exceptionDetails); + return false; } return true;