guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 解决图片本地化，拉取失败增加重试流程，解决查重1.6w条数据导致内存被拉崩。

Browse Source
pull/79/head
shuliYao 1 year ago
parent 352bb37ed9
commit d96a911c3f
6 changed files with 80 additions and 54 deletions
Show Stats Download Patch File Download Diff File

7
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java
Unescape View File

@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(value = "similarity_score") @TableField(value = "similarity_score")
private Integer similarityscore; private Integer similarityscore;
@ApiModelProperty(value = "下载到本地，失败次数")
@TableField(value = "download_error_count")
private Integer downloadErrorCount;
@ApiModelProperty(value = "图片查重的比对图片集合") @ApiModelProperty(value = "图片查重的比对图片集合")
@TableField(exist = false) @TableField(exist = false)
private List<OcrPicture> listCom; private List<OcrPicture> listCom;
@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(exist = false) @TableField(exist = false)
private String taskId; private String taskId;
//保存当前登录用户的数据权限范围的搜索条件 //保存当前登录用户的数据权限范围的搜索条件
@TableField(exist = false) @TableField(exist = false)
@JsonIgnore @JsonIgnore

62
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java
Unescape View File

@ -198,10 +198,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) { if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) {
String url = ocrPicture.getLocalpictrueurl(); String url = ocrPicture.getLocalpictrueurl();
if(StringUtils.isBlank(url)){ if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPicture.getImgurl())){ // if(StringUtils.isBlank(ocrPicture.getImgurl())){
continue; // }
} // url = ocrPicture.getImgurl();
url = ocrPicture.getImgurl(); continue;
} }
Map<String, String> map = this.getOcrPictureClassifyAndHash(url); Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
@ -228,10 +228,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
logger.info("图片没有hash值，进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext)); logger.info("图片没有hash值，进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext));
String url = ocrPictureNext.getLocalpictrueurl(); String url = ocrPictureNext.getLocalpictrueurl();
if(StringUtils.isBlank(url)){ if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPictureNext.getImgurl())){ // if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
continue; // continue;
} // }
url = ocrPictureNext.getImgurl(); // url = ocrPictureNext.getImgurl();
continue;
} }
Map<String, String> map = this.getOcrPictureClassifyAndHash(url); Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
if (map != null) { if (map != null) {
@ -291,30 +292,31 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString())); ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString()));
} }
saveList.add(ocrCheckDuplicateResult); saveList.add(ocrCheckDuplicateResult);
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
throw new Exception("批量入库比对结果失败!");
}
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
saveList = new ArrayList<>();
}
//结果执行
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
} }
}
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
throw new Exception("批量入库比对结果失败!");
} }
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
} }
@Override @Override

18
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java
Unescape View File

@ -35,9 +35,9 @@ public class ImageDownloadMonitoring {
@Override @Override
public void run() { public void run() {
OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService"); OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService");
while (true) { // while (true) {
logger.debug("图片下载监测中..."); logger.debug("图片下载监测中...");
try { // try {
// 查询数据库中local_picture_url字段为null的图片数据 // 查询数据库中local_picture_url字段为null的图片数据
LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>(); LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>();
ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl); ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
@ -63,13 +63,13 @@ public class ImageDownloadMonitoring {
} }
} }
} }
// 等待一段时间再继续检查 // // 等待一段时间再继续检查
Thread.sleep(60000); // Thread.sleep(60000);
} catch (Exception e) { // } catch (Exception e) {
logger.error("图片下载监测出错{}", e.getMessage()); // logger.error("图片下载监测出错{}", e.getMessage());
Thread.sleep(5000); // Thread.sleep(5000);
} // }
} // }
} }
}; };
Thread thread = new Thread(runnable); Thread thread = new Thread(runnable);

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java
Unescape View File

@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask; import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
import liquibase.pro.packaged.E;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;

38
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java
Unescape View File

@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils; import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture; import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService; import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity; import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil; import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
import cn.jyjz.xiaoyao.ocr.util.ImageUtils; import cn.jyjz.xiaoyao.ocr.util.ImageUtils;
@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable {
if (ObjectUtil.isNotEmpty(picture)) { if (ObjectUtil.isNotEmpty(picture)) {
picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath()); picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath());
picture.setIsdownload(1); picture.setIsdownload(1);
try { // try {
picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182)); // picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl())); // picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
} catch (IOException e) { // } catch (IOException e) {
throw new RuntimeException(e); // throw new RuntimeException(e);
} // }
Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl()); // Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
if(ocrPictureClassifyAndHash!=null){ // if(ocrPictureClassifyAndHash!=null){
picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); // picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); // picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
} // }
// 获取拍照时间 // 获取拍照时间
// Long timestamp = ocrPictureService.getPhotoDateTime(picture); // Long timestamp = ocrPictureService.getPhotoDateTime(picture);
// if(timestamp != 0L){ // if(timestamp != 0L){
// picture.setPhotoDateTimestamp(timestamp); // picture.setPhotoDateTimestamp(timestamp);
// } // }
ocrPictureService.updateById(picture); // ocrPictureService.updateById(picture);
logger.info("图片本地化，回写数据完成!"); logger.info("图片本地化，回写数据完成!");
logger.info("开启获取图片信息操作!"); logger.info("开启获取图片信息操作!");
ocrPictureService.savePicturesInfo(picture); // ocrPictureService.savePicturesInfo(picture);
logger.info("获取图片信息操作,结束!"); logger.info("获取图片信息操作,结束!");
} }
}else{
//判断是否是第一次失败!
if(picture.getDownloadErrorCount()==null){
picture.setDownloadErrorCount(0);
}
//判断如果没超过5次则放回队列进行重试
if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){
TaskQueue.pictureImgToLocalPushData(pictureImgToLocal);
picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1);
}
ocrPictureService.updateById(picture);
} }
} }

8
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java
Unescape View File

@ -34,12 +34,16 @@ public class DownloadImgUtil {
while ((n = in.read(buffer)) != -1) { while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n); out.write(buffer, 0, n);
} }
out.flush();
} catch (Exception e) { } catch (Exception e) {
logger.error("转储异常问题msg:{}",e.getMessage());
logger.error("转储异常问题json:{}",e.getStackTrace());
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw); e.printStackTrace(pw);
String exceptionDetails = sw.toString(); // 包含堆栈跟踪信息的字符串 String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串
logger.error("图片本地化，转储图片异常:",exceptionDetails); logger.error("图片本地化，转储图片异常:{}",exceptionDetails);
return false; return false;
} }
return true; return true;

Write Preview
Loading…
Cancel
Save