guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: 解决图片本地化，拉取失败增加重试流程，解决查重1.6w条数据导致内存被拉崩。

Browse Source
pull/79/head
shuliYao 1 year ago
parent 352bb37ed9
commit d96a911c3f
6 changed files with 80 additions and 54 deletions
Show Stats Download Patch File Download Diff File

7
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java
Unescape View File

@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(value = "similarity_score")
private Integer similarityscore;
@ApiModelProperty(value = "下载到本地，失败次数")
@TableField(value = "download_error_count")
private Integer downloadErrorCount;
@ApiModelProperty(value = "图片查重的比对图片集合")
@TableField(exist = false)
private List<OcrPicture> listCom;
@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(exist = false)
private String taskId;
//保存当前登录用户的数据权限范围的搜索条件
@TableField(exist = false)
@JsonIgnore

62
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java
Unescape View File

@ -198,10 +198,10 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) {
String url = ocrPicture.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPicture.getImgurl())){
continue;
}
url = ocrPicture.getImgurl();
// if(StringUtils.isBlank(ocrPicture.getImgurl())){
// }
// url = ocrPicture.getImgurl();
continue;
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
@ -228,10 +228,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
logger.info("图片没有hash值，进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext));
String url = ocrPictureNext.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
continue;
}
url = ocrPictureNext.getImgurl();
// if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
// continue;
// }
// url = ocrPictureNext.getImgurl();
continue;
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
if (map != null) {
@ -291,30 +292,31 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString()));
}
saveList.add(ocrCheckDuplicateResult);
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
throw new Exception("批量入库比对结果失败!");
}
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
saveList = new ArrayList<>();
}
//结果执行
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
}
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
throw new Exception("批量入库比对结果失败!");
}
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
}
@Override

18
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java
Unescape View File

@ -35,9 +35,9 @@ public class ImageDownloadMonitoring {
@Override
public void run() {
OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService");
while (true) {
// while (true) {
logger.debug("图片下载监测中...");
try {
// try {
// 查询数据库中local_picture_url字段为null的图片数据
LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>();
ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
@ -63,13 +63,13 @@ public class ImageDownloadMonitoring {
}
}
}
// 等待一段时间再继续检查
Thread.sleep(60000);
} catch (Exception e) {
logger.error("图片下载监测出错{}", e.getMessage());
Thread.sleep(5000);
}
}
// // 等待一段时间再继续检查
// Thread.sleep(60000);
// } catch (Exception e) {
// logger.error("图片下载监测出错{}", e.getMessage());
// Thread.sleep(5000);
// }
// }
}
};
Thread thread = new Thread(runnable);

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java
Unescape View File

@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
import liquibase.pro.packaged.E;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

38
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java
Unescape View File

@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
import cn.jyjz.xiaoyao.ocr.util.ImageUtils;
@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable {
if (ObjectUtil.isNotEmpty(picture)) {
picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath());
picture.setIsdownload(1);
try {
picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
} catch (IOException e) {
throw new RuntimeException(e);
}
Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
if(ocrPictureClassifyAndHash!=null){
picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
}
// try {
// picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
// picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
// if(ocrPictureClassifyAndHash!=null){
// picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
// picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
// }
// 获取拍照时间
// Long timestamp = ocrPictureService.getPhotoDateTime(picture);
// if(timestamp != 0L){
// picture.setPhotoDateTimestamp(timestamp);
// }
ocrPictureService.updateById(picture);
// ocrPictureService.updateById(picture);
logger.info("图片本地化，回写数据完成!");
logger.info("开启获取图片信息操作!");
ocrPictureService.savePicturesInfo(picture);
// ocrPictureService.savePicturesInfo(picture);
logger.info("获取图片信息操作,结束!");
}
}else{
//判断是否是第一次失败!
if(picture.getDownloadErrorCount()==null){
picture.setDownloadErrorCount(0);
}
//判断如果没超过5次则放回队列进行重试
if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){
TaskQueue.pictureImgToLocalPushData(pictureImgToLocal);
picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1);
}
ocrPictureService.updateById(picture);
}
}

8
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java
Unescape View File

@ -34,12 +34,16 @@ public class DownloadImgUtil {
while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n);
}
out.flush();
} catch (Exception e) {
logger.error("转储异常问题msg:{}",e.getMessage());
logger.error("转储异常问题json:{}",e.getStackTrace());
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
String exceptionDetails = sw.toString(); // 包含堆栈跟踪信息的字符串
logger.error("图片本地化，转储图片异常:",exceptionDetails);
String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串
logger.error("图片本地化，转储图片异常:{}",exceptionDetails);
return false;
}
return true;

Write Preview
Loading…
Cancel
Save