guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'fix/checkDuplicateBug' (#79) from fix/checkDuplicateBug into test

Browse Source 
Reviewed-on: #79
fix/localPictureUrlIsNotNull
yaoshuli 1 year ago
parent 352bb37ed9 d76ba019c2
commit 294463c6a1
7 changed files with 102 additions and 51 deletions
Show Stats Download Patch File Download Diff File

3
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrCheckDuplicateResult.java
Unescape View File

@ -41,7 +41,8 @@ public class OcrCheckDuplicateResult implements Serializable {
@ApiModelProperty(value = "图片id")
private Long pictureId;
@ApiModelProperty(value = "当前图片与任务包其他图片比对结果")
@ApiModelProperty(value = "当前图片与任务包其他图片比对结果 (废弃，入库导致数据量过大)")
@TableField(exist = false)
private String compareResultJson;
@ApiModelProperty(value = "查重任务id")

7
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/dataobject/OcrPicture.java
Unescape View File

@ -311,6 +311,10 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(value = "similarity_score")
private Integer similarityscore;
@ApiModelProperty(value = "下载到本地，失败次数")
@TableField(value = "download_error_count")
private Integer downloadErrorCount;
@ApiModelProperty(value = "图片查重的比对图片集合")
@TableField(exist = false)
private List<OcrPicture> listCom;
@ -367,6 +371,9 @@ public class OcrPicture implements BaseDto, java.io.Serializable {
@TableField(exist = false)
private String taskId;
//保存当前登录用户的数据权限范围的搜索条件
@TableField(exist = false)
@JsonIgnore

72
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java
Unescape View File

@ -178,8 +178,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
* 6.回写ocr_picture表，similarity_score字段，字段值是该图片所有历史比对的最高值，如果本次比对的最高阈值小于最高值，则不进行回写。
*/
@Override
@Transactional(rollbackFor = Exception.class)
public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId) throws Exception {
// @Transactional(rollbackFor = Exception.class)
public void comparePictureResultSave(List<OcrPicture> ocrPictureList, UserToken sysUser, Long checkDuplicateId){
List<OcrCheckDuplicateResult> saveList = new ArrayList<>();
//需要创建任务的图片集合
@ -198,11 +198,11 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if (org.apache.commons.lang3.StringUtils.isBlank(ocrPicture.getImgHash())) {
String url = ocrPicture.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPicture.getImgurl())){
// if(StringUtils.isBlank(ocrPicture.getImgurl())){
// }
// url = ocrPicture.getImgurl();
continue;
}
url = ocrPicture.getImgurl();
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
if (map != null) {
@ -228,11 +228,12 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
logger.info("图片没有hash值，进入分类并获取hash接口:{}",JSONObject.toJSON(ocrPictureNext));
String url = ocrPictureNext.getLocalpictrueurl();
if(StringUtils.isBlank(url)){
if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
// if(StringUtils.isBlank(ocrPictureNext.getImgurl())){
// continue;
// }
// url = ocrPictureNext.getImgurl();
continue;
}
url = ocrPictureNext.getImgurl();
}
Map<String, String> map = this.getOcrPictureClassifyAndHash(url);
if (map != null) {
ocrPictureNext.setImgHash(map.get("hash"));
@ -290,32 +291,57 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
if(similarityGroupOrderMap.get(ocrPicture.getId().toString())!=null){
ocrCheckDuplicateResult.setSimilarityGroupOrder(similarityGroupOrderMap.get(ocrPicture.getId().toString()));
}
//1.3开启入库操作,不能进行批量入库，会导致内存被拉崩
saveList.add(ocrCheckDuplicateResult);
List<OcrPicture> pictures = new ArrayList<>();
pictures.add(ocrPicture);
try {
this.saveCheckDuplicateData(checkDuplicateId,saveList,pictures,ocrPictureList.get(0).getTenantId().toString(),sysUser);
} catch (Exception e) {
logger.error("比对结果入库异常! error:{}",e.getMessage());
}
saveList = new ArrayList<>();
}
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
logger.info("查重任务:{},结束创建任务",checkDuplicateId);
//批量入库
logger.info("查重任务:{},开启批量入库任务",checkDuplicateId);
}
/**
* 保存查重数据
* @return
*/
@Transactional(rollbackFor = Exception.class)
public boolean saveCheckDuplicateData(Long checkDuplicateId,List<OcrCheckDuplicateResult> saveList,List<OcrPicture> ocrPictureList, String tenantId,UserToken sysUser) throws Exception {
//1.批量入库
logger.debug("查重任务:{},开启批量入库任务",checkDuplicateId);
boolean b = checkDuplicateResultService.saveOrUpdateBatch(saveList);
if (!b) {
logger.error("入库比对结果失败,{}",checkDuplicateId);
throw new Exception("批量入库比对结果失败!");
}
logger.info("查重任务:{},结束批量入库任务",checkDuplicateId);
//开始更新图片最高阈值
logger.info("查重任务:{},更新最高阈值",checkDuplicateId);
this.updatePictureSimilarityScore(saveList);
logger.info("查重任务:{},结束更新最高阈值",checkDuplicateId);
//开始更新 图片历史比对数据结果
//2.开始更新图片最高阈值
logger.debug("查重任务:{},更新最高阈值",checkDuplicateId);
b = this.updatePictureSimilarityScore(saveList);
if (!b) {
logger.error("更新最高阈值,{}",checkDuplicateId);
throw new Exception("更新最高阈值失败!");
}
//3.开始更新 图片历史比对数据结果
logger.info("查重任务:{},更新图片历史比对结果",checkDuplicateId);
this.updateCompareToPictureDuplicateHis(saveList);
logger.info("查重任务:{},结束图片历史比对结果",checkDuplicateId);
b = this.updateCompareToPictureDuplicateHis(saveList);
if (!b) {
logger.error("更新图片历史比对结果失败,{}",checkDuplicateId);
throw new Exception("更新图片历史比对结果失败!");
}
//4.开始更新 图片历史比对数据结果
logger.info("查重任务:{},开启创建任务",checkDuplicateId);
if (CollectionUtils.isNotEmpty(ocrPictureList)) {
ocrTaskchildPictureService.createTask(ocrPictureList, ocrPictureList.get(0).getTenantId().toString(), sysUser);
}
return true;
}
@Override
public Boolean removeCheckDuplicate(String checkDuplicateNo, String tenantId) {

18
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/ImageDownloadMonitoring.java
Unescape View File

@ -35,9 +35,9 @@ public class ImageDownloadMonitoring {
@Override
public void run() {
OcrPictureService ocrPictureService = SpringUtils.getBean("ocrPictureService");
while (true) {
// while (true) {
logger.debug("图片下载监测中...");
try {
// try {
// 查询数据库中local_picture_url字段为null的图片数据
LambdaQueryWrapper<OcrPicture> ocrPictureLambdaQueryWrapper = new LambdaQueryWrapper<>();
ocrPictureLambdaQueryWrapper.isNull(OcrPicture::getLocalpictrueurl);
@ -63,13 +63,13 @@ public class ImageDownloadMonitoring {
}
}
}
// 等待一段时间再继续检查
Thread.sleep(60000);
} catch (Exception e) {
logger.error("图片下载监测出错{}", e.getMessage());
Thread.sleep(5000);
}
}
// // 等待一段时间再继续检查
// Thread.sleep(60000);
// } catch (Exception e) {
// logger.error("图片下载监测出错{}", e.getMessage());
// Thread.sleep(5000);
// }
// }
}
};
Thread thread = new Thread(runnable);

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java
Unescape View File

@ -4,6 +4,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
import liquibase.pro.packaged.E;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

38
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java
Unescape View File

@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
import cn.jyjz.xiaoyao.ocr.util.ImageUtils;
@ -63,28 +64,39 @@ public class PictureImgToLocalTask implements Runnable {
if (ObjectUtil.isNotEmpty(picture)) {
picture.setLocalpictrueurl(pictureImgToLocal.getLocalPath());
picture.setIsdownload(1);
try {
picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
} catch (IOException e) {
throw new RuntimeException(e);
}
Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
if(ocrPictureClassifyAndHash!=null){
picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
}
// try {
// picture.setLocalThumbnailUrl(ImageUtils.generateThumbnail(pictureImgToLocal.getLocalPath(),182));
// picture.setServerThumbnailUrl(ocrPictureService.getServerUrl()+ImageUtils.getFileName(picture.getLocalThumbnailUrl()));
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// Map<String, String> ocrPictureClassifyAndHash = ocrPictureService.getOcrPictureClassifyAndHash(picture.getLocalpictrueurl());
// if(ocrPictureClassifyAndHash!=null){
// picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
// picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
// }
// 获取拍照时间
// Long timestamp = ocrPictureService.getPhotoDateTime(picture);
// if(timestamp != 0L){
// picture.setPhotoDateTimestamp(timestamp);
// }
ocrPictureService.updateById(picture);
// ocrPictureService.updateById(picture);
logger.info("图片本地化，回写数据完成!");
logger.info("开启获取图片信息操作!");
ocrPictureService.savePicturesInfo(picture);
// ocrPictureService.savePicturesInfo(picture);
logger.info("获取图片信息操作,结束!");
}
}else{
//判断是否是第一次失败!
if(picture.getDownloadErrorCount()==null){
picture.setDownloadErrorCount(0);
}
//判断如果没超过5次则放回队列进行重试
if( picture.getDownloadErrorCount()!=null && picture.getDownloadErrorCount()<5){
TaskQueue.pictureImgToLocalPushData(pictureImgToLocal);
picture.setDownloadErrorCount(picture.getDownloadErrorCount()+1);
}
ocrPictureService.updateById(picture);
}
}

6
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java
Unescape View File

@ -34,12 +34,16 @@ public class DownloadImgUtil {
while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n);
}
out.flush();
} catch (Exception e) {
logger.error("转储异常问题msg:{}",e.getMessage());
logger.error("转储异常问题json:{}",e.getStackTrace());
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
e.printStackTrace(pw);
String exceptionDetails = sw.toString();// 包含堆栈跟踪信息的字符串
logger.error("图片本地化，转储图片异常:",exceptionDetails);
logger.error("图片本地化，转储图片异常:{}",exceptionDetails);
return false;
}
return true;

Write Preview
Loading…
Cancel
Save