guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:任务图片本地化缓存

Browse Source
dev
shuliYao 1 year ago
parent fb536bee37
commit 8412e854ce
10 changed files with 241 additions and 77 deletions
Show Stats Download Patch File Download Diff File

1
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/flowable/entity/DeModel.java
Unescape View File

@ -4,7 +4,6 @@ import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
import java.sql.Blob;
import java.sql.Date;
import java.time.LocalDateTime;
@TableName("act_de_model")
public class DeModel implements Serializable {

5
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -7,13 +7,10 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import org.apache.commons.lang.time.DateFormatUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import java.time.Instant;
import java.util.Date;
@ -64,7 +61,7 @@ public class ApiTestController {
}
size++;
//将可以处理数据放入处理队列中
TaskQueue.pictureDownloadPushData(pictureSourceResult);
TaskQueue.pictureDisposePushData(pictureSourceResult);
}
return "当前区间内图片总数:"+count+"条，本次获取："+localCount+"条，可处理数据:"+size+"条";
} catch (Exception e) {

45
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskQueue.java
Unescape View File

@ -1,6 +1,7 @@
package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -16,20 +17,25 @@ import java.util.concurrent.LinkedBlockingQueue;
public class TaskQueue {
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
/**
* 无量云接口数据处理 任务队列
*/
public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue();
/**
* 图片下载任务队列
* picture图片转储到本地 任务队列
*/
public static LinkedBlockingQueue pictureDownloadQueue = new LinkedBlockingQueue();
public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
/**
* 图片下载队列 push 队列值
* 无量云接口数据处理 push 队列值
* @return
* @param pictureSourceResult //三方拉取 获取到的图片对象
*/
public static boolean pictureDownloadPushData(PictureSourceResult pictureSourceResult){
public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult){
try {
pictureDownloadQueue.put(pictureSourceResult);
pictureDisposeQueue.put(pictureSourceResult);
} catch (InterruptedException e) {
logger.error("任务队列添加异常:{}",e.getMessage());
return false;
@ -38,11 +44,34 @@ public class TaskQueue {
}
/**
* 图片下载队列 拉取队列值
* 无量云接口数据处理 拉取队列值
* @return
*/
public static PictureSourceResult pictureDisposePullData(){
return (PictureSourceResult) pictureDisposeQueue.poll();
}
/**
* 图片缓存本地队列 push 队列值
* @return
* @param pictureImgToLocalEntity //图片下载对象
*/
public static PictureSourceResult pictureDownloadPullData(){
return (PictureSourceResult) pictureDownloadQueue.poll();
public static boolean pictureImgToLocalPushData(PictureImgToLocalEntity pictureImgToLocalEntity){
try {
pictureImgToLocalQueue.put(pictureImgToLocalEntity);
} catch (InterruptedException e) {
logger.error("图片下载任务队列添加异常:{}",e.getMessage());
return false;
}
return true;
}
/**
* 无量云接口数据处理 拉取队列值
* @return
*/
public static PictureImgToLocalEntity pictureImgToLocalPullData(){
return (PictureImgToLocalEntity) pictureDisposeQueue.poll();
}
}

3
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskRunner.java
Unescape View File

@ -15,6 +15,7 @@ public class TaskRunner implements CommandLineRunner {
@Override
public void run(String... args) throws Exception {
TaskThreadPool taskThreadPool = new TaskThreadPool();
taskThreadPool.startControllerPull();
taskThreadPool.startPictureDisposePull();
taskThreadPool.startPictureImgToLocalPull();
}
}

39
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java
Unescape View File

@ -1,6 +1,8 @@
package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDownloadTask;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
import lombok.SneakyThrows;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -23,7 +25,7 @@ public class TaskThreadPool {
if(threadPool ==null){
threadPool = new ThreadPoolExecutor(
10,
20,
40,
3,
TimeUnit.SECONDS,
new LinkedBlockingDeque<>(),
@ -36,16 +38,41 @@ public class TaskThreadPool {
* 启动任务 入库消费者
* @return
*/
public boolean startControllerPull(){
public boolean startPictureDisposePull(){
Runnable runnable = new Runnable() {
@SneakyThrows
@Override
public void run() {
logger.info("图片下载任务线程检查中...");
logger.debug("无量云接口数据处理消费线程检测中...");
while (true){
PictureSourceResult pictureSourceResult = TaskQueue.pictureDownloadPullData();
PictureSourceResult pictureSourceResult = TaskQueue.pictureDisposePullData();
if(pictureSourceResult!=null){
threadPool.execute(new PictureDownloadTask(pictureSourceResult));
threadPool.execute(new PictureDisposeTask(pictureSourceResult));
}else{
Thread.sleep(5000);
}
}
}
};
Thread thread=new Thread(runnable);
thread.start();
return true;
}
/**
* 启动任务 入库消费者
* @return
*/
public boolean startPictureImgToLocalPull(){
Runnable runnable = new Runnable() {
@SneakyThrows
@Override
public void run() {
logger.debug("图片下载消费线程检测中...");
while (true){
PictureImgToLocalEntity pictureImgToLocalEntity = TaskQueue.pictureImgToLocalPullData();
if(pictureImgToLocalEntity!=null){
threadPool.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
}else{
Thread.sleep(5000);
}

31
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/entity/PictureImgToLocalEntity.java
Unescape View File

@ -0,0 +1,31 @@
package cn.jyjz.xiaoyao.ocr.thread.entity;
import lombok.Data;
/**
* 图片转储本地处理对象
*
* @author hugh(shuli.yao) 1397940314@qq.com
* @version 1.0
* @date 2024/3/16 17:11
*/
@Data
public class PictureImgToLocalEntity {
/**
* 图片表id
*/
private Long pictureId;
/**
* 图片地址
*/
private String ImgUrl;
/**
* 本地存储地址
*/
private String localPath;
}

6
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureDownloadTask.java → jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureDisposeTask.java
Unescape View File

@ -13,18 +13,18 @@ import java.util.List;
import java.util.Map;
/**
* 图片下载任务
* 无量云接口 数据处理队列
*
* @author hugh(shuli.yao) 1397940314@qq.com
* @version 1.0
* @date 2024/3/14 11:15
*/
public class PictureDownloadTask implements Runnable{
public class PictureDisposeTask implements Runnable{
protected Logger logger = LoggerFactory.getLogger(getClass());
private PictureSourceResult pictureSourceResult;
public PictureDownloadTask(PictureSourceResult pictureSourceResult){
public PictureDisposeTask(PictureSourceResult pictureSourceResult){
this.pictureSourceResult = pictureSourceResult;
}

40
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java
Unescape View File

@ -0,0 +1,40 @@
package cn.jyjz.xiaoyao.ocr.thread.tasks;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.common.base.util.StringUtils;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* 无量云接口 数据处理队列
*
* @author hugh(shuli.yao) 1397940314@qq.com
* @version 1.0
* @date 2024/3/14 11:15
*/
public class PictureImgToLocalTask implements Runnable{
protected Logger logger = LoggerFactory.getLogger(getClass());
private PictureImgToLocalEntity pictureImgToLocal;
public PictureImgToLocalTask(PictureImgToLocalEntity pictureImgToLocal){
this.pictureImgToLocal = pictureImgToLocal;
}
@Override
public void run() {
//1.开始转储图片
DownloadImgUtil.downloadImage(pictureImgToLocal.getImgUrl(), pictureImgToLocal.getLocalPath());
}
}

26
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DataDictionaryUtil.java
Unescape View File

@ -49,7 +49,7 @@ public class DataDictionaryUtil {
for (OcrPicture picture : ocrPictureList) {
//1.检查提报人字典是否包含数据
if(picture.getUpname()!=null && picture.getUpuserid()!=null){
OcrDictionaryGroup upNameDG = ocrDictionaryService.queryGroupByField("upName");
OcrDictionaryGroup upNameDG = ocrDictionaryService.queryGroupByField("person");
if(upNameDG!=null && ocrDictionaryService.isValueExists(upNameDG.getId(),picture.getUpname(),picture.getTenantId())){
OcrDictionary dictionary= new OcrDictionary();
dictionary.setLable(picture.getUpname());
@ -68,7 +68,7 @@ public class DataDictionaryUtil {
//2.任务计划
if(picture.getPlanname()!=null && picture.getPlanid()!=null) {
OcrDictionaryGroup planNameDG = ocrDictionaryService.queryGroupByField("planName");
OcrDictionaryGroup planNameDG = ocrDictionaryService.queryGroupByField("plan");
if (planNameDG!=null && ocrDictionaryService.isValueExists(planNameDG.getId(), picture.getPlanname(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getPlanname());
@ -84,7 +84,7 @@ public class DataDictionaryUtil {
}
//3.拜访客户类型
if(picture.getField2()!=null) {
OcrDictionaryGroup field2DG = ocrDictionaryService.queryGroupByField("field2");
OcrDictionaryGroup field2DG = ocrDictionaryService.queryGroupByField("izcustomtype");
if (field2DG!=null && ocrDictionaryService.isValueExists(field2DG.getId(), picture.getField2(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField2());
@ -101,7 +101,7 @@ public class DataDictionaryUtil {
//4.拜访客户名称
if(picture.getField3()!=null) {
OcrDictionaryGroup field3DG = ocrDictionaryService.queryGroupByField("field3");
OcrDictionaryGroup field3DG = ocrDictionaryService.queryGroupByField("izcustomname");
if (field3DG!=null && ocrDictionaryService.isValueExists(field3DG.getId(), picture.getField3(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField3());
@ -118,7 +118,7 @@ public class DataDictionaryUtil {
//5.任务来源
if(picture.getField4()!=null) {
OcrDictionaryGroup field4DG = ocrDictionaryService.queryGroupByField("field4");
OcrDictionaryGroup field4DG = ocrDictionaryService.queryGroupByField("iztaskrrom");
if (field4DG!=null && ocrDictionaryService.isValueExists(field4DG.getId(), picture.getField4(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField4());
@ -134,7 +134,7 @@ public class DataDictionaryUtil {
}
//6.厂商
if(picture.getField5()!=null) {
OcrDictionaryGroup field5DG = ocrDictionaryService.queryGroupByField("field5");
OcrDictionaryGroup field5DG = ocrDictionaryService.queryGroupByField("izfirm");
if (field5DG!=null && ocrDictionaryService.isValueExists(field5DG.getId(), picture.getField5(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField5());
@ -150,7 +150,7 @@ public class DataDictionaryUtil {
}
//7.拜访客户级别
if(picture.getField6()!=null) {
OcrDictionaryGroup field6DG = ocrDictionaryService.queryGroupByField("field6");
OcrDictionaryGroup field6DG = ocrDictionaryService.queryGroupByField("izcustomlevel");
if (field6DG!=null && ocrDictionaryService.isValueExists(field6DG.getId(), picture.getField6(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField6());
@ -166,7 +166,7 @@ public class DataDictionaryUtil {
}
//8.拜访项目类别
if(picture.getField17()!=null) {
OcrDictionaryGroup field17DG = ocrDictionaryService.queryGroupByField("field17");
OcrDictionaryGroup field17DG = ocrDictionaryService.queryGroupByField("izprojecttype");
if (field17DG!=null && ocrDictionaryService.isValueExists(field17DG.getId(), picture.getField17(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField17());
@ -183,7 +183,7 @@ public class DataDictionaryUtil {
//9.任务状态
if(picture.getTaskstatus()!=null) {
OcrDictionaryGroup taskStatusDG = ocrDictionaryService.queryGroupByField("taskStatus");
OcrDictionaryGroup taskStatusDG = ocrDictionaryService.queryGroupByField("iztaskstatus");
if (taskStatusDG!=null && ocrDictionaryService.isValueExists(taskStatusDG.getId(), picture.getTaskstatus(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getTaskstatus());
@ -200,7 +200,7 @@ public class DataDictionaryUtil {
//10.产品名称
if(picture.getField9()!=null) {
OcrDictionaryGroup field9DG = ocrDictionaryService.queryGroupByField("field9");
OcrDictionaryGroup field9DG = ocrDictionaryService.queryGroupByField("izproductname");
if (field9DG!=null && ocrDictionaryService.isValueExists(field9DG.getId(), picture.getField9(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getField9());
@ -217,7 +217,7 @@ public class DataDictionaryUtil {
//11.地区(城市信息)
if(picture.getReleasearea()!=null) {
OcrDictionaryGroup releaseAreaDG = ocrDictionaryService.queryGroupByField("releaseArea");
OcrDictionaryGroup releaseAreaDG = ocrDictionaryService.queryGroupByField("izvisitcity");
if (releaseAreaDG!=null && ocrDictionaryService.isValueExists(releaseAreaDG.getId(), picture.getReleasearea(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getReleasearea());
@ -234,7 +234,7 @@ public class DataDictionaryUtil {
//12.地区(省信息)
if(picture.getReleaseprovince()!=null) {
OcrDictionaryGroup releaseProvinceDG = ocrDictionaryService.queryGroupByField("releaseProvince");
OcrDictionaryGroup releaseProvinceDG = ocrDictionaryService.queryGroupByField("izvisitpro");
if (releaseProvinceDG!=null && ocrDictionaryService.isValueExists(releaseProvinceDG.getId(), picture.getReleaseprovince(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getReleaseprovince());
@ -251,7 +251,7 @@ public class DataDictionaryUtil {
//13.项目信息
if(picture.getProjectidname()!=null) {
OcrDictionaryGroup projectNameDG = ocrDictionaryService.queryGroupByField("projectName");
OcrDictionaryGroup projectNameDG = ocrDictionaryService.queryGroupByField("project");
if (projectNameDG!=null && ocrDictionaryService.isValueExists(projectNameDG.getId(), picture.getProjectidname(), picture.getTenantId())) {
OcrDictionary dictionary = new OcrDictionary();
dictionary.setLable(picture.getProjectidname());

40
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java
Unescape View File

@ -0,0 +1,40 @@
package cn.jyjz.xiaoyao.ocr.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.URL;
/**
* 转储图片工具类
*
* @author hugh(shuli.yao) 1397940314@qq.com
* @version 1.0
* @date 2024/3/16 18:37
*/
public class DownloadImgUtil {
private static Logger logger = LoggerFactory.getLogger(DownloadImgUtil.class);
/**
* 下载网络图片缓存到本地服务器
* @param imageUrl
* @param destinationFile
*/
public static void downloadImage(String imageUrl, String destinationFile) {
try (InputStream in = new URL(imageUrl).openStream();
FileOutputStream out = new FileOutputStream(destinationFile)) {
byte[] buffer = new byte[4096];
int n;
while ((n = in.read(buffer)) != -1) {
out.write(buffer, 0, n);
}
} catch (Exception e) {
logger.error(e.getMessage());
}
}
}
Write Preview
Loading…
Cancel
Save