feat:任务图片本地化缓存

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/flowable/entity/DeModel.java

@@ -4,7 +4,6 @@ import com.baomidou.mybatisplus.annotation.TableName;
 import java.io.Serializable;
 import java.sql.Blob;
 import java.sql.Date;
-import java.time.LocalDateTime;
 @TableName("act_de_model")
 public class DeModel implements Serializable {
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java

@@ -7,13 +7,10 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
 import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
 import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
 import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
-import org.apache.commons.lang.time.DateFormatUtils;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
-
 import javax.servlet.http.HttpServletRequest;
 import java.time.Instant;
 import java.util.Date;
@@ -64,7 +61,7 @@ public class ApiTestController {
                 }
                 size++;
                 //将可以处理数据放入处理队列中
-                TaskQueue.pictureDownloadPushData(pictureSourceResult);
+                TaskQueue.pictureDisposePushData(pictureSourceResult);
             }
             return "当前区间内图片总数:"+count+"条，本次获取："+localCount+"条，可处理数据:"+size+"条";
         } catch (Exception e) {

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskQueue.java

@@ -1,6 +1,7 @@
 package cn.jyjz.xiaoyao.ocr.thread;
 
 import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
+import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -16,20 +17,25 @@ import java.util.concurrent.LinkedBlockingQueue;
 public class TaskQueue {
     private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
 
+    /**
+     * 无量云接口数据处理 任务队列
+     */
+    public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue();
 
     /**
-     * 图片下载任务队列
+     * picture图片转储到本地 任务队列
      */
-    public static LinkedBlockingQueue pictureDownloadQueue = new LinkedBlockingQueue();
+    public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
+
 
     /**
-     * 图片下载队列 push 队列值
+     *  无量云接口数据处理 push 队列值
      * @return
      * @param pictureSourceResult //三方拉取 获取到的图片对象
      */
-    public static boolean pictureDownloadPushData(PictureSourceResult pictureSourceResult){
+    public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult){
         try {
-            pictureDownloadQueue.put(pictureSourceResult);
+            pictureDisposeQueue.put(pictureSourceResult);
         } catch (InterruptedException e) {
             logger.error("任务队列添加异常:{}",e.getMessage());
             return false;
@@ -38,11 +44,34 @@ public class TaskQueue {
     }
 
     /**
-     * 图片下载队列 拉取队列值
+     *  无量云接口数据处理 拉取队列值
+     * @return
+     */
+    public static PictureSourceResult  pictureDisposePullData(){
+        return (PictureSourceResult) pictureDisposeQueue.poll();
+    }
+
+
+    /**
+     *  图片缓存本地队列 push 队列值
      * @return
+     * @param pictureImgToLocalEntity //图片下载对象
      */
-    public static PictureSourceResult  pictureDownloadPullData(){
+    public static boolean pictureImgToLocalPushData(PictureImgToLocalEntity pictureImgToLocalEntity){
-        return (PictureSourceResult) pictureDownloadQueue.poll();
+        try {
+            pictureImgToLocalQueue.put(pictureImgToLocalEntity);
+        } catch (InterruptedException e) {
+            logger.error("图片下载任务队列添加异常:{}",e.getMessage());
+            return false;
+        }
+        return  true;
     }
 
+    /**
+     *  无量云接口数据处理 拉取队列值
+     * @return
+     */
+    public static PictureImgToLocalEntity  pictureImgToLocalPullData(){
+        return (PictureImgToLocalEntity) pictureDisposeQueue.poll();
+    }
 }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskRunner.java

@@ -15,6 +15,7 @@ public class TaskRunner implements CommandLineRunner {
     @Override
     public void run(String... args) throws Exception {
         TaskThreadPool taskThreadPool = new TaskThreadPool();
-        taskThreadPool.startControllerPull();
+        taskThreadPool.startPictureDisposePull();
+        taskThreadPool.startPictureImgToLocalPull();
     }
 }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java

@@ -1,6 +1,8 @@
 package cn.jyjz.xiaoyao.ocr.thread;
 import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
-import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDownloadTask;
+import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
+import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureDisposeTask;
+import cn.jyjz.xiaoyao.ocr.thread.tasks.PictureImgToLocalTask;
 import lombok.SneakyThrows;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,7 +25,7 @@ public class TaskThreadPool {
         if(threadPool ==null){
             threadPool = new ThreadPoolExecutor(
                     10,
-                    20,
+                    40,
                     3,
                     TimeUnit.SECONDS,
                     new LinkedBlockingDeque<>(),
@@ -36,16 +38,41 @@ public class TaskThreadPool {
      * 启动任务 入库消费者
      * @return
      */
-    public boolean startControllerPull(){
+    public boolean startPictureDisposePull(){
         Runnable runnable = new Runnable() {
             @SneakyThrows
             @Override
             public void run() {
-                logger.info("图片下载任务线程检查中...");
+                logger.debug("无量云接口数据处理消费线程检测中...");
                 while (true){
-                    PictureSourceResult pictureSourceResult =  TaskQueue.pictureDownloadPullData();
+                    PictureSourceResult pictureSourceResult =  TaskQueue.pictureDisposePullData();
                     if(pictureSourceResult!=null){
-                        threadPool.execute(new PictureDownloadTask(pictureSourceResult));
+                        threadPool.execute(new PictureDisposeTask(pictureSourceResult));
+                    }else{
+                        Thread.sleep(5000);
+                    }
+                }
+            }
+        };
+        Thread thread=new Thread(runnable);
+        thread.start();
+        return true;
+    }
+
+    /**
+     * 启动任务 入库消费者
+     * @return
+     */
+    public boolean startPictureImgToLocalPull(){
+        Runnable runnable = new Runnable() {
+            @SneakyThrows
+            @Override
+            public void run() {
+                logger.debug("图片下载消费线程检测中...");
+                while (true){
+                    PictureImgToLocalEntity pictureImgToLocalEntity =  TaskQueue.pictureImgToLocalPullData();
+                    if(pictureImgToLocalEntity!=null){
+                        threadPool.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
                     }else{
                         Thread.sleep(5000);
                     }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/entity/PictureImgToLocalEntity.java

@@ -0,0 +1,31 @@
+package cn.jyjz.xiaoyao.ocr.thread.entity;
+
+import lombok.Data;
+
+/**
+ * 图片转储本地处理对象
+ *
+ * @author hugh(shuli.yao) 1397940314@qq.com
+ * @version 1.0
+ * @date 2024/3/16 17:11
+ */
+@Data
+public class PictureImgToLocalEntity {
+
+    /**
+     * 图片表id
+     */
+    private Long pictureId;
+
+    /**
+     * 图片地址
+     */
+    private String ImgUrl;
+
+
+    /**
+     * 本地存储地址
+     */
+    private String localPath;
+
+}

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureDisposeTask.java

@@ -13,18 +13,18 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * 图片下载任务
+ * 无量云接口 数据处理队列
  *
  * @author hugh(shuli.yao) 1397940314@qq.com
  * @version 1.0
  * @date 2024/3/14 11:15
  */
-public class PictureDownloadTask implements Runnable{
+public class PictureDisposeTask implements Runnable{
     protected Logger logger = LoggerFactory.getLogger(getClass());
 
     private PictureSourceResult pictureSourceResult;
 
-    public PictureDownloadTask(PictureSourceResult pictureSourceResult){
+    public PictureDisposeTask(PictureSourceResult pictureSourceResult){
         this.pictureSourceResult = pictureSourceResult;
     }
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java

@@ -0,0 +1,40 @@
+package cn.jyjz.xiaoyao.ocr.thread.tasks;
+
+import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
+import cn.jyjz.xiaoyao.common.base.util.StringUtils;
+import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
+import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
+import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
+import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
+import cn.jyjz.xiaoyao.ocr.util.DownloadImgUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * 无量云接口 数据处理队列
+ *
+ * @author hugh(shuli.yao) 1397940314@qq.com
+ * @version 1.0
+ * @date 2024/3/14 11:15
+ */
+public class PictureImgToLocalTask implements Runnable{
+    protected Logger logger = LoggerFactory.getLogger(getClass());
+
+    private PictureImgToLocalEntity pictureImgToLocal;
+
+    public PictureImgToLocalTask(PictureImgToLocalEntity pictureImgToLocal){
+        this.pictureImgToLocal = pictureImgToLocal;
+    }
+
+    @Override
+    public void run() {
+        //1.开始转储图片
+        DownloadImgUtil.downloadImage(pictureImgToLocal.getImgUrl(), pictureImgToLocal.getLocalPath());
+    }
+
+
+}

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DataDictionaryUtil.java

@@ -49,7 +49,7 @@ public class DataDictionaryUtil {
         for (OcrPicture picture : ocrPictureList) {
             //1.检查提报人字典是否包含数据
             if(picture.getUpname()!=null && picture.getUpuserid()!=null){
-                OcrDictionaryGroup upNameDG = ocrDictionaryService.queryGroupByField("upName");
+                OcrDictionaryGroup upNameDG = ocrDictionaryService.queryGroupByField("person");
                 if(upNameDG!=null && ocrDictionaryService.isValueExists(upNameDG.getId(),picture.getUpname(),picture.getTenantId())){
                     OcrDictionary dictionary=  new OcrDictionary();
                     dictionary.setLable(picture.getUpname());
@@ -68,7 +68,7 @@ public class DataDictionaryUtil {
 
             //2.任务计划
             if(picture.getPlanname()!=null && picture.getPlanid()!=null) {
-                OcrDictionaryGroup planNameDG = ocrDictionaryService.queryGroupByField("planName");
+                OcrDictionaryGroup planNameDG = ocrDictionaryService.queryGroupByField("plan");
                 if (planNameDG!=null && ocrDictionaryService.isValueExists(planNameDG.getId(), picture.getPlanname(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getPlanname());
@@ -84,7 +84,7 @@ public class DataDictionaryUtil {
             }
             //3.拜访客户类型
             if(picture.getField2()!=null) {
-                OcrDictionaryGroup field2DG = ocrDictionaryService.queryGroupByField("field2");
+                OcrDictionaryGroup field2DG = ocrDictionaryService.queryGroupByField("izcustomtype");
                 if (field2DG!=null && ocrDictionaryService.isValueExists(field2DG.getId(), picture.getField2(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField2());
@@ -101,7 +101,7 @@ public class DataDictionaryUtil {
 
             //4.拜访客户名称
             if(picture.getField3()!=null) {
-                OcrDictionaryGroup field3DG = ocrDictionaryService.queryGroupByField("field3");
+                OcrDictionaryGroup field3DG = ocrDictionaryService.queryGroupByField("izcustomname");
                 if (field3DG!=null && ocrDictionaryService.isValueExists(field3DG.getId(), picture.getField3(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField3());
@@ -118,7 +118,7 @@ public class DataDictionaryUtil {
 
             //5.任务来源
             if(picture.getField4()!=null) {
-                OcrDictionaryGroup field4DG = ocrDictionaryService.queryGroupByField("field4");
+                OcrDictionaryGroup field4DG = ocrDictionaryService.queryGroupByField("iztaskrrom");
                 if (field4DG!=null && ocrDictionaryService.isValueExists(field4DG.getId(), picture.getField4(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField4());
@@ -134,7 +134,7 @@ public class DataDictionaryUtil {
             }
             //6.厂商
             if(picture.getField5()!=null) {
-                OcrDictionaryGroup field5DG = ocrDictionaryService.queryGroupByField("field5");
+                OcrDictionaryGroup field5DG = ocrDictionaryService.queryGroupByField("izfirm");
                 if (field5DG!=null && ocrDictionaryService.isValueExists(field5DG.getId(), picture.getField5(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField5());
@@ -150,7 +150,7 @@ public class DataDictionaryUtil {
             }
             //7.拜访客户级别
             if(picture.getField6()!=null) {
-                OcrDictionaryGroup field6DG = ocrDictionaryService.queryGroupByField("field6");
+                OcrDictionaryGroup field6DG = ocrDictionaryService.queryGroupByField("izcustomlevel");
                 if (field6DG!=null && ocrDictionaryService.isValueExists(field6DG.getId(), picture.getField6(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField6());
@@ -166,7 +166,7 @@ public class DataDictionaryUtil {
             }
             //8.拜访项目类别
             if(picture.getField17()!=null) {
-                OcrDictionaryGroup field17DG = ocrDictionaryService.queryGroupByField("field17");
+                OcrDictionaryGroup field17DG = ocrDictionaryService.queryGroupByField("izprojecttype");
                 if (field17DG!=null && ocrDictionaryService.isValueExists(field17DG.getId(), picture.getField17(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField17());
@@ -183,7 +183,7 @@ public class DataDictionaryUtil {
 
             //9.任务状态
             if(picture.getTaskstatus()!=null) {
-                OcrDictionaryGroup taskStatusDG = ocrDictionaryService.queryGroupByField("taskStatus");
+                OcrDictionaryGroup taskStatusDG = ocrDictionaryService.queryGroupByField("iztaskstatus");
                 if (taskStatusDG!=null && ocrDictionaryService.isValueExists(taskStatusDG.getId(), picture.getTaskstatus(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getTaskstatus());
@@ -200,7 +200,7 @@ public class DataDictionaryUtil {
 
             //10.产品名称
             if(picture.getField9()!=null) {
-                OcrDictionaryGroup field9DG = ocrDictionaryService.queryGroupByField("field9");
+                OcrDictionaryGroup field9DG = ocrDictionaryService.queryGroupByField("izproductname");
                 if (field9DG!=null && ocrDictionaryService.isValueExists(field9DG.getId(), picture.getField9(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getField9());
@@ -217,7 +217,7 @@ public class DataDictionaryUtil {
 
             //11.地区(城市信息)
             if(picture.getReleasearea()!=null) {
-                OcrDictionaryGroup releaseAreaDG = ocrDictionaryService.queryGroupByField("releaseArea");
+                OcrDictionaryGroup releaseAreaDG = ocrDictionaryService.queryGroupByField("izvisitcity");
                 if (releaseAreaDG!=null && ocrDictionaryService.isValueExists(releaseAreaDG.getId(), picture.getReleasearea(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getReleasearea());
@@ -234,7 +234,7 @@ public class DataDictionaryUtil {
 
             //12.地区(省信息)
             if(picture.getReleaseprovince()!=null) {
-                OcrDictionaryGroup releaseProvinceDG = ocrDictionaryService.queryGroupByField("releaseProvince");
+                OcrDictionaryGroup releaseProvinceDG = ocrDictionaryService.queryGroupByField("izvisitpro");
                 if (releaseProvinceDG!=null && ocrDictionaryService.isValueExists(releaseProvinceDG.getId(), picture.getReleaseprovince(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getReleaseprovince());
@@ -251,7 +251,7 @@ public class DataDictionaryUtil {
 
             //13.项目信息
             if(picture.getProjectidname()!=null) {
-                OcrDictionaryGroup projectNameDG = ocrDictionaryService.queryGroupByField("projectName");
+                OcrDictionaryGroup projectNameDG = ocrDictionaryService.queryGroupByField("project");
                 if (projectNameDG!=null && ocrDictionaryService.isValueExists(projectNameDG.getId(), picture.getProjectidname(), picture.getTenantId())) {
                     OcrDictionary dictionary = new OcrDictionary();
                     dictionary.setLable(picture.getProjectidname());

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java

@@ -0,0 +1,40 @@
+package cn.jyjz.xiaoyao.ocr.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.net.URL;
+
+/**
+ * 转储图片工具类
+ *
+ * @author hugh(shuli.yao) 1397940314@qq.com
+ * @version 1.0
+ * @date 2024/3/16 18:37
+ */
+public class DownloadImgUtil {
+    private static Logger logger = LoggerFactory.getLogger(DownloadImgUtil.class);
+
+    /**
+     * 下载网络图片缓存到本地服务器
+     * @param imageUrl
+     * @param destinationFile
+     */
+    public static void downloadImage(String imageUrl, String destinationFile) {
+        try (InputStream in = new URL(imageUrl).openStream();
+             FileOutputStream out = new FileOutputStream(destinationFile)) {
+
+            byte[] buffer = new byte[4096];
+            int n;
+            while ((n = in.read(buffer)) != -1) {
+                out.write(buffer, 0, n);
+            }
+        } catch (Exception e) {
+            logger.error(e.getMessage());
+        }
+    }
+
+
+}