fix: 解决拉取无量云2.0数据丢失，线程未进行等待，请求接口太频繁问题

Reviewed-on: #71

jyjz-common/jyjz-common-base/src/main/java/cn/jyjz/xiaoyao/common/base/util/requestFormat/SearchQuery.java

@@ -124,6 +124,9 @@ public class SearchQuery implements java.io.Serializable{
     public void addGreaterEqual(String field,Date value){
         addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
     }
+    public void addGreaterEqual(String field,Long value){
+        addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
+    }
     public void addLess(String field,Integer value){
         addQuery(field,value, SearchQueryConstant.SEARCH_LESS);
     }
@@ -155,6 +158,10 @@ public class SearchQuery implements java.io.Serializable{
         addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
     }
 
+    public void addLessEqual(String field,Long value){
+        addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
+    }
+
     public void addRulesOp(String field,String rulesOp){
         SearchQueryForm searchQueryForm = new SearchQueryForm();
         if(query.get(field)!=null){

jyjz-framework/jyjz-framework-securityweb/src/main/java/cn/jyjz/xiaoyao/framework/security/authentication/JwtAuthenticationFilter.java

@@ -60,7 +60,7 @@ public class JwtAuthenticationFilter extends OncePerRequestFilter {
         String[] excludespath = systemopenurlService.getOpenUrl();
 
         //不需要过滤的url
-        String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs"};
+        String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs","/modeler","/app"};
 
         String[] urls =  ArrayUtils.addAll(excludespath, excludesOther);
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/flowable/service/impl/FlowTaskServiceImpl.java

@@ -1317,14 +1317,14 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
         query.put("searchassignee", user.getLoginname());
         query.putAll(searchQuery);
 
-//        if (!StringUtils.isBlank(userSearchId)) {
-//            OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId);
-//            SearchQuery additionalQuery = new SearchQuery();
-//            for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) {
-//                this.formattingSearchChild1(additionalQuery, usersearchchild);
-//            }
-//            query.putAll(additionalQuery.getQuery());
-//        }
+        if (!StringUtils.isBlank(userSearchId)) {
+            OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId);
+            SearchQuery additionalQuery = new SearchQuery();
+            for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) {
+                this.formattingSearchChild1(additionalQuery, usersearchchild);
+            }
+            query.putAll(additionalQuery.getQuery());
+        }
 
         Long count = ocrTaskchildPictureService.listmytaskCount(query);
 
@@ -1953,132 +1953,134 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
         return queryWrapper;
     }
 
-//    private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
-//        switch (usersearchchild.getSearchRelationType()) {
-//            case "where":
-//                // 添加 "where" 查询条件
-//                this.formattingSearchType1(searchQuery, usersearchchild);
-//                break;
-//            case "and":
-//                // 添加 "and" 查询条件
-//                searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and");
-//                SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild);
-//                searchQuery.setSearchQuery(andQuery);
-//                break;
-//            case "or":
-//                // 添加 "or" 查询条件
-//                searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or");
-//                SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild);
-//                searchQuery.setSearchQuery(orQuery);
-//                break;
-//            default:
-//                break;
-//        }
-//        return searchQuery;
-//    }
-//
-//    private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
-//        //查询类型 相等true 不相等fase
-//        boolean queryType = "eq".equals(usersearchchild.getSearchtype());
-//
-//        // 1.获取检查数据，确定数据类型
-//        String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue());
-//        List<String> arrayData = null; //集合型数据
-//        List<Integer> sectionInts = null; //纯数字区间型数据
-//        String[] sections = null;    //字符串区间型数据
-//        DateTime beginTime = null; //日期时间区间型数据
-//        DateTime endTime = null;  //日期时间区间型数据
-//
-//        // 2.根据数据类型，解构数据
-//        switch (valueType) {
-//            case "array":
-//                //处理多文件
-//                String[] datas = usersearchchild.getSearchvalue().split(",");
-//                arrayData = Arrays.asList(datas);
-//                break;
-//            case "section":
-//                //格式化数据,仅在做区间的时候去除空格，其他地方不进行空格去除.
-//                sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
-//                //校验是否纯数字,纯数字的话，要转成int,否则比对时会有问题
-//                sectionInts = new ArrayList<>();
-//                for (String data : sections) {
-//                    if (data.matches("\\d+")) {
-//                        sectionInts.add(Integer.parseInt(data));
-//                    }
-//                }
-//                break;
-//            case "time":
-//                //格式化时间数据
-//                String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
-//                beginTime = DateUtil.parse(times[0], "yyyy/MM/dd");
-//                endTime = DateUtil.parse(times[1], "yyyy/MM/dd");
-//                break;
-//            default:
-//        }
-//
-//        // 3.根据查询类型 组装查询条件 相等
-//        switch (valueType) {
-//            case "array":
-//                //集合数据
-//                //相等
-//                if (queryType) {
-//                    searchQuery.addIn(usersearchchild.getSearchfield(), arrayData);
-//                    break;
-//                }
-//                //不相等
-//                searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData);
-//                break;
-//
-//            case "section":
-//                //数字区间查询
-//                if (sectionInts != null && sectionInts.size() == 2) {
-//                    if (queryType) {
-//                        //相等
-//                        searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0));
-//                        searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1));
-//                        break;
-//                    }
-//                    //不相等
-//                    searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
-//                    searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
-//                    break;
-//                }
-//
-//                //字符串区间查询
-//                if (queryType) {
-//                    //相等
-//                    searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]);
-//                    searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]);
-//                    break;
-//                }
-//                //不相等
-//                searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]);
-//                searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]);
-//                break;
-//
-//
-//            case "time":
-//                //时间区间数据
-//                //相等
-//                if (queryType) {
-//                    searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), beginTime);
-//                    searchQuery.addLessEqual(usersearchchild.getSearchfield(), endTime);
-//                    break;
-//                }
-//                //不相等
-//                searchQuery.addLess(usersearchchild.getSearchfield(), beginTime);
-//                searchQuery.addGreater(usersearchchild.getSearchfield(), endTime);
-//                break;
-//
-//            default:
-//                if (queryType) {
-//                    searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
-//                    break;
-//                }
-//                searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
-//        }
-//        return searchQuery;
-//    }
+    private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
+        switch (usersearchchild.getSearchRelationType()) {
+            case "where":
+                // 添加 "where" 查询条件
+                this.formattingSearchType1(searchQuery, usersearchchild);
+                break;
+            case "and":
+                // 添加 "and" 查询条件
+                searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and");
+                SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild);
+                searchQuery.setSearchQuery(andQuery);
+                break;
+            case "or":
+                // 添加 "or" 查询条件
+                searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or");
+                SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild);
+                searchQuery.setSearchQuery(orQuery);
+                break;
+            default:
+                break;
+        }
+        return searchQuery;
+    }
+
+    private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
+        //查询类型 相等true 不相等fase
+        boolean queryType = "eq".equals(usersearchchild.getSearchtype());
+
+        // 1.获取检查数据，确定数据类型
+        String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue());
+        List<String> arrayData = null; //集合型数据
+        List<Integer> sectionInts = null; //纯数字区间型数据
+        String[] sections = null;    //字符串区间型数据
+        DateTime beginTime = null; //日期时间区间型数据
+        DateTime endTime = null;  //日期时间区间型数据
+
+        // 2.根据数据类型，解构数据
+        switch (valueType) {
+            case "array":
+                //处理多文件
+                String[] datas = usersearchchild.getSearchvalue().split(",");
+                arrayData = Arrays.asList(datas);
+                break;
+            case "section":
+                //格式化数据,仅在做区间的时候去除空格，其他地方不进行空格去除.
+                sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
+                //校验是否纯数字,纯数字的话，要转成int,否则比对时会有问题
+                sectionInts = new ArrayList<>();
+                for (String data : sections) {
+                    if (data.matches("\\d+")) {
+                        sectionInts.add(Integer.parseInt(data));
+                    }
+                }
+                break;
+            case "time":
+                //格式化时间数据
+                String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
+                beginTime = DateUtil.parse(times[0]+" 00:00:00", "yyyy/MM/dd HH:mm:ss");
+                endTime = DateUtil.parse(times[1]+" 23:59:59", "yyyy/MM/dd HH:mm:ss");
+                break;
+            default:
+        }
+
+        // 3.根据查询类型 组装查询条件 相等
+        switch (valueType) {
+            case "array":
+                //集合数据
+                //相等
+                if (queryType) {
+                    searchQuery.addIn(usersearchchild.getSearchfield(), arrayData);
+                    break;
+                }
+                //不相等
+                searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData);
+                break;
+
+            case "section":
+                //数字区间查询
+                if (sectionInts != null && sectionInts.size() == 2) {
+                    if (queryType) {
+                        //相等
+                        searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0));
+                        searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1));
+                        break;
+                    }
+                    //不相等
+
+                    searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
+                    searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
+                    break;
+                }
+
+                //字符串区间查询
+                if (queryType) {
+                    //相等
+                    searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]);
+                    searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]);
+                    break;
+                }
+                //不相等
+                searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]);
+                searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]);
+                break;
+
+
+            case "time":
+                //时间区间数据
+                //相等
+                if (queryType) {
+                    searchQuery.addGreaterEqual(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
+                    searchQuery.addLessEqual(usersearchchild.getSearchfield()+"End", endTime.getTime());
+                    break;
+                }
+                //不相等
+                searchQuery.addRulesOp(usersearchchild.getSearchfield()+"Start","or");
+                searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
+                searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime());
+                break;
+
+            default:
+                if (queryType) {
+                    searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
+                    break;
+                }
+                searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
+        }
+        return searchQuery;
+    }
 
 
 }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java

@@ -1,5 +1,6 @@
 package cn.jyjz.xiaoyao.ocr.controller;
 
+import cn.hutool.core.date.DateField;
 import cn.hutool.core.date.DateTime;
 import cn.hutool.core.date.DateUtil;
 import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
@@ -7,12 +8,14 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
 import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
 import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
 import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 import javax.servlet.http.HttpServletRequest;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
@@ -23,6 +26,7 @@ import java.util.List;
  * @version 1.0
  * @date 2024/3/14 19:47
  */
+@Slf4j
 @RestController
 @RequestMapping("/test/api")
 public class ApiTestController {
@@ -71,4 +75,98 @@ public class ApiTestController {
         return "处理失败";
     }
 
+
+    @GetMapping("psTest01")
+    public String psTset01(HttpServletRequest request){
+        String beginStr = request.getParameter("beginStr");
+        String endStr = request.getParameter("endStr");
+        String pageSize = request.getParameter("pageSize");
+        String pageNo = request.getParameter("pageNo");
+
+        PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
+        pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
+        pictureSourceParameter.setTenantNo(1714548920583360512L);
+        pictureSourceParameter.setAccountNo(1714887674875514880L);
+
+        List<String> notAddedToQueueTaskIds = new ArrayList<>();
+
+        Thread thread = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    int totalCount = 0;
+                    int totalLocalCount = 0;
+                    int totalSize = 0;
+                    DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
+                    DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
+                    while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
+                        int currentPage = Integer.parseInt(pageNo);
+                        boolean hasNextPage = true;
+                        boolean isCountAdded = false;
+
+                        while (hasNextPage) {
+                            // 设置当天的开始和结束时间
+                            Instant startInstant = startDate.toInstant();
+                            Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
+                            pictureSourceParameter.setStartTime(Date.from(startInstant));
+                            pictureSourceParameter.setEndTime(Date.from(endInstant));
+                            pictureSourceParameter.setPageNo(currentPage);
+
+                            ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
+                            List<PictureSourceResult> listResult = apiPage.getRecords();
+                            int count = apiPage.getTotal();
+                            int localCount = listResult.size();
+                            int size = 0;
+
+                            for (PictureSourceResult pictureSourceResult : listResult) {
+                                int retryCount = 0;
+                                boolean addedToQueue = false;
+                                if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
+                                    size++;
+                                    // 尝试将数据加入队列
+                                    while (!addedToQueue && retryCount < 5) { // 最多重试5次
+                                        addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult);
+                                        if (!addedToQueue) {
+                                            Thread.sleep(60000);
+                                            retryCount++;
+                                        }
+                                    }
+                                    if (!addedToQueue) {
+                                        log.error("无法将数据添加到队列，开始时间：{}，任务ID：{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
+                                        notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
+                                    }
+                                }
+                            }
+
+                            totalLocalCount += localCount;
+                            totalSize += size;
+
+                            if (!isCountAdded) {
+                                totalCount += count;
+                                isCountAdded = true; // 标记当天的统计信息已经被累加过了
+                            }
+
+                            // 输出当前日期范围内的统计信息
+                            log.error("{}图片总数:{}条，本次获取：{}条，可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
+                            // 准备处理下一页的数据
+                            currentPage++;
+                            // 检查是否有下一页
+                            if (localCount < Integer.parseInt(pageSize)) {
+                                hasNextPage = false;
+                            }
+                        }
+                        // 准备处理下一天的数据
+                        startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
+                    }
+                    log.error("未加入队列的任务ID列表：{}",notAddedToQueueTaskIds);
+                    log.error("当前区间内图片总数:{}条，本次获取：{}条，可处理数据:{}条",totalCount,totalLocalCount,totalSize);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+        thread.start();
+        return "开启入库";
+    }
 }
+

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/OcrCheckDuplicateController.java

@@ -81,7 +81,11 @@ public class OcrCheckDuplicateController {
         }
         //调用查询参数解析方法，解析为queryWrapper
         queryWrapper = SearchParaFormatting.searchParaParse(queryWrapper, paraMap);
-
+        //提报人模糊搜索
+        String upUserName = req.getParameter("upUserName");
+        if(StringUtils.isNotBlank(upUserName)){
+            queryWrapper.like("upName",upUserName);
+        }
         queryWrapper.eq("tenant_id", tenantId);
         queryWrapper.eq("suspiciousfile", "0");
         queryWrapper.orderBy(true,true,"submit_date_timestamp");
@@ -114,6 +118,9 @@ public class OcrCheckDuplicateController {
 
         OcrCheckDuplicate dto = this.ocrCheckDuplicateService.getCheckDuplicateStatus(checkDuplicateNo);
         if (null != dto) {
+            if(dto.getStatus()==2){
+                return ResultVoUtil.success("查重任务执行完毕",dto);
+            }
             return ResultVoUtil.success(dto);
         } else {
             return ResultVoUtil.error();

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/OcrPictureService.java

@@ -6,6 +6,7 @@ import cn.jyjz.xiaoyao.common.base.vo.ResultVo;
 import cn.jyjz.xiaoyao.common.base.vo.UserToken;
 import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseService;
 import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
+import cn.jyjz.xiaoyao.ocr.dataobject.OcrPictureInfo;
 import com.alibaba.fastjson.JSONObject;
 import com.baomidou.mybatisplus.core.conditions.Wrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
@@ -61,6 +62,12 @@ public interface OcrPictureService extends BaseService<OcrPicture>  {
      */
     int savePictures(List<OcrPicture> ocrPictureList);
 
+    /**
+     * 新增图片信息
+     * @param ocrPicture
+     */
+    void savePicturesInfo(OcrPicture ocrPicture);
+
     ResultVo createPackageTask(List<OcrPicture> pageList, String tenantId, UserToken userToken, String searchMonth, HttpServletRequest req, String buessinessno, Map<String, Object> paramMap);
 
     Map<String,String> getOcrPictureClassifyAndHash(String img);

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrCheckDuplicateServiceImpl.java

@@ -478,8 +478,7 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
 
         Page<OcrPicture> pageVO = new Page<OcrPicture>(pageNo, pageSize);
         pageVO.setRecords(collect);
-        pageVO.setTotal(records.size());
-
+        pageVO.setTotal(ocrCheckDuplicateResults.size());
         return pageVO;
     }
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/service/impl/OcrPictureServiceImpl.java

@@ -26,6 +26,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import liquibase.pro.packaged.E;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -38,7 +39,9 @@ import javax.imageio.ImageIO;
 import javax.servlet.http.HttpServletRequest;
 import java.awt.color.ColorSpace;
 import java.awt.image.BufferedImage;
+import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.math.BigDecimal;
 import java.net.URL;
 import java.net.URLConnection;
@@ -665,7 +668,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
     }
 
     @Override
-    public int savePictures(List<OcrPicture> ocrPictureList) {
+    public int savePictures(List<OcrPicture> ocrPictureList){
         //1.字典数据 解析入库
         boolean b = dataDictionaryUtil.pictureDataDictionarySaveArray(ocrPictureList);
 
@@ -673,16 +676,26 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
         //2.调整图片地址，补齐域名
         for (OcrPicture picture : ocrPictureList) {
             picture.setCreateTime(System.currentTimeMillis()); //获取创建时间戳
-//            picture.setImgurl(interfaceDomain + picture.getImgurl());
         }
 
-        boolean saveBatch = this.saveBatch(ocrPictureList);
-        //3.添加图片信息
-        if (saveBatch) {
-            this.savePicturesInfo(ocrPictureList);
-            this.convertImage(ocrPictureList);
-            return ocrPictureList.size();
+        try {
+            boolean saveBatch = false;
+            synchronized(lock) {
+                // 同步的代码块
+                saveBatch  = this.saveBatch(ocrPictureList);
+            }
+
+            //3.添加图片信息
+            if (saveBatch) {
+//            this.savePicturesInfo(ocrPictureList);
+                this.convertImage(ocrPictureList);
+                return ocrPictureList.size();
+            }
+            log.error("存储图片失败:{}",JSONObject.toJSON(ocrPictureList));
+        }catch (Exception e){
+            log.error("存储图片异常:{}",e.getMessage());
         }
+
         return 0;
     }
 
@@ -690,7 +703,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
         List<OcrPictureInfo> list = new ArrayList<>();
         for (OcrPicture ocrPicture : ocrPictureList) {
             if (StringUtils.isNotBlank(ocrPicture.getImgurl())) {
-                OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl());
+                OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl(),true);
                 if (ObjectUtil.isNotEmpty(imageInfo)) {
                     imageInfo.setPictureId(ocrPicture.getId());
                     imageInfo.setImgName(ocrPicture.getImgname());
@@ -701,18 +714,49 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
         ocrPictureInfoService.saveBatch(list);
     }
 
-    private OcrPictureInfo getImageInfo(String imageUrl) {
+    /**
+     * 新增图片信息表
+     * @param ocrPicture
+     */
+    @Override
+    public void savePicturesInfo(OcrPicture ocrPicture) {
+        boolean isConnection = true;
+        String url = ocrPicture.getImgurl();
+        if(!StringUtils.isEmpty(ocrPicture.getLocalpictrueurl())){
+            url = ocrPicture.getLocalpictrueurl();
+            isConnection = false;
+        }
+        OcrPictureInfo imageInfo = getImageInfo(url,isConnection);
+        imageInfo.setPictureId(ocrPicture.getId());
+        imageInfo.setImgName(ocrPicture.getImgname());
+        ocrPictureInfoService.save(imageInfo);
+    }
 
-        try {
-            // 创建URL对象
-            URL url = new URL(imageUrl);
-            // 打开连接
-            URLConnection urlConnection = url.openConnection();
-            // 获取图片大小
-            Integer imageSize = urlConnection.getContentLength();
+    /**
+     * 获取图片信息
+     * @param imageUrl 图片地址 两种 网络或者本地图片
+     * @param isConnection 是否是url 开启网络请求
+     * @return
+     */
+    private OcrPictureInfo getImageInfo(String imageUrl,boolean isConnection) {
 
-            // 使用ImageIO读取图片
-            BufferedImage image = ImageIO.read(url);
+        try {
+            BufferedImage image = null;
+            Integer imageSize = 0;
+            if(isConnection){
+                // 创建URL对象
+                URL url = new URL(imageUrl);
+                // 打开连接
+                URLConnection urlConnection = url.openConnection();
+                // 获取图片大小
+                imageSize = urlConnection.getContentLength();
+                // 使用ImageIO读取图片
+                image= ImageIO.read(url);
+            }else{
+                File file = new File(imageUrl);
+                image = ImageIO.read(file);
+                imageSize =(int)file.length();
+            }
 
             if (image != null) {
                 OcrPictureInfo info = new OcrPictureInfo();
@@ -751,12 +795,14 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
                 log.info("Cannot read image from URL.");
                 return null;
             }
-        } catch (IOException e) {
+        } catch (Exception e) {
             log.error("Error: " + e.getMessage());
         }
         return null;
     }
 
+
+
     private void convertImage(List<OcrPicture> list) {
         for (OcrPicture ocrPicture : list) {
             PictureImgToLocalEntity entity = new PictureImgToLocalEntity();
@@ -764,7 +810,22 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
             entity.setImgUrl(ocrPicture.getImgurl());
             String imgurl = ocrPicture.getImgurl();
             entity.setLocalPath(localImagePath + imgurl.substring(imgurl.lastIndexOf("/")));
-            TaskQueue.pictureImgToLocalPushData(entity);
+
+            boolean addedToQueue = false;
+            int count = 1;
+            while (!addedToQueue && count<=5){
+                try {
+                    addedToQueue = TaskQueue.pictureImgToLocalPushData(entity);
+                    if(addedToQueue) {
+                        break;
+                    }
+                    Thread.sleep(30000);
+                } catch (InterruptedException e) {
+                    log.error("放入图片下载队列,等待休眠异常:{}",e.getMessage());
+                }
+                count++;
+            }
+
         }
     }
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskQueue.java

@@ -20,12 +20,12 @@ public class TaskQueue {
     /**
      * 无量云接口数据处理 任务队列
      */
-    public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue();
+    public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue(1000);
 
     /**
      * picture图片转储到本地 任务队列
      */
-    public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
+    public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue(1000);
 
 
     /**

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/TaskThreadPool.java

@@ -21,6 +21,7 @@ public class TaskThreadPool {
 
     private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
     ExecutorService threadPool = null;
+    ExecutorService threadPoolImg = null;
 
     public TaskThreadPool() {
         if (threadPool == null) {
@@ -32,6 +33,15 @@ public class TaskThreadPool {
                     new LinkedBlockingDeque<>(),
                     Executors.defaultThreadFactory(),
                     new ThreadPoolExecutor.DiscardOldestPolicy());
+
+            threadPoolImg = new ThreadPoolExecutor(
+                    10,
+                    40,
+                    3,
+                    TimeUnit.SECONDS,
+                    new LinkedBlockingDeque<>(),
+                    Executors.defaultThreadFactory(),
+                    new ThreadPoolExecutor.DiscardOldestPolicy());
         }
     }
 
@@ -45,8 +55,8 @@ public class TaskThreadPool {
             @SneakyThrows
             @Override
             public void run() {
-                logger.debug("无量云接口数据处理消费线程检测中...");
                 while (true) {
+                    logger.debug("无量云接口数据处理消费线程检测中...,队列数量:{}",TaskQueue.pictureDisposeQueue.size());
                     PictureSourceResult pictureSourceResult = TaskQueue.pictureDisposePullData();
                     if (pictureSourceResult != null) {
                         threadPool.execute(new PictureDisposeTask(pictureSourceResult));
@@ -71,13 +81,12 @@ public class TaskThreadPool {
             @SneakyThrows
             @Override
             public void run() {
-                logger.debug("图片下载消费线程检测中...");
-
                 while (true) {
+                    logger.debug("图片下载消费线程检测中...,队列数量:{}",TaskQueue.pictureImgToLocalQueue.size());
                     try {
                         PictureImgToLocalEntity pictureImgToLocalEntity = TaskQueue.pictureImgToLocalPullData();
                         if (pictureImgToLocalEntity != null) {
-                            threadPool.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
+                            threadPoolImg.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
                         } else {
                             Thread.sleep(5000);
                         }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureDisposeTask.java

@@ -205,9 +205,9 @@ public class PictureDisposeTask implements Runnable{
                 picture.setSubmitDateTimestamp(pictureSourceResult.getSubmitTime());
 //                picture.setSubmitDateTimestamp(System.currentTimeMillis());
                 picture.setPhotoDateTimestamp(System.currentTimeMillis());
-
-                System.out.println("SubmitDateTimestamp1=================>>>>>>>"+pictureSourceResult.getSubmitTime());
-                System.out.println("SubmitDateTimestamp2=================>>>>>>>"+picture.getSubmitDateTimestamp());
+//
+//                System.out.println("SubmitDateTimestamp1=================>>>>>>>"+pictureSourceResult.getSubmitTime());
+//                System.out.println("SubmitDateTimestamp2=================>>>>>>>"+picture.getSubmitDateTimestamp());
 
                 //定位距离
                 picture.setField12(pictureSourceResult.getDistance());
@@ -230,6 +230,7 @@ public class PictureDisposeTask implements Runnable{
 
         //调用图片入库方法
         int size = ocrPictureService.savePictures(ocrPictureList);
+
     }
 
 

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/thread/tasks/PictureImgToLocalTask.java

@@ -74,12 +74,17 @@ public class PictureImgToLocalTask implements Runnable {
                     picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
                     picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
                 }
+
                 // 获取拍照时间
 //                Long timestamp = ocrPictureService.getPhotoDateTime(picture);
 //                if(timestamp != 0L){
 //                    picture.setPhotoDateTimestamp(timestamp);
 //                }
                 ocrPictureService.updateById(picture);
+                logger.info("图片本地化，回写数据完成!");
+                logger.info("开启获取图片信息操作!");
+                ocrPictureService.savePicturesInfo(picture);
+                logger.info("获取图片信息操作,结束!");
             }
         }
     }

jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/util/DownloadImgUtil.java

@@ -33,7 +33,7 @@ public class DownloadImgUtil {
                 out.write(buffer, 0, n);
             }
         } catch (Exception e) {
-            logger.error(e.getMessage());
+            logger.error("图片本地化，转储图片异常:",e.getMessage());
             return false;
         }
         return true;

jyjz-system/jyjz-system-oa/src/main/resources/mapper/oa/OcrPictureMapper.xml

@@ -236,7 +236,7 @@
 		<if test="pictureId != null and pictureId != ''">
 			and t1.picture_id = #{pictureId}
 		</if>
-        ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp DESC
+        ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp asc
         LIMIT 1
     </select>
 

jyjz-system/jyjz-system-oa/src/main/resources/mapper/oa/OcrTaskchildPictureMapper.xml

@@ -482,6 +482,91 @@
             </trim>
         </where>
     </sql>
+
+    <sql id="Base_Where_Union_New">
+        <where>
+            <trim prefixOverrides="and|or">
+                <if test="izupuser != null">
+                    ${izupuser.rulesOp} tcp.FROMUSERID ${izupuser.dataOp} ${izupuser.likestar}
+                    <!--				#{fromuserid.value}-->
+                    <if test="izupuser.value instanceof java.util.List">
+                        <foreach item="item" collection="izupuser.value" open="" separator="," close="">
+                            #{item}
+                        </foreach>
+                    </if>
+                    <if test="!izupuser.value instanceof java.util.List">
+                        #{izupuser.value}
+                    </if>
+                    ${izupuser.likeend}
+                </if>
+                <if test="izApprovalStatus != null">
+                    ${izApprovalStatus.rulesOp} tcp.FROMUSERID ${izApprovalStatus.dataOp} ${izApprovalStatus.likestar}
+                    <!--				#{fromuserid.value}-->
+                    <if test="izApprovalStatus.value instanceof java.util.List">
+                        <foreach item="item" collection="izApprovalStatus.value" open="" separator="," close="">
+                            #{item}
+                        </foreach>
+                    </if>
+                    <if test="!izApprovalStatus.value instanceof java.util.List">
+                        #{izApprovalStatus.value}
+                    </if>
+                    ${izApprovalStatus.likeend}
+                </if>
+                <if test="iztaskrrom != null">
+                    ${iztaskrrom.rulesOp} tcp.FROMSOURCEID ${iztaskrrom.dataOp} ${iztaskrrom.likestar}
+                    <!--				#{fromuserid.value}-->
+                    <if test="iztaskrrom.value instanceof java.util.List">
+                        <foreach item="item" collection="iztaskrrom.value" open="" separator="," close="">
+                            #{item}
+                        </foreach>
+                    </if>
+                    <if test="!iztaskrrom.value instanceof java.util.List">
+                        #{iztaskrrom.value}
+                    </if>
+                    ${iztaskrrom.likeend}
+                </if>
+                <if test="izproject != null">
+                    ${izproject.rulesOp} tcp.FROMPROJECTID ${izproject.dataOp} ${izproject.likestar}
+                    <!--				#{fromuserid.value}-->
+                    <if test="izproject.value instanceof java.util.List">
+                        <foreach item="item" collection="izproject.value" open="" separator="," close="">
+                            #{item}
+                        </foreach>
+                    </if>
+                    <if test="!izproject.value instanceof java.util.List">
+                        #{izproject.value}
+                    </if>
+                    ${izproject.likeend}
+                </if>
+<!--                <if test="izuptime != null">-->
+<!--                    ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}-->
+<!--                </if>-->
+<!--                <if test="izuptimeStart != null">-->
+<!--                    and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'-->
+<!--                </if>-->
+<!--                <if test="izuptimeEnd != null">-->
+<!--                    and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'-->
+<!--                </if>-->
+                <if test="izuptimeStart != null">
+                    <!-- 如果 izuptimeStart.dataOp 的值为 "<"，则使用 "or" 连接 -->
+                    <choose>
+                        <when test="izuptimeStart.rulesOp == 'or' || izuptimeEnd.rulesOp=='or'">
+                            <if test="izuptimeEnd != null">
+                                and (tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}' or tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}')
+                            </if>
+                        </when>
+                        <otherwise>
+                            <!-- 其他情况下仍然拼接 "and" 连接 -->
+                            and tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}'
+                            <if test="izuptimeEnd != null">
+                                and tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}'
+                            </if>
+                        </otherwise>
+                    </choose>
+                </if>
+            </trim>
+        </where>
+    </sql>
     <select id="listFinal" resultType="cn.jyjz.xiaoyao.ocr.dataobject.OcrTaskchildPicture"
             parameterType="java.util.Map">
         select tcp.*, pic.field11 reporttime, pic.similarity_score similarityscore from (
@@ -556,42 +641,45 @@
             parameterType="java.util.Map">
         select tcp.* from (
         select tc.*,
-               uf.COMMENTHIS,
-               uf.TASKNAME as TASKNAMEHIS,
-               uf.STATSHIS,
-               uf.TASKINDEX,
-               uf.TASKNO,
+        uf.COMMENTHIS,
+        uf.TASKNAME as TASKNAMEHIS,
+        uf.STATSHIS,
+        uf.TASKINDEX,
+        uf.TASKNO,
+
                op.field6 as fromCustomerLevel,
                op.releaseArea as fromCity
         from ocr_taskchild_picture tc
-                 LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID
-                 left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0
+        LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID
+        left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0
         where tc.ASSIGNEE = #{searchassignee}
-          and tc.TENANTID = #{searchtenantid}
-          and tc.STATES = 2
+        and tc.TENANTID = #{searchtenantid}
+        and tc.STATES = 2
         UNION
         select tc.*,
-               uf.COMMENTHIS,
-               uf.TASKNAME as TASKNAMEHIS,
-               uf.STATSHIS,
-               uf.TASKINDEX,
-               uf.TASKNO,
+        uf.COMMENTHIS,
+        uf.TASKNAME as TASKNAMEHIS,
+        uf.STATSHIS,
+        uf.TASKINDEX,
+        uf.TASKNO,
+
                op.field6 as fromCustomerLevel,
                op.releaseArea as fromCity
         from ocr_taskchild_picture tc,
-             OA_USERAPPROVE_T uf,
-             ocr_picture op
+        OA_USERAPPROVE_T uf,
+        ocr_picture op
         where tc.ID = uf.FORMID
-          and tc.PICTUREID = op.ID
-          and op.suspiciousfile = 0
-          and tc.TENANTID = #{searchtenantid}
+        and tc.PICTUREID = op.ID
+        and op.suspiciousfile = 0
+        and tc.TENANTID = #{searchtenantid}
         <if test="thisLoginUserid != null">
             and uf.USERID = #{thisLoginUserid}
         </if>
         ) as tcp
         <include refid="Base_Where_union">
         </include>
-
+        <include refid="Base_Where_Union_New">
+        </include>
         <if test="page != null">
             <if test="page.sortname != null">
                 order by tcp.${page.sortname} ${page.sortorder}
@@ -604,20 +692,22 @@
         select tc.*
         from ocr_taskchild_picture tc
         where tc.ASSIGNEE = #{searchassignee}
-          and tc.TENANTID = #{searchtenantid}
-          and tc.STATES = 2
+        and tc.TENANTID = #{searchtenantid}
+        and tc.STATES = 2
         UNION
         select tc.*
         from ocr_taskchild_picture tc,
-             OA_USERAPPROVE_T uf
+        OA_USERAPPROVE_T uf
         where tc.ID = uf.FORMID
-          and tc.TENANTID = #{searchtenantid}
+        and tc.TENANTID = #{searchtenantid}
         <if test="thisLoginUserid != null">
             and uf.USERID = #{thisLoginUserid}
         </if>
         ) as tcp
         <include refid="Base_Where_union">
         </include>
+        <include refid="Base_Where_Union_New">
+        </include>
     </select>
 
 
@@ -784,7 +874,7 @@
             and t1.picture_id != #{pictureId}
         </if>
         <if test="oderName == 'similarityScore'">
-            order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp ${oderType}
+            order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp asc
         </if>
         <if test="oderName == 'createdate'">
             order by t2.submit_date_timestamp ${oderType}