Compare commits

..

1 Commits

@ -124,9 +124,6 @@ public class SearchQuery implements java.io.Serializable{
public void addGreaterEqual(String field,Date value){ public void addGreaterEqual(String field,Date value){
addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL); addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
} }
public void addGreaterEqual(String field,Long value){
addQuery(field,value, SearchQueryConstant.SEARCH_GREATER_EQUAL);
}
public void addLess(String field,Integer value){ public void addLess(String field,Integer value){
addQuery(field,value, SearchQueryConstant.SEARCH_LESS); addQuery(field,value, SearchQueryConstant.SEARCH_LESS);
} }
@ -158,10 +155,6 @@ public class SearchQuery implements java.io.Serializable{
addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL); addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
} }
public void addLessEqual(String field,Long value){
addQuery(field,value, SearchQueryConstant.SEARCH_LESS_EQUAL);
}
public void addRulesOp(String field,String rulesOp){ public void addRulesOp(String field,String rulesOp){
SearchQueryForm searchQueryForm = new SearchQueryForm(); SearchQueryForm searchQueryForm = new SearchQueryForm();
if(query.get(field)!=null){ if(query.get(field)!=null){

@ -60,7 +60,7 @@ public class JwtAuthenticationFilter extends OncePerRequestFilter {
String[] excludespath = systemopenurlService.getOpenUrl(); String[] excludespath = systemopenurlService.getOpenUrl();
//不需要过滤的url //不需要过滤的url
String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs","/modeler","/app"}; String[] excludesOther = {"/static","/json",".js",".css",".ico",".jpg",".png",".pcm","/druid","/doc.html","/swagger-ui.html","/swagger-resources","/webjars/**","/v2/api-docs"};
String[] urls = ArrayUtils.addAll(excludespath, excludesOther); String[] urls = ArrayUtils.addAll(excludespath, excludesOther);

@ -1317,14 +1317,14 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
query.put("searchassignee", user.getLoginname()); query.put("searchassignee", user.getLoginname());
query.putAll(searchQuery); query.putAll(searchQuery);
if (!StringUtils.isBlank(userSearchId)) { // if (!StringUtils.isBlank(userSearchId)) {
OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId); // OcrUsersearch ocrUsersearch = ocrUsersearchService.selectByUserSearchId(userSearchId);
SearchQuery additionalQuery = new SearchQuery(); // SearchQuery additionalQuery = new SearchQuery();
for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) { // for (OcrUsersearchchild usersearchchild : ocrUsersearch.getOcrUsersearchchildList()) {
this.formattingSearchChild1(additionalQuery, usersearchchild); // this.formattingSearchChild1(additionalQuery, usersearchchild);
} // }
query.putAll(additionalQuery.getQuery()); // query.putAll(additionalQuery.getQuery());
} // }
Long count = ocrTaskchildPictureService.listmytaskCount(query); Long count = ocrTaskchildPictureService.listmytaskCount(query);
@ -1953,134 +1953,132 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
return queryWrapper; return queryWrapper;
} }
private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) { // private SearchQuery formattingSearchChild1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
switch (usersearchchild.getSearchRelationType()) { // switch (usersearchchild.getSearchRelationType()) {
case "where": // case "where":
// 添加 "where" 查询条件 // // 添加 "where" 查询条件
this.formattingSearchType1(searchQuery, usersearchchild); // this.formattingSearchType1(searchQuery, usersearchchild);
break; // break;
case "and": // case "and":
// 添加 "and" 查询条件 // // 添加 "and" 查询条件
searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and"); // searchQuery.addRulesOp(usersearchchild.getSearchfield(),"and");
SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild); // SearchQuery andQuery = this.formattingSearchType1(searchQuery, usersearchchild);
searchQuery.setSearchQuery(andQuery); // searchQuery.setSearchQuery(andQuery);
break; // break;
case "or": // case "or":
// 添加 "or" 查询条件 // // 添加 "or" 查询条件
searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or"); // searchQuery.addRulesOp(usersearchchild.getSearchfield(),"or");
SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild); // SearchQuery orQuery = this.formattingSearchType1(searchQuery, usersearchchild);
searchQuery.setSearchQuery(orQuery); // searchQuery.setSearchQuery(orQuery);
break; // break;
default: // default:
break; // break;
} // }
return searchQuery; // return searchQuery;
} // }
//
private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) { // private SearchQuery formattingSearchType1(SearchQuery searchQuery, OcrUsersearchchild usersearchchild) {
//查询类型 相等true 不相等fase // //查询类型 相等true 不相等fase
boolean queryType = "eq".equals(usersearchchild.getSearchtype()); // boolean queryType = "eq".equals(usersearchchild.getSearchtype());
//
// 1.获取检查数据,确定数据类型 // // 1.获取检查数据,确定数据类型
String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue()); // String valueType = SearchParaFormatting.clickSearchValue(usersearchchild.getSearchvalue());
List<String> arrayData = null; //集合型数据 // List<String> arrayData = null; //集合型数据
List<Integer> sectionInts = null; //纯数字区间型数据 // List<Integer> sectionInts = null; //纯数字区间型数据
String[] sections = null; //字符串区间型数据 // String[] sections = null; //字符串区间型数据
DateTime beginTime = null; //日期时间区间型数据 // DateTime beginTime = null; //日期时间区间型数据
DateTime endTime = null; //日期时间区间型数据 // DateTime endTime = null; //日期时间区间型数据
//
// 2.根据数据类型,解构数据 // // 2.根据数据类型,解构数据
switch (valueType) { // switch (valueType) {
case "array": // case "array":
//处理多文件 // //处理多文件
String[] datas = usersearchchild.getSearchvalue().split(","); // String[] datas = usersearchchild.getSearchvalue().split(",");
arrayData = Arrays.asList(datas); // arrayData = Arrays.asList(datas);
break; // break;
case "section": // case "section":
//格式化数据,仅在做区间的时候去除空格,其他地方不进行空格去除. // //格式化数据,仅在做区间的时候去除空格,其他地方不进行空格去除.
sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-"); // sections = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
//校验是否纯数字,纯数字的话要转成int,否则比对时会有问题 // //校验是否纯数字,纯数字的话要转成int,否则比对时会有问题
sectionInts = new ArrayList<>(); // sectionInts = new ArrayList<>();
for (String data : sections) { // for (String data : sections) {
if (data.matches("\\d+")) { // if (data.matches("\\d+")) {
sectionInts.add(Integer.parseInt(data)); // sectionInts.add(Integer.parseInt(data));
} // }
} // }
break; // break;
case "time": // case "time":
//格式化时间数据 // //格式化时间数据
String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-"); // String[] times = usersearchchild.getSearchvalue().trim().replaceAll("\\s+", "").split("-");
beginTime = DateUtil.parse(times[0]+" 00:00:00", "yyyy/MM/dd HH:mm:ss"); // beginTime = DateUtil.parse(times[0], "yyyy/MM/dd");
endTime = DateUtil.parse(times[1]+" 23:59:59", "yyyy/MM/dd HH:mm:ss"); // endTime = DateUtil.parse(times[1], "yyyy/MM/dd");
break; // break;
default: // default:
} // }
//
// 3.根据查询类型 组装查询条件 相等 // // 3.根据查询类型 组装查询条件 相等
switch (valueType) { // switch (valueType) {
case "array": // case "array":
//集合数据 // //集合数据
//相等 // //相等
if (queryType) { // if (queryType) {
searchQuery.addIn(usersearchchild.getSearchfield(), arrayData); // searchQuery.addIn(usersearchchild.getSearchfield(), arrayData);
break; // break;
} // }
//不相等 // //不相等
searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData); // searchQuery.addNotIn(usersearchchild.getSearchfield(), arrayData);
break; // break;
//
case "section": // case "section":
//数字区间查询 // //数字区间查询
if (sectionInts != null && sectionInts.size() == 2) { // if (sectionInts != null && sectionInts.size() == 2) {
if (queryType) { // if (queryType) {
//相等 // //相等
searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0)); // searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1)); // searchQuery.addLessEqual(usersearchchild.getSearchfield(), sectionInts.get(1));
break; // break;
} // }
//不相等 // //不相等
// searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0)); // searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1)); // break;
break; // }
} //
// //字符串区间查询
//字符串区间查询 // if (queryType) {
if (queryType) { // //相等
//相等 // searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]);
searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), sections[0]); // searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]);
searchQuery.addLessEqual(usersearchchild.getSearchfield(), sections[1]); // break;
break; // }
} // //不相等
//不相等 // searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]);
searchQuery.addLess(usersearchchild.getSearchfield(), sections[0]); // searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]);
searchQuery.addGreater(usersearchchild.getSearchfield(), sections[1]); // break;
break; //
//
// case "time":
case "time": // //时间区间数据
//时间区间数据 // //相等
//相等 // if (queryType) {
if (queryType) { // searchQuery.addGreaterEqual(usersearchchild.getSearchfield(), beginTime);
searchQuery.addGreaterEqual(usersearchchild.getSearchfield()+"Start", beginTime.getTime()); // searchQuery.addLessEqual(usersearchchild.getSearchfield(), endTime);
searchQuery.addLessEqual(usersearchchild.getSearchfield()+"End", endTime.getTime()); // break;
break; // }
} // //不相等
//不相等 // searchQuery.addLess(usersearchchild.getSearchfield(), beginTime);
searchQuery.addRulesOp(usersearchchild.getSearchfield()+"Start","or"); // searchQuery.addGreater(usersearchchild.getSearchfield(), endTime);
searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime()); // break;
searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime()); //
break; // default:
// if (queryType) {
default: // searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
if (queryType) { // break;
searchQuery.addEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue()); // }
break; // searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue());
} // }
searchQuery.addNotEqual(usersearchchild.getSearchfield(), usersearchchild.getSearchvalue()); // return searchQuery;
} // }
return searchQuery;
}
} }

@ -1,6 +1,5 @@
package cn.jyjz.xiaoyao.ocr.controller; package cn.jyjz.xiaoyao.ocr.controller;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime; import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil; import cn.hutool.core.date.DateUtil;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi; import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
@ -8,14 +7,12 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage; import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@ -26,7 +23,6 @@ import java.util.List;
* @version 1.0 * @version 1.0
* @date 2024/3/14 19:47 * @date 2024/3/14 19:47
*/ */
@Slf4j
@RestController @RestController
@RequestMapping("/test/api") @RequestMapping("/test/api")
public class ApiTestController { public class ApiTestController {
@ -75,98 +71,4 @@ public class ApiTestController {
return "处理失败"; return "处理失败";
} }
@GetMapping("psTest01")
public String psTset01(HttpServletRequest request){
String beginStr = request.getParameter("beginStr");
String endStr = request.getParameter("endStr");
String pageSize = request.getParameter("pageSize");
String pageNo = request.getParameter("pageNo");
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
pictureSourceParameter.setPageSize(Integer.parseInt(pageSize));
pictureSourceParameter.setTenantNo(1714548920583360512L);
pictureSourceParameter.setAccountNo(1714887674875514880L);
List<String> notAddedToQueueTaskIds = new ArrayList<>();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
int totalCount = 0;
int totalLocalCount = 0;
int totalSize = 0;
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true;
boolean isCountAdded = false;
while (hasNextPage) {
// 设置当天的开始和结束时间
Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
int retryCount = 0;
boolean addedToQueue = false;
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
size++;
// 尝试将数据加入队列
while (!addedToQueue && retryCount < 5) { // 最多重试5次
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult);
if (!addedToQueue) {
Thread.sleep(60000);
retryCount++;
}
}
if (!addedToQueue) {
log.error("无法将数据添加到队列,开始时间:{}任务ID{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
}
}
}
totalLocalCount += localCount;
totalSize += size;
if (!isCountAdded) {
totalCount += count;
isCountAdded = true; // 标记当天的统计信息已经被累加过了
}
// 输出当前日期范围内的统计信息
log.error("{}图片总数:{}条,本次获取:{}条,可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
}
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
}
log.error("未加入队列的任务ID列表{}",notAddedToQueueTaskIds);
log.error("当前区间内图片总数:{}条,本次获取:{}条,可处理数据:{}条",totalCount,totalLocalCount,totalSize);
} catch (Exception e) {
e.printStackTrace();
}
}
});
thread.start();
return "开启入库";
}
} }

@ -81,11 +81,7 @@ public class OcrCheckDuplicateController {
} }
//调用查询参数解析方法解析为queryWrapper //调用查询参数解析方法解析为queryWrapper
queryWrapper = SearchParaFormatting.searchParaParse(queryWrapper, paraMap); queryWrapper = SearchParaFormatting.searchParaParse(queryWrapper, paraMap);
//提报人模糊搜索
String upUserName = req.getParameter("upUserName");
if(StringUtils.isNotBlank(upUserName)){
queryWrapper.like("upName",upUserName);
}
queryWrapper.eq("tenant_id", tenantId); queryWrapper.eq("tenant_id", tenantId);
queryWrapper.eq("suspiciousfile", "0"); queryWrapper.eq("suspiciousfile", "0");
queryWrapper.orderBy(true,true,"submit_date_timestamp"); queryWrapper.orderBy(true,true,"submit_date_timestamp");
@ -118,9 +114,6 @@ public class OcrCheckDuplicateController {
OcrCheckDuplicate dto = this.ocrCheckDuplicateService.getCheckDuplicateStatus(checkDuplicateNo); OcrCheckDuplicate dto = this.ocrCheckDuplicateService.getCheckDuplicateStatus(checkDuplicateNo);
if (null != dto) { if (null != dto) {
if(dto.getStatus()==2){
return ResultVoUtil.success("查重任务执行完毕",dto);
}
return ResultVoUtil.success(dto); return ResultVoUtil.success(dto);
} else { } else {
return ResultVoUtil.error(); return ResultVoUtil.error();

@ -6,7 +6,6 @@ import cn.jyjz.xiaoyao.common.base.vo.ResultVo;
import cn.jyjz.xiaoyao.common.base.vo.UserToken; import cn.jyjz.xiaoyao.common.base.vo.UserToken;
import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseService; import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseService;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture; import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPictureInfo;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.Wrapper; import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
@ -62,12 +61,6 @@ public interface OcrPictureService extends BaseService<OcrPicture> {
*/ */
int savePictures(List<OcrPicture> ocrPictureList); int savePictures(List<OcrPicture> ocrPictureList);
/**
*
* @param ocrPicture
*/
void savePicturesInfo(OcrPicture ocrPicture);
ResultVo createPackageTask(List<OcrPicture> pageList, String tenantId, UserToken userToken, String searchMonth, HttpServletRequest req, String buessinessno, Map<String, Object> paramMap); ResultVo createPackageTask(List<OcrPicture> pageList, String tenantId, UserToken userToken, String searchMonth, HttpServletRequest req, String buessinessno, Map<String, Object> paramMap);
Map<String,String> getOcrPictureClassifyAndHash(String img); Map<String,String> getOcrPictureClassifyAndHash(String img);

@ -478,7 +478,8 @@ public class OcrCheckDuplicateServiceImpl extends ServiceImpl<OcrCheckDuplicateM
Page<OcrPicture> pageVO = new Page<OcrPicture>(pageNo, pageSize); Page<OcrPicture> pageVO = new Page<OcrPicture>(pageNo, pageSize);
pageVO.setRecords(collect); pageVO.setRecords(collect);
pageVO.setTotal(ocrCheckDuplicateResults.size()); pageVO.setTotal(records.size());
return pageVO; return pageVO;
} }

@ -26,7 +26,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import liquibase.pro.packaged.E;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -39,9 +38,7 @@ import javax.imageio.ImageIO;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.awt.color.ColorSpace; import java.awt.color.ColorSpace;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.net.URL; import java.net.URL;
import java.net.URLConnection; import java.net.URLConnection;
@ -668,7 +665,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
} }
@Override @Override
public int savePictures(List<OcrPicture> ocrPictureList){ public int savePictures(List<OcrPicture> ocrPictureList) {
//1.字典数据 解析入库 //1.字典数据 解析入库
boolean b = dataDictionaryUtil.pictureDataDictionarySaveArray(ocrPictureList); boolean b = dataDictionaryUtil.pictureDataDictionarySaveArray(ocrPictureList);
@ -676,26 +673,16 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
//2.调整图片地址,补齐域名 //2.调整图片地址,补齐域名
for (OcrPicture picture : ocrPictureList) { for (OcrPicture picture : ocrPictureList) {
picture.setCreateTime(System.currentTimeMillis()); //获取创建时间戳 picture.setCreateTime(System.currentTimeMillis()); //获取创建时间戳
// picture.setImgurl(interfaceDomain + picture.getImgurl());
} }
try { boolean saveBatch = this.saveBatch(ocrPictureList);
boolean saveBatch = false; //3.添加图片信息
synchronized(lock) { if (saveBatch) {
// 同步的代码块 this.savePicturesInfo(ocrPictureList);
saveBatch = this.saveBatch(ocrPictureList); this.convertImage(ocrPictureList);
} return ocrPictureList.size();
//3.添加图片信息
if (saveBatch) {
// this.savePicturesInfo(ocrPictureList);
this.convertImage(ocrPictureList);
return ocrPictureList.size();
}
log.error("存储图片失败:{}",JSONObject.toJSON(ocrPictureList));
}catch (Exception e){
log.error("存储图片异常:{}",e.getMessage());
} }
return 0; return 0;
} }
@ -703,7 +690,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
List<OcrPictureInfo> list = new ArrayList<>(); List<OcrPictureInfo> list = new ArrayList<>();
for (OcrPicture ocrPicture : ocrPictureList) { for (OcrPicture ocrPicture : ocrPictureList) {
if (StringUtils.isNotBlank(ocrPicture.getImgurl())) { if (StringUtils.isNotBlank(ocrPicture.getImgurl())) {
OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl(),true); OcrPictureInfo imageInfo = getImageInfo(ocrPicture.getImgurl());
if (ObjectUtil.isNotEmpty(imageInfo)) { if (ObjectUtil.isNotEmpty(imageInfo)) {
imageInfo.setPictureId(ocrPicture.getId()); imageInfo.setPictureId(ocrPicture.getId());
imageInfo.setImgName(ocrPicture.getImgname()); imageInfo.setImgName(ocrPicture.getImgname());
@ -714,49 +701,18 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
ocrPictureInfoService.saveBatch(list); ocrPictureInfoService.saveBatch(list);
} }
/** private OcrPictureInfo getImageInfo(String imageUrl) {
*
* @param ocrPicture
*/
@Override
public void savePicturesInfo(OcrPicture ocrPicture) {
boolean isConnection = true;
String url = ocrPicture.getImgurl();
if(!StringUtils.isEmpty(ocrPicture.getLocalpictrueurl())){
url = ocrPicture.getLocalpictrueurl();
isConnection = false;
}
OcrPictureInfo imageInfo = getImageInfo(url,isConnection);
imageInfo.setPictureId(ocrPicture.getId());
imageInfo.setImgName(ocrPicture.getImgname());
ocrPictureInfoService.save(imageInfo);
}
/**
*
* @param imageUrl
* @param isConnection url
* @return
*/
private OcrPictureInfo getImageInfo(String imageUrl,boolean isConnection) {
try { try {
BufferedImage image = null; // 创建URL对象
Integer imageSize = 0; URL url = new URL(imageUrl);
if(isConnection){ // 打开连接
// 创建URL对象 URLConnection urlConnection = url.openConnection();
URL url = new URL(imageUrl); // 获取图片大小
// 打开连接 Integer imageSize = urlConnection.getContentLength();
URLConnection urlConnection = url.openConnection();
// 获取图片大小 // 使用ImageIO读取图片
imageSize = urlConnection.getContentLength(); BufferedImage image = ImageIO.read(url);
// 使用ImageIO读取图片
image= ImageIO.read(url);
}else{
File file = new File(imageUrl);
image = ImageIO.read(file);
imageSize =(int)file.length();
}
if (image != null) { if (image != null) {
OcrPictureInfo info = new OcrPictureInfo(); OcrPictureInfo info = new OcrPictureInfo();
@ -795,14 +751,12 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
log.info("Cannot read image from URL."); log.info("Cannot read image from URL.");
return null; return null;
} }
} catch (Exception e) { } catch (IOException e) {
log.error("Error: " + e.getMessage()); log.error("Error: " + e.getMessage());
} }
return null; return null;
} }
private void convertImage(List<OcrPicture> list) { private void convertImage(List<OcrPicture> list) {
for (OcrPicture ocrPicture : list) { for (OcrPicture ocrPicture : list) {
PictureImgToLocalEntity entity = new PictureImgToLocalEntity(); PictureImgToLocalEntity entity = new PictureImgToLocalEntity();
@ -810,22 +764,7 @@ public class OcrPictureServiceImpl extends BaseServiceImpl<OcrPictureMybatisDao,
entity.setImgUrl(ocrPicture.getImgurl()); entity.setImgUrl(ocrPicture.getImgurl());
String imgurl = ocrPicture.getImgurl(); String imgurl = ocrPicture.getImgurl();
entity.setLocalPath(localImagePath + imgurl.substring(imgurl.lastIndexOf("/"))); entity.setLocalPath(localImagePath + imgurl.substring(imgurl.lastIndexOf("/")));
TaskQueue.pictureImgToLocalPushData(entity);
boolean addedToQueue = false;
int count = 1;
while (!addedToQueue && count<=5){
try {
addedToQueue = TaskQueue.pictureImgToLocalPushData(entity);
if(addedToQueue) {
break;
}
Thread.sleep(30000);
} catch (InterruptedException e) {
log.error("放入图片下载队列,等待休眠异常:{}",e.getMessage());
}
count++;
}
} }
} }

@ -20,12 +20,12 @@ public class TaskQueue {
/** /**
* *
*/ */
public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue(1000); public static LinkedBlockingQueue pictureDisposeQueue = new LinkedBlockingQueue();
/** /**
* picture * picture
*/ */
public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue(1000); public static LinkedBlockingQueue pictureImgToLocalQueue = new LinkedBlockingQueue();
/** /**

@ -21,7 +21,6 @@ public class TaskThreadPool {
private static Logger logger = LoggerFactory.getLogger(TaskQueue.class); private static Logger logger = LoggerFactory.getLogger(TaskQueue.class);
ExecutorService threadPool = null; ExecutorService threadPool = null;
ExecutorService threadPoolImg = null;
public TaskThreadPool() { public TaskThreadPool() {
if (threadPool == null) { if (threadPool == null) {
@ -33,15 +32,6 @@ public class TaskThreadPool {
new LinkedBlockingDeque<>(), new LinkedBlockingDeque<>(),
Executors.defaultThreadFactory(), Executors.defaultThreadFactory(),
new ThreadPoolExecutor.DiscardOldestPolicy()); new ThreadPoolExecutor.DiscardOldestPolicy());
threadPoolImg = new ThreadPoolExecutor(
10,
40,
3,
TimeUnit.SECONDS,
new LinkedBlockingDeque<>(),
Executors.defaultThreadFactory(),
new ThreadPoolExecutor.DiscardOldestPolicy());
} }
} }
@ -55,8 +45,8 @@ public class TaskThreadPool {
@SneakyThrows @SneakyThrows
@Override @Override
public void run() { public void run() {
logger.debug("无量云接口数据处理消费线程检测中...");
while (true) { while (true) {
logger.debug("无量云接口数据处理消费线程检测中...,队列数量:{}",TaskQueue.pictureDisposeQueue.size());
PictureSourceResult pictureSourceResult = TaskQueue.pictureDisposePullData(); PictureSourceResult pictureSourceResult = TaskQueue.pictureDisposePullData();
if (pictureSourceResult != null) { if (pictureSourceResult != null) {
threadPool.execute(new PictureDisposeTask(pictureSourceResult)); threadPool.execute(new PictureDisposeTask(pictureSourceResult));
@ -81,12 +71,13 @@ public class TaskThreadPool {
@SneakyThrows @SneakyThrows
@Override @Override
public void run() { public void run() {
logger.debug("图片下载消费线程检测中...");
while (true) { while (true) {
logger.debug("图片下载消费线程检测中...,队列数量:{}",TaskQueue.pictureImgToLocalQueue.size());
try { try {
PictureImgToLocalEntity pictureImgToLocalEntity = TaskQueue.pictureImgToLocalPullData(); PictureImgToLocalEntity pictureImgToLocalEntity = TaskQueue.pictureImgToLocalPullData();
if (pictureImgToLocalEntity != null) { if (pictureImgToLocalEntity != null) {
threadPoolImg.execute(new PictureImgToLocalTask(pictureImgToLocalEntity)); threadPool.execute(new PictureImgToLocalTask(pictureImgToLocalEntity));
} else { } else {
Thread.sleep(5000); Thread.sleep(5000);
} }

@ -205,9 +205,9 @@ public class PictureDisposeTask implements Runnable{
picture.setSubmitDateTimestamp(pictureSourceResult.getSubmitTime()); picture.setSubmitDateTimestamp(pictureSourceResult.getSubmitTime());
// picture.setSubmitDateTimestamp(System.currentTimeMillis()); // picture.setSubmitDateTimestamp(System.currentTimeMillis());
picture.setPhotoDateTimestamp(System.currentTimeMillis()); picture.setPhotoDateTimestamp(System.currentTimeMillis());
//
// System.out.println("SubmitDateTimestamp1=================>>>>>>>"+pictureSourceResult.getSubmitTime()); System.out.println("SubmitDateTimestamp1=================>>>>>>>"+pictureSourceResult.getSubmitTime());
// System.out.println("SubmitDateTimestamp2=================>>>>>>>"+picture.getSubmitDateTimestamp()); System.out.println("SubmitDateTimestamp2=================>>>>>>>"+picture.getSubmitDateTimestamp());
//定位距离 //定位距离
picture.setField12(pictureSourceResult.getDistance()); picture.setField12(pictureSourceResult.getDistance());
@ -230,7 +230,6 @@ public class PictureDisposeTask implements Runnable{
//调用图片入库方法 //调用图片入库方法
int size = ocrPictureService.savePictures(ocrPictureList); int size = ocrPictureService.savePictures(ocrPictureList);
} }

@ -74,17 +74,12 @@ public class PictureImgToLocalTask implements Runnable {
picture.setImgHash(ocrPictureClassifyAndHash.get("hash")); picture.setImgHash(ocrPictureClassifyAndHash.get("hash"));
picture.setClassificationid(ocrPictureClassifyAndHash.get("classId")); picture.setClassificationid(ocrPictureClassifyAndHash.get("classId"));
} }
// 获取拍照时间 // 获取拍照时间
// Long timestamp = ocrPictureService.getPhotoDateTime(picture); // Long timestamp = ocrPictureService.getPhotoDateTime(picture);
// if(timestamp != 0L){ // if(timestamp != 0L){
// picture.setPhotoDateTimestamp(timestamp); // picture.setPhotoDateTimestamp(timestamp);
// } // }
ocrPictureService.updateById(picture); ocrPictureService.updateById(picture);
logger.info("图片本地化,回写数据完成!");
logger.info("开启获取图片信息操作!");
ocrPictureService.savePicturesInfo(picture);
logger.info("获取图片信息操作,结束!");
} }
} }
} }

@ -33,7 +33,7 @@ public class DownloadImgUtil {
out.write(buffer, 0, n); out.write(buffer, 0, n);
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("图片本地化,转储图片异常:",e.getMessage()); logger.error(e.getMessage());
return false; return false;
} }
return true; return true;

@ -236,7 +236,7 @@
<if test="pictureId != null and pictureId != ''"> <if test="pictureId != null and pictureId != ''">
and t1.picture_id = #{pictureId} and t1.picture_id = #{pictureId}
</if> </if>
ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp asc ORDER BY t1.max_similarity desc,t1.similarity_group_order asc, t2.submit_date_timestamp DESC
LIMIT 1 LIMIT 1
</select> </select>

@ -482,91 +482,6 @@
</trim> </trim>
</where> </where>
</sql> </sql>
<sql id="Base_Where_Union_New">
<where>
<trim prefixOverrides="and|or">
<if test="izupuser != null">
${izupuser.rulesOp} tcp.FROMUSERID ${izupuser.dataOp} ${izupuser.likestar}
<!-- #{fromuserid.value}-->
<if test="izupuser.value instanceof java.util.List">
<foreach item="item" collection="izupuser.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izupuser.value instanceof java.util.List">
#{izupuser.value}
</if>
${izupuser.likeend}
</if>
<if test="izApprovalStatus != null">
${izApprovalStatus.rulesOp} tcp.FROMUSERID ${izApprovalStatus.dataOp} ${izApprovalStatus.likestar}
<!-- #{fromuserid.value}-->
<if test="izApprovalStatus.value instanceof java.util.List">
<foreach item="item" collection="izApprovalStatus.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izApprovalStatus.value instanceof java.util.List">
#{izApprovalStatus.value}
</if>
${izApprovalStatus.likeend}
</if>
<if test="iztaskrrom != null">
${iztaskrrom.rulesOp} tcp.FROMSOURCEID ${iztaskrrom.dataOp} ${iztaskrrom.likestar}
<!-- #{fromuserid.value}-->
<if test="iztaskrrom.value instanceof java.util.List">
<foreach item="item" collection="iztaskrrom.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!iztaskrrom.value instanceof java.util.List">
#{iztaskrrom.value}
</if>
${iztaskrrom.likeend}
</if>
<if test="izproject != null">
${izproject.rulesOp} tcp.FROMPROJECTID ${izproject.dataOp} ${izproject.likestar}
<!-- #{fromuserid.value}-->
<if test="izproject.value instanceof java.util.List">
<foreach item="item" collection="izproject.value" open="" separator="," close="">
#{item}
</foreach>
</if>
<if test="!izproject.value instanceof java.util.List">
#{izproject.value}
</if>
${izproject.likeend}
</if>
<!-- <if test="izuptime != null">-->
<!-- ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}-->
<!-- </if>-->
<!-- <if test="izuptimeStart != null">-->
<!-- and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'-->
<!-- </if>-->
<!-- <if test="izuptimeEnd != null">-->
<!-- and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'-->
<!-- </if>-->
<if test="izuptimeStart != null">
<!-- 如果 izuptimeStart.dataOp 的值为 "<",则使用 "or" 连接 -->
<choose>
<when test="izuptimeStart.rulesOp == 'or' || izuptimeEnd.rulesOp=='or'">
<if test="izuptimeEnd != null">
and (tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}' or tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}')
</if>
</when>
<otherwise>
<!-- 其他情况下仍然拼接 "and" 连接 -->
and tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}'
<if test="izuptimeEnd != null">
and tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}'
</if>
</otherwise>
</choose>
</if>
</trim>
</where>
</sql>
<select id="listFinal" resultType="cn.jyjz.xiaoyao.ocr.dataobject.OcrTaskchildPicture" <select id="listFinal" resultType="cn.jyjz.xiaoyao.ocr.dataobject.OcrTaskchildPicture"
parameterType="java.util.Map"> parameterType="java.util.Map">
select tcp.*, pic.field11 reporttime, pic.similarity_score similarityscore from ( select tcp.*, pic.field11 reporttime, pic.similarity_score similarityscore from (
@ -641,45 +556,42 @@
parameterType="java.util.Map"> parameterType="java.util.Map">
select tcp.* from ( select tcp.* from (
select tc.*, select tc.*,
uf.COMMENTHIS, uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS, uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS, uf.STATSHIS,
uf.TASKINDEX, uf.TASKINDEX,
uf.TASKNO, uf.TASKNO,
op.field6 as fromCustomerLevel, op.field6 as fromCustomerLevel,
op.releaseArea as fromCity op.releaseArea as fromCity
from ocr_taskchild_picture tc from ocr_taskchild_picture tc
LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID LEFT JOIN OA_USERAPPROVE_T uf ON tc.ID = uf.ID
left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0 left join ocr_picture op on tc.PICTUREID = op.ID and op.suspiciousfile = 0
where tc.ASSIGNEE = #{searchassignee} where tc.ASSIGNEE = #{searchassignee}
and tc.TENANTID = #{searchtenantid} and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2 and tc.STATES = 2
UNION UNION
select tc.*, select tc.*,
uf.COMMENTHIS, uf.COMMENTHIS,
uf.TASKNAME as TASKNAMEHIS, uf.TASKNAME as TASKNAMEHIS,
uf.STATSHIS, uf.STATSHIS,
uf.TASKINDEX, uf.TASKINDEX,
uf.TASKNO, uf.TASKNO,
op.field6 as fromCustomerLevel, op.field6 as fromCustomerLevel,
op.releaseArea as fromCity op.releaseArea as fromCity
from ocr_taskchild_picture tc, from ocr_taskchild_picture tc,
OA_USERAPPROVE_T uf, OA_USERAPPROVE_T uf,
ocr_picture op ocr_picture op
where tc.ID = uf.FORMID where tc.ID = uf.FORMID
and tc.PICTUREID = op.ID and tc.PICTUREID = op.ID
and op.suspiciousfile = 0 and op.suspiciousfile = 0
and tc.TENANTID = #{searchtenantid} and tc.TENANTID = #{searchtenantid}
<if test="thisLoginUserid != null"> <if test="thisLoginUserid != null">
and uf.USERID = #{thisLoginUserid} and uf.USERID = #{thisLoginUserid}
</if> </if>
) as tcp ) as tcp
<include refid="Base_Where_union"> <include refid="Base_Where_union">
</include> </include>
<include refid="Base_Where_Union_New">
</include>
<if test="page != null"> <if test="page != null">
<if test="page.sortname != null"> <if test="page.sortname != null">
order by tcp.${page.sortname} ${page.sortorder} order by tcp.${page.sortname} ${page.sortorder}
@ -692,22 +604,20 @@
select tc.* select tc.*
from ocr_taskchild_picture tc from ocr_taskchild_picture tc
where tc.ASSIGNEE = #{searchassignee} where tc.ASSIGNEE = #{searchassignee}
and tc.TENANTID = #{searchtenantid} and tc.TENANTID = #{searchtenantid}
and tc.STATES = 2 and tc.STATES = 2
UNION UNION
select tc.* select tc.*
from ocr_taskchild_picture tc, from ocr_taskchild_picture tc,
OA_USERAPPROVE_T uf OA_USERAPPROVE_T uf
where tc.ID = uf.FORMID where tc.ID = uf.FORMID
and tc.TENANTID = #{searchtenantid} and tc.TENANTID = #{searchtenantid}
<if test="thisLoginUserid != null"> <if test="thisLoginUserid != null">
and uf.USERID = #{thisLoginUserid} and uf.USERID = #{thisLoginUserid}
</if> </if>
) as tcp ) as tcp
<include refid="Base_Where_union"> <include refid="Base_Where_union">
</include> </include>
<include refid="Base_Where_Union_New">
</include>
</select> </select>
@ -874,7 +784,7 @@
and t1.picture_id != #{pictureId} and t1.picture_id != #{pictureId}
</if> </if>
<if test="oderName == 'similarityScore'"> <if test="oderName == 'similarityScore'">
order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp asc order by t1.max_similarity ${oderType},t1.similarity_group_order asc,t2.submit_date_timestamp ${oderType}
</if> </if>
<if test="oderName == 'createdate'"> <if test="oderName == 'createdate'">
order by t2.submit_date_timestamp ${oderType} order by t2.submit_date_timestamp ${oderType}

Loading…
Cancel
Save