guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'fix:解决高级筛选sql查询问题（关于时间区间or的情况），解决无量云批量拉取数据问题' (#71) from fix/Pull_infinite_cloud_data_2945 into test

Browse Source 
Reviewed-on: #71
fix/pullWLYDataBug
cuihaojie 1 year ago
parent 1ce888607b 3d71a7fca9
commit 32f5a5995b
3 changed files with 102 additions and 60 deletions
Show Stats Download Patch File Download Diff File

2
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/flowable/service/impl/FlowTaskServiceImpl.java
Unescape View File

@ -2039,6 +2039,7 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
break;
}
//不相等
searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
break;
@ -2066,6 +2067,7 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
break;
}
//不相等
searchQuery.addRulesOp(usersearchchild.getSearchfield()+"Start","or");
searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime());
break;

135
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -8,12 +8,14 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -24,6 +26,7 @@ import java.util.List;
* @version 1.0
* @date 2024/3/14 19:47
*/
@Slf4j
@RestController
@RequestMapping("/test/api")
public class ApiTestController {
@ -85,65 +88,85 @@ public class ApiTestController {
pictureSourceParameter.setTenantNo(1714548920583360512L);
pictureSourceParameter.setAccountNo(1714887674875514880L);
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
try {
int totalCount = 0;
int totalLocalCount = 0;
int totalSize = 0;
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true;
while (hasNextPage) {
// 设置当天的开始和结束时间
Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
if (pictureSourceResult.getLivePhoto() != null) {
size++;
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult);
List<String> notAddedToQueueTaskIds = new ArrayList<>();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
int totalCount = 0;
int totalLocalCount = 0;
int totalSize = 0;
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
int currentPage = Integer.parseInt(pageNo);
boolean hasNextPage = true;
boolean isCountAdded = false;
while (hasNextPage) {
// 设置当天的开始和结束时间
Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant));
pictureSourceParameter.setEndTime(Date.from(endInstant));
pictureSourceParameter.setPageNo(currentPage);
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
int retryCount = 0;
boolean addedToQueue = false;
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
size++;
// 尝试将数据加入队列
while (!addedToQueue && retryCount < 5) { // 最多重试5次
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult);
if (!addedToQueue) {
Thread.sleep(60000);
retryCount++;
}
}
if (!addedToQueue) {
log.error("无法将数据添加到队列，开始时间：{}，任务ID：{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
}
}
}
totalLocalCount += localCount;
totalSize += size;
if (!isCountAdded) {
totalCount += count;
isCountAdded = true; // 标记当天的统计信息已经被累加过了
}
// 输出当前日期范围内的统计信息
log.error("{}图片总数:{}条，本次获取：{}条，可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
}
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
}
totalCount += count;
totalLocalCount += localCount;
totalSize += size;
// 输出当前日期范围内的统计信息
System.out.println(startDate.toString("yyyy-MM-dd") + " 图片总数:" + count + "条，本次获取：" + localCount + "条，可处理数据:" + size + "条");
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
log.error("未加入队列的任务ID列表：{}",notAddedToQueueTaskIds);
log.error("当前区间内图片总数:{}条，本次获取：{}条，可处理数据:{}条",totalCount,totalLocalCount,totalSize);
} catch (Exception e) {
e.printStackTrace();
}
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
}
return "当前区间内图片总数:" + totalCount + "条，本次获取：" + totalLocalCount + "条，可处理数据:" + totalSize + "条";
} catch (Exception e) {
e.printStackTrace();
}
return "处理失败";
});
thread.start();
return "开启入库";
}
}

25
jyjz-system/jyjz-system-oa/src/main/resources/mapper/oa/OcrTaskchildPictureMapper.xml
Unescape View File

@ -540,12 +540,29 @@
</if>
<!-- <if test="izuptime != null">-->
<!-- ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}-->
<!-- </if>-->
<!-- <if test="izuptimeStart != null">-->
<!-- and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'-->
<!-- </if>-->
<!-- <if test="izuptimeEnd != null">-->
<!-- and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'-->
<!-- </if>-->
<if test="izuptimeStart != null">
and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'
</if>
<if test="izuptimeEnd != null">
and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'
<!-- 如果 izuptimeStart.dataOp 的值为 "<"，则使用 "or" 连接 -->
<choose>
<when test="izuptimeStart.rulesOp == 'or' || izuptimeEnd.rulesOp=='or'">
<if test="izuptimeEnd != null">
and (tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}' or tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}')
</if>
</when>
<otherwise>
<!-- 其他情况下仍然拼接 "and" 连接 -->
and tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}'
<if test="izuptimeEnd != null">
and tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}'
</if>
</otherwise>
</choose>
</if>
</trim>
</where>

Write Preview
Loading…
Cancel
Save