guoxiangbin
/
ocr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'fix:解决高级筛选sql查询问题（关于时间区间or的情况），解决无量云批量拉取数据问题' (#71) from fix/Pull_infinite_cloud_data_2945 into test

Browse Source 
Reviewed-on: #71
fix/pullWLYDataBug
cuihaojie 1 year ago
parent 1ce888607b 3d71a7fca9
commit 32f5a5995b
3 changed files with 102 additions and 60 deletions
Show Stats Download Patch File Download Diff File

2
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/flowable/service/impl/FlowTaskServiceImpl.java
Unescape View File

@ -2039,6 +2039,7 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
break; break;
} }
//不相等 //不相等
searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0)); searchQuery.addLess(usersearchchild.getSearchfield(), sectionInts.get(0));
searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1)); searchQuery.addGreater(usersearchchild.getSearchfield(), sectionInts.get(1));
break; break;
@ -2066,6 +2067,7 @@ public class FlowTaskServiceImpl extends FlowServiceFactory implements IFlowTask
break; break;
} }
//不相等 //不相等
searchQuery.addRulesOp(usersearchchild.getSearchfield()+"Start","or");
searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime()); searchQuery.addLess(usersearchchild.getSearchfield()+"Start", beginTime.getTime());
searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime()); searchQuery.addGreater(usersearchchild.getSearchfield()+"End", endTime.getTime());
break; break;

135
jyjz-system/jyjz-system-oa/src/main/java/cn/jyjz/xiaoyao/ocr/controller/ApiTestController.java
Unescape View File

@ -8,12 +8,14 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult; import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage; import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue; import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.time.Instant; import java.time.Instant;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@ -24,6 +26,7 @@ import java.util.List;
* @version 1.0 * @version 1.0
* @date 2024/3/14 19:47 * @date 2024/3/14 19:47
*/ */
@Slf4j
@RestController @RestController
@RequestMapping("/test/api") @RequestMapping("/test/api")
public class ApiTestController { public class ApiTestController {
@ -85,65 +88,85 @@ public class ApiTestController {
pictureSourceParameter.setTenantNo(1714548920583360512L); pictureSourceParameter.setTenantNo(1714548920583360512L);
pictureSourceParameter.setAccountNo(1714887674875514880L); pictureSourceParameter.setAccountNo(1714887674875514880L);
DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss"); List<String> notAddedToQueueTaskIds = new ArrayList<>();
DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
Thread thread = new Thread(new Runnable() {
try { @Override
int totalCount = 0; public void run() {
int totalLocalCount = 0; try {
int totalSize = 0; int totalCount = 0;
int totalLocalCount = 0;
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) { int totalSize = 0;
int currentPage = Integer.parseInt(pageNo); DateTime startDate = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
boolean hasNextPage = true; DateTime endDate = DateUtil.parse(endStr + " 23:59:59", "yyyy-MM-dd HH:mm:ss");
while (startDate.isBefore(endDate) || startDate.compareTo(endDate) == 0) {
while (hasNextPage) { int currentPage = Integer.parseInt(pageNo);
// 设置当天的开始和结束时间 boolean hasNextPage = true;
Instant startInstant = startDate.toInstant(); boolean isCountAdded = false;
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
pictureSourceParameter.setStartTime(Date.from(startInstant)); while (hasNextPage) {
pictureSourceParameter.setEndTime(Date.from(endInstant)); // 设置当天的开始和结束时间
pictureSourceParameter.setPageNo(currentPage); Instant startInstant = startDate.toInstant();
Instant endInstant = startDate.offsetNew(DateField.DAY_OF_YEAR, 1).toInstant();
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter); pictureSourceParameter.setStartTime(Date.from(startInstant));
List<PictureSourceResult> listResult = apiPage.getRecords(); pictureSourceParameter.setEndTime(Date.from(endInstant));
int count = apiPage.getTotal(); pictureSourceParameter.setPageNo(currentPage);
int localCount = listResult.size();
int size = 0; ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
List<PictureSourceResult> listResult = apiPage.getRecords();
for (PictureSourceResult pictureSourceResult : listResult) { int count = apiPage.getTotal();
if (pictureSourceResult.getLivePhoto() != null) { int localCount = listResult.size();
size++; int size = 0;
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult); for (PictureSourceResult pictureSourceResult : listResult) {
int retryCount = 0;
boolean addedToQueue = false;
if (pictureSourceResult.getLivePhoto() != null && !pictureSourceResult.getLivePhoto().isEmpty()) {
size++;
// 尝试将数据加入队列
while (!addedToQueue && retryCount < 5) { // 最多重试5次
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult);
if (!addedToQueue) {
Thread.sleep(60000);
retryCount++;
}
}
if (!addedToQueue) {
log.error("无法将数据添加到队列，开始时间：{}，任务ID：{}",startDate.toString("yyyy-MM-dd HH:mm:ss"),pictureSourceResult.getTaskId().toString());
notAddedToQueueTaskIds.add(pictureSourceResult.getTaskId().toString());
}
}
}
totalLocalCount += localCount;
totalSize += size;
if (!isCountAdded) {
totalCount += count;
isCountAdded = true; // 标记当天的统计信息已经被累加过了
}
// 输出当前日期范围内的统计信息
log.error("{}图片总数:{}条，本次获取：{}条，可处理数据:{}条",startDate.toString("yyyy-MM-dd"),count,localCount,size);
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
} }
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
} }
log.error("未加入队列的任务ID列表：{}",notAddedToQueueTaskIds);
totalCount += count; log.error("当前区间内图片总数:{}条，本次获取：{}条，可处理数据:{}条",totalCount,totalLocalCount,totalSize);
totalLocalCount += localCount; } catch (Exception e) {
totalSize += size; e.printStackTrace();
// 输出当前日期范围内的统计信息
System.out.println(startDate.toString("yyyy-MM-dd") + " 图片总数:" + count + "条，本次获取：" + localCount + "条，可处理数据:" + size + "条");
// 准备处理下一页的数据
currentPage++;
// 检查是否有下一页
if (localCount < Integer.parseInt(pageSize)) {
hasNextPage = false;
}
} }
// 准备处理下一天的数据
startDate = startDate.offsetNew(DateField.DAY_OF_YEAR, 1);
} }
});
return "当前区间内图片总数:" + totalCount + "条，本次获取：" + totalLocalCount + "条，可处理数据:" + totalSize + "条"; thread.start();
} catch (Exception e) { return "开启入库";
e.printStackTrace();
}
return "处理失败";
} }
} }

25
jyjz-system/jyjz-system-oa/src/main/resources/mapper/oa/OcrTaskchildPictureMapper.xml
Unescape View File

@ -540,12 +540,29 @@
</if> </if>
<!-- <if test="izuptime != null">--> <!-- <if test="izuptime != null">-->
<!-- ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}--> <!-- ${izuptime.rulesOp} tcp.FROMUPTIME ${izuptime.dataOp} ${izuptime.likestar}#{izuptime.value}${izuptime.likeend}-->
<!-- </if>-->
<!-- <if test="izuptimeStart != null">-->
<!-- and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}'-->
<!-- </if>-->
<!-- <if test="izuptimeEnd != null">-->
<!-- and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}'-->
<!-- </if>--> <!-- </if>-->
<if test="izuptimeStart != null"> <if test="izuptimeStart != null">
and tcp.FROMUPTIME &gt;= '${izuptimeStart.value}' <!-- 如果 izuptimeStart.dataOp 的值为 "<"，则使用 "or" 连接 -->
</if> <choose>
<if test="izuptimeEnd != null"> <when test="izuptimeStart.rulesOp == 'or' || izuptimeEnd.rulesOp=='or'">
and tcp.FROMUPTIME &lt;= '${izuptimeEnd.value}' <if test="izuptimeEnd != null">
and (tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}' or tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}')
</if>
</when>
<otherwise>
<!-- 其他情况下仍然拼接 "and" 连接 -->
and tcp.FROMUPTIME ${izuptimeStart.dataOp} '${izuptimeStart.value}'
<if test="izuptimeEnd != null">
and tcp.FROMUPTIME ${izuptimeEnd.dataOp} '${izuptimeEnd.value}'
</if>
</otherwise>
</choose>
</if> </if>
</trim> </trim>
</where> </where>

Write Preview
Loading…
Cancel
Save