feat: 添加拉取图像日志

feature/pullPictureLog
Vincent 1 year ago
parent ae69b0ff62
commit 68b112185d

@ -0,0 +1,138 @@
<#include "/common/taglibs.html" />
<!DOCTYPE html>
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>数据拉取日志</title>
<#include "/head/headlist.html" />
</head>
<body class="gray-bg">
<div class="wrapper wrapper-content animated fadeInRight">
<div class="ibox ">
<div class="ibox-content">
<div class="ibox-content">
<form id="searcheuser" class="form-inline">
<!-- <div class="form-group">
<input name='search_callTime' id='search_callTime' class='form-control' placeholder='字段名称'/>
</div>-->
<div class="form-group">
<select class="form-control" name='callStatus' id='callStatus'>
<option value="100">成功</option>
<option value="999">失败</option>
<option value="500">异常</option>
</select>
</div>
<div class="form-group">
<a class="btn btn-primary btn-rounded btn-sm" onclick="refreshTable()"><i
class="fa fa-search"></i>&nbsp;搜索</a>
<a class="btn btn-warning btn-rounded btn-sm" onclick="$('#searcheuser')[0].reset()"><i
class="fa fa-refresh"></i>&nbsp;重置</a>
</div>
</form>
</div>
</div>
</div>
<div class="ibox-content">
<div class="row row-lg">
<div class="col-sm-12">
<div class="container-div">
<div class="btn-group-sm" id="_mybutton" role="group">
</div>
<br/>
</div>
</div>
<div class="col-sm-12 select-table table-striped">
<table id="Post_list"></table>
</div>
</div>
</div>
</div>
<#include "/head/footlist.html" />
</body>
</html>
<script type="text/javascript">
var _tableListPost;
var jsptoolbar = '';
var jspOperation = '';
jQuery(document).ready(function ($) {
parent.axgetXiaoyao("${ctx}/backstage/admin/user/toolsbarOperation?" + "resourceid=${resourceid}", null, true,
function (data) {
if (data.code == "OK") {
jsptoolbar = data.data.Jsptoolbar;
jspOperation = data.data.JspOperation;
$("#_mybutton").empty();
_tableListPost = $('#Post_list').bootstrapTable({
url: '${ctx}/ocr/prevailCloudLog/page',
pagination: true, //是否显示分页(*
pageNumber: 1, //初始化加载第一页,默认第一页
pageSize: 10, //每页的记录行数(*
sidePagination: "server",
pageList: [10, 25, 50],
ajaxOptions: {
headers: {"logintoken": parent.getLoginToken()}
},
queryParams: function queryParams(params) {
return {
sortorder: params.order,
sortname: params.sort,
pageSize: params.limit,
start: (params.offset / params.limit) + 1,
status: $("#callStatus").val(),
callTimeStart: params.callTimeStart,
callTimeEnd: params.callTimeEnd,
};
},
responseHandler: function (res) {
return {"rows": res.data.records, "total": res.data.total};
},
columns: [
{checkbox: true},
{width: 100, valign: "middle", field: 'createTime', title: '创建时间', sortable: true},
{width: 150, valign: "middle", field: 'formId', title: '表单id', sortable: true},
{
width: 80, valign: "middle", field: 'status', title: '调用状态', sortable: true,
formatter: function (value, oData, index) {
var listOperation = "";
if (oData.status == 100) {
listOperation = "成功";
} else if (oData.status == 999) {
listOperation = "失败";
} else if (oData.status == 500) {
listOperation = "异常";
} else {
listOperation = "成功";
}
return listOperation;
}
},
{
width: 100, valign: "middle", field: 'type', title: '类型', sortable: true,
formatter: function (value, oData, index) {
var listOperation = "";
if (oData.type == 0) {
listOperation = "图片";
} else if (oData.type == 1) {
listOperation = "工作流";
} else if (oData.type == 2) {
listOperation = "账号";
} else {
listOperation = "未知";
}
return listOperation;
}
},
{width: 80, valign: "middle", field: 'requestParam', title: '请求参数', sortable: true},
{width: 80, valign: "middle", field: 'responseParam', title: '响应参数', sortable: true}
]
});
}
}
);
});
function refreshTable(sourceid){
_tableListPost.bootstrapTable('refreshOptions',{pageNumber:1});
}
</script>

@ -204,4 +204,8 @@ public class PictureSourceResult {
*/
Long submitTime;
/**
* log
*/
PictureSourceParameter requestParam;
}

@ -135,7 +135,6 @@ public class PrevailCloudApiImpl implements PrevailCloudApi {
return resultApiPage;
} else {
logger.debug("请求图片接口数据失败,参数:{},返回值:{}", json, responseJsonStr);
}
prevailCloudLogService.save(prevailCloudLog);
return null;

@ -12,6 +12,7 @@ import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiConfig;
import cn.jyjz.xiaoyao.ocr.api.utils.ApiPage;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
@ -56,34 +57,38 @@ public class ApiTestController {
@Resource
private OcrPictureService ocrPictureService;
@Resource
private OcrPrevailCloudLogService ocrPrevailCloudLogService;
/**
*
* pull wyl task data
*
* @param request
* @return
*/
@GetMapping("pullPictureByOneAccountNo")
public String pullPictureByOneAccountNo(HttpServletRequest request){
public String pullPictureByOneAccountNo(HttpServletRequest request) {
String beginStr = request.getParameter("beginStr");
String endStr = request.getParameter("endStr");
String pageSize = request.getParameter("pageSize");
String pageNo = request.getParameter("pageNo");
String tenantNoStr = request.getParameter("tenantNo");
if(StringUtils.isBlank(tenantNoStr)){
if (StringUtils.isBlank(tenantNoStr)) {
return "租户不能为空";
}
String accountNoStr = request.getParameter("accountNo");
if(StringUtils.isBlank(accountNoStr)){
if (StringUtils.isBlank(accountNoStr)) {
return "业务主体不能为空";
}
Long tenantNo = Long.parseLong(request.getParameter("tenantNo"));
Long accountNo = Long.parseLong(request.getParameter("accountNo"));
PictureSourceParameter pictureSourceParameter = new PictureSourceParameter();
DateTime dateTime = DateUtil.parse(beginStr+" 00:00:00","yyyy-MM-dd HH:mm:ss");
DateTime dateTime = DateUtil.parse(beginStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime = DateUtil.parse("2024-03-31 07:00:000","yyyy-MM-dd HH:mm:ss");
Instant binstant = dateTime.toInstant();
DateTime dateTime1 = DateUtil.parse(endStr+" 00:00:00","yyyy-MM-dd HH:mm:ss");
DateTime dateTime1 = DateUtil.parse(endStr + " 00:00:00", "yyyy-MM-dd HH:mm:ss");
// DateTime dateTime1 = DateUtil.parse("2024-03-31 09:00:00","yyyy-MM-dd HH:mm:ss");
Instant einstant = dateTime1.toInstant();
pictureSourceParameter.setStartTime(Date.from(binstant));
@ -93,30 +98,31 @@ public class ApiTestController {
pictureSourceParameter.setTenantNo(tenantNo);
pictureSourceParameter.setAccountNo(accountNo);
try {
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
if(apiPage==null){
ApiPage<PictureSourceResult> apiPage = prevailCloudApi.pullPictureSource(pictureSourceParameter);
if (apiPage == null) {
return "获取数据为空!";
}
List<PictureSourceResult> listResult= apiPage.getRecords();
List<PictureSourceResult> listResult = apiPage.getRecords();
int count = apiPage.getTotal();
int localCount = listResult.size();
int size = 0;
for (PictureSourceResult pictureSourceResult : listResult) {
//1.如果不存在图片默认一个默认图片,方便走测试。
if(pictureSourceResult.getLivePhoto()==null){
List<Map<String,Object>> livePhoto = new ArrayList<>();
Map<String,Object> map = new HashMap<>();
map.put("imgUrl","http://47.93.59.251/api/image/default.png");
map.put("imgName","default.png");
if (pictureSourceResult.getLivePhoto() == null) {
List<Map<String, Object>> livePhoto = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
map.put("imgUrl", "http://47.93.59.251/api/image/default.png");
map.put("imgName", "default.png");
livePhoto.add(map);
pictureSourceResult.setLivePhoto(livePhoto);
pictureSourceResult.setRequestParam(pictureSourceParameter);
// continue;
}
size++;
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService);
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService, ocrPrevailCloudLogService);
}
return "当前区间内图片总数:"+count+"条,本次获取:"+localCount+"条,可处理数据:"+size+"条";
return "当前区间内图片总数:" + count + "条,本次获取:" + localCount + "条,可处理数据:" + size + "条";
} catch (Exception e) {
e.printStackTrace();
}
@ -191,9 +197,10 @@ public class ApiTestController {
map.put("imgName", "default.png");
livePhoto.add(map);
pictureSourceResult.setLivePhoto(livePhoto);
pictureSourceResult.setRequestParam(pictureSourceParameter);
}
//将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService);
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService, ocrPrevailCloudLogService);
}
return "处理成功";
@ -293,7 +300,8 @@ public class ApiTestController {
size++;
// 尝试将数据加入队列
while (!addedToQueue && retryCount < 5) { // 最多重试5次
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService);
pictureSourceResult.setRequestParam(pictureSourceParameter);
addedToQueue = TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService, ocrPrevailCloudLogService);
if (!addedToQueue) {
Thread.sleep(60000);
retryCount++;

@ -0,0 +1,47 @@
package cn.jyjz.xiaoyao.ocr.controller;
import cn.jyjz.xiaoyao.common.base.param.ParamterPage;
import cn.jyjz.xiaoyao.common.base.vo.ResultVo;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPrevailCloudLog;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
@RestController
@RequestMapping("/ocr/prevailCloudLog")
public class OcrPrevailCloudLogController {
@Resource
private OcrPrevailCloudLogService ocrPrevailCloudLogService;
/**
*
*
* @return
*/
@ApiOperation(value = "根据评审类型查询字段列表", notes = "根据评审类型查询字段列表")
@GetMapping(value = "/page")
public ResultVo<IPage<OcrPrevailCloudLog>> list(@RequestParam(name = "start", defaultValue = "1") Integer start,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
@RequestParam(name = "status", required = false) String status,
HttpServletRequest request) {
QueryWrapper<OcrPrevailCloudLog> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("status", status);
ResultVo<IPage<OcrPrevailCloudLog>> resultVo = new ResultVo<>();
ParamterPage paramterPage = new ParamterPage();
paramterPage.setPagesize(pageSize);
paramterPage.setStart(start);
IPage<OcrPrevailCloudLog> ocrPictureLogIPage = ocrPrevailCloudLogService.selectSearchListPage(paramterPage, queryWrapper);
resultVo.setData(ocrPictureLogIPage);
resultVo.addOK("查询成功");
return resultVo;
}
}

@ -0,0 +1,20 @@
package cn.jyjz.xiaoyao.ocr.controller;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@Controller
@RequestMapping("/backstage/oa/prevailcloudlog")
public class OcrPrevailCloudLogHtmlController {
@GetMapping("/list")
public String toListOcrConfiguration(HttpServletResponse response, HttpServletRequest request) {
String resourceid = request.getParameter("resourceid");
request.setAttribute("resourceid", resourceid);
return "/backstage/oa/ocr/prevailcloudlog/list";
}
}

@ -5,7 +5,10 @@ import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.util.Date;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -23,6 +26,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
@TableName(value = "ocr_prevail_cloud_log")
@Builder
public class OcrPrevailCloudLog {
/**
* ID
@ -70,6 +74,7 @@ public class OcrPrevailCloudLog {
*
*/
@TableField(value = "create_time")
@JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
@ -82,6 +87,7 @@ public class OcrPrevailCloudLog {
*
*/
@TableField(value = "update_time")
@JsonFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/**

@ -69,11 +69,10 @@ public interface IOcrCheckDuplicateService extends IService<OcrCheckDuplicate> {
/**
*
*
* @param checkDuplicateNo
* @param tenantId
* @param pageNo
* @param pageSize
* @param pictureId
* @param orderType
* @param orderValue
* @return
*/

@ -1,14 +1,13 @@
package cn.jyjz.xiaoyao.ocr.service;
import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseService;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPrevailCloudLog;
import com.baomidou.mybatisplus.extension.service.IService;
/**
/**
* ClassName $className$.java
* Description:
* Description:
* Author scl.
* Date 2024/4/25 12:52
*/
public interface OcrPrevailCloudLogService extends IService<OcrPrevailCloudLog>{
public interface OcrPrevailCloudLogService extends BaseService<OcrPrevailCloudLog> {
}

@ -1,19 +1,11 @@
package cn.jyjz.xiaoyao.ocr.service.impl;
import org.springframework.stereotype.Service;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPrevailCloudLog;
import cn.jyjz.xiaoyao.common.mybatisplus.base.BaseServiceImpl;
import cn.jyjz.xiaoyao.ocr.dataDao.OcrPrevailCloudLogMapper;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPrevailCloudLog;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
/**
* ClassName $className$.java
* Description:
* Author scl.
* Date 2024/4/25 12:52
*/
@Service
public class OcrPrevailCloudLogServiceImpl extends ServiceImpl<OcrPrevailCloudLogMapper, OcrPrevailCloudLog> implements OcrPrevailCloudLogService{
import org.springframework.stereotype.Service;
@Service
public class OcrPrevailCloudLogServiceImpl extends BaseServiceImpl<OcrPrevailCloudLogMapper, OcrPrevailCloudLog> implements OcrPrevailCloudLogService {
}

@ -1,8 +1,11 @@
package cn.jyjz.xiaoyao.ocr.thread;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.common.PrevailCloudConstant;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPrevailCloudLog;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import cn.jyjz.xiaoyao.ocr.thread.entity.PictureImgToLocalEntity;
import com.google.common.collect.Sets;
@ -37,7 +40,7 @@ public class TaskQueue {
* @param pictureSourceResult //三方拉取 获取到的图片对象
* @return
*/
public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult, OcrPictureService ocrPictureService) {
public static boolean pictureDisposePushData(PictureSourceResult pictureSourceResult, OcrPictureService ocrPictureService, OcrPrevailCloudLogService ocrPrevailCloudLogService) {
try {
Long taskId = pictureSourceResult.getTaskId();
OcrPicture ocrPicture = ocrPictureService.selectByRemark(String.valueOf(taskId));
@ -45,9 +48,25 @@ public class TaskQueue {
return true;
}
OcrPrevailCloudLog ocrPrevailCloudLog = OcrPrevailCloudLog.builder()
.formId(pictureSourceResult.getTaskId())
.requestParam(pictureSourceResult.getRequestParam().toString())
.responseParam(pictureSourceResult.toString())
.type(PrevailCloudConstant.PULL_PICTURE_SOURCE)
.status(100)
.build();
ocrPrevailCloudLogService.save(ocrPrevailCloudLog);
resultSet.add(String.valueOf(taskId));
pictureDisposeQueue.put(pictureSourceResult);
} catch (InterruptedException e) {
OcrPrevailCloudLog ocrPrevailCloudLog = OcrPrevailCloudLog.builder()
.formId(pictureSourceResult.getTaskId())
.requestParam(pictureSourceResult.getRequestParam().toString())
.responseParam(pictureSourceResult.toString())
.type(PrevailCloudConstant.PULL_PICTURE_SOURCE)
.status(500)
.build();
ocrPrevailCloudLogService.save(ocrPrevailCloudLog);
return false;
}
return true;

@ -1,20 +1,18 @@
package cn.jyjz.xiaoyao.ocr.thread.tasks;
import cn.hutool.core.date.DateUtil;
import cn.jyjz.xiaoyao.common.base.util.SpringUtils;
import cn.jyjz.xiaoyao.common.base.util.StringUtils;
import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.dataobject.OcrPicture;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import com.alibaba.fastjson.JSONObject;
//import javafx.scene.input.DataFormat;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
*
@ -32,9 +30,6 @@ public class PictureDisposeTask implements Runnable{
this.pictureSourceResult = pictureSourceResult;
}
@Autowired
private OcrPictureService ocrPictureService;
@Override
public void run() {
//1.对图片对象进行入库操作

@ -8,6 +8,7 @@ import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Lists;
@ -47,6 +48,9 @@ public class PictureSourceTimerJob {
@Resource
private OcrPictureService ocrPictureService;
@Resource
private OcrPrevailCloudLogService ocrPrevailCloudLogService;
private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
/**
@ -94,8 +98,9 @@ public class PictureSourceTimerJob {
for (PictureSourceResult pictureSourceResult : totalList) {
if (pictureSourceResult.getLivePhoto() != null) {
size++;
pictureSourceResult.setRequestParam(pictureSourceParameter);
// 将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService);
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService, ocrPrevailCloudLogService);
}
}

@ -8,6 +8,7 @@ import cn.jyjz.xiaoyao.ocr.api.PrevailCloudApi;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceParameter;
import cn.jyjz.xiaoyao.ocr.api.entity.PictureSourceResult;
import cn.jyjz.xiaoyao.ocr.service.OcrPictureService;
import cn.jyjz.xiaoyao.ocr.service.OcrPrevailCloudLogService;
import cn.jyjz.xiaoyao.ocr.thread.TaskQueue;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.common.collect.Maps;
@ -36,6 +37,9 @@ public class PullPictureDataJob {
@Resource
private OcrPictureService ocrPictureService;
@Resource
private OcrPrevailCloudLogService ocrPrevailCloudLogService;
// @Scheduled(cron = "0 5 0 * * ?") // 每天晚上0点五分执行一次
public void pullPicture() {
//配置查询参数信息
@ -100,7 +104,8 @@ public class PullPictureDataJob {
if (pictureSourceResult.getLivePhoto() != null) {
size++;
// 将可以处理数据放入处理队列中
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService);
pictureSourceResult.setRequestParam(pictureSourceParameter);
TaskQueue.pictureDisposePushData(pictureSourceResult, ocrPictureService, ocrPrevailCloudLogService);
}
}
System.out.println("当前区间内图片总数:" + count + "条,可处理数据:" + size + "条");

Loading…
Cancel
Save