完善附件管理与生成任务跟踪

2026-07-02 18:26:50 +08:00
parent d3530ab0b7
commit 401e8cf57b
16 changed files with 873 additions and 203 deletions
@@ -40,3 +40,8 @@ async def init_db():
                await conn.execute(text("ALTER TABLE ai_models ADD COLUMN enable_reasoning BOOLEAN DEFAULT 0"))
            else:
                await conn.execute(text("ALTER TABLE ai_models ADD COLUMN enable_reasoning TINYINT(1) DEFAULT 0"))
+        document_columns = await conn.run_sync(
+            lambda sync_conn: [column["name"] for column in inspect(sync_conn).get_columns("documents")]
+        )
+        if "request_payload_json" not in document_columns:
+            await conn.execute(text("ALTER TABLE documents ADD COLUMN request_payload_json TEXT"))
@@ -11,5 +11,6 @@ class Document(Base):
    status = Column(String(20), default="pending", comment="pending/generating/completed/failed/cancelled")
    file_path = Column(String(500), default="", comment="生成的文件路径")
    error = Column(Text, default="", comment="错误信息")
+    request_payload_json = Column(Text, nullable=True, comment="提交任务时的文件与段落配置快照")
    created_at = Column(DateTime, server_default=func.now())
    updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
@@ -17,7 +17,7 @@ from models.generation_log import GenerationLog
 from models.paragraph import Paragraph
 from models.reference_file import ReferenceFile
 from models.template import Template
-from schemas.schemas import GenerateFullRequest, GenerateTestRequest, Response
+from schemas.schemas import GenerateFullRequest, GenerateTestRequest, ReferenceFileUpdate, Response
 from services.ai_service import call_ai, stream_ai_preview
 from services.file_summary import summarize_minio_files
 from services.generation_runtime import (
@@ -27,12 +27,18 @@ from services.generation_runtime import (
    run_generation,
    update_progress,
 )
-from services.minio_client import upload_bytes
+from services.minio_client import delete_object, split_bucket_path, upload_bytes, get_presigned_url

 router = APIRouter()


 def _serialize_document(document: Document) -> dict:
+    request_payload = {}
+    if document.request_payload_json:
+        try:
+            request_payload = json.loads(document.request_payload_json)
+        except Exception:
+            request_payload = {}
    return {
        "id": document.id,
        "template_id": document.template_id,
@@ -42,6 +48,7 @@ def _serialize_document(document: Document) -> dict:
        "status": document.status,
        "file_path": document.file_path,
        "error": document.error,
+        "request_payload": request_payload,
        "created_at": document.created_at,
        "updated_at": document.updated_at,
    }
@@ -54,9 +61,25 @@ def _serialize_reference_file(file: ReferenceFile) -> dict:
        "file_path": file.file_path,
        "file_size": file.file_size,
        "content_type": file.content_type,
-        "created_at": file.created_at,
+        "created_at": file.created_at.isoformat() if file.created_at else None,
    }

+
+async def _build_reference_name_mapping(db: AsyncSession, file_paths: list[str]) -> dict[str, str]:
+    if not file_paths:
+        return {}
+    result = await db.execute(
+        select(ReferenceFile.file_path, ReferenceFile.file_name).where(ReferenceFile.file_path.in_(file_paths))
+    )
+    return {file_path: file_name for file_path, file_name in result.all()}
+
+
+async def _build_reference_records_mapping(db: AsyncSession, file_paths: list[str]) -> dict[str, ReferenceFile]:
+    if not file_paths:
+        return {}
+    result = await db.execute(select(ReferenceFile).where(ReferenceFile.file_path.in_(file_paths)))
+    return {item.file_path: item for item in result.scalars().all()}
+
@router.post("/test")
 async def generate_test(body: GenerateTestRequest, db: AsyncSession = Depends(get_db)):
    paragraph = await db.get(Paragraph, body.paragraph_id)
@@ -71,7 +94,12 @@ async def generate_test(body: GenerateTestRequest, db: AsyncSession = Depends(ge
    elif paragraph.model_id:
        model = await db.get(AiModel, paragraph.model_id)

-    file_summaries = await asyncio.to_thread(summarize_minio_files, body.file_paths or []) if body.file_paths else []
+    file_name_mapping = await _build_reference_name_mapping(db, body.file_paths or [])
+    file_summaries = (
+        await asyncio.to_thread(summarize_minio_files, body.file_paths or [], file_name_mapping)
+        if body.file_paths
+        else []
+    )
    if model is None or model.status != "enabled":
        content = build_mock_content(paragraph)
        message = "当前未找到可用模型，返回本地模拟生成结果。"
@@ -109,7 +137,12 @@ async def generate_test_stream(body: GenerateTestRequest, db: AsyncSession = Dep
    if not model.supports_streaming:
        raise HTTPException(status_code=400, detail="当前模型未开启流式传输")

-    file_summaries = await asyncio.to_thread(summarize_minio_files, body.file_paths or []) if body.file_paths else []
+    file_name_mapping = await _build_reference_name_mapping(db, body.file_paths or [])
+    file_summaries = (
+        await asyncio.to_thread(summarize_minio_files, body.file_paths or [], file_name_mapping)
+        if body.file_paths
+        else []
+    )
    setattr(paragraph, "enable_reasoning", bool(model.enable_reasoning))

    async def event_stream():
@@ -236,6 +269,41 @@ async def list_reference_files(
    return Response(data={"items": items, "total": total, "page": page, "page_size": page_size})


+@router.get("/reference-files/{file_id}/download")
+async def download_reference_file(file_id: int, db: AsyncSession = Depends(get_db)):
+    record = await db.get(ReferenceFile, file_id)
+    if record is None:
+        raise HTTPException(status_code=404, detail="附件记录不存在")
+    bucket, object_name = split_bucket_path(record.file_path)
+    return Response(data={"url": get_presigned_url(bucket, object_name), "file_name": record.file_name})
+
+
+@router.put("/reference-files/{file_id}")
+async def update_reference_file(file_id: int, body: ReferenceFileUpdate, db: AsyncSession = Depends(get_db)):
+    record = await db.get(ReferenceFile, file_id)
+    if record is None:
+        raise HTTPException(status_code=404, detail="附件记录不存在")
+    record.file_name = body.file_name.strip()
+    await db.commit()
+    await db.refresh(record)
+    return Response(data=_serialize_reference_file(record))
+
+
+@router.delete("/reference-files/{file_id}")
+async def delete_reference_file(file_id: int, db: AsyncSession = Depends(get_db)):
+    record = await db.get(ReferenceFile, file_id)
+    if record is None:
+        raise HTTPException(status_code=404, detail="附件记录不存在")
+    try:
+        bucket, object_name = split_bucket_path(record.file_path)
+        await asyncio.to_thread(delete_object, bucket, object_name)
+    except Exception:
+        pass
+    await db.delete(record)
+    await db.commit()
+    return Response(data={"id": file_id})
+
+
@router.post("/full")
 async def generate_full(body: GenerateFullRequest, db: AsyncSession = Depends(get_db)):
    template = await db.get(Template, body.template_id)
@@ -251,6 +319,36 @@ async def generate_full(body: GenerateFullRequest, db: AsyncSession = Depends(ge
    if not paragraphs:
        raise HTTPException(status_code=400, detail="模板下暂无可生成段落")

+    normalized_file_map: dict[str, list[str]] = {}
+    for key, values in (body.file_map or {}).items():
+        if not isinstance(values, list):
+            continue
+        normalized_file_map[str(key)] = [item for item in values if item]
+    all_file_paths = [item for values in normalized_file_map.values() for item in values]
+    file_records = await _build_reference_records_mapping(db, all_file_paths)
+    paragraph_snapshot = []
+    for paragraph in paragraphs:
+        selected_paths = normalized_file_map.get(str(paragraph.id), [])
+        paragraph_snapshot.append(
+            {
+                "paragraph_id": paragraph.id,
+                "title": paragraph.title,
+                "sort_index": paragraph.sort_index,
+                "need_file": bool(paragraph.need_file),
+                "file_note": paragraph.file_note or "",
+                "selected_files": [
+                    _serialize_reference_file(file_records[file_path])
+                    for file_path in selected_paths
+                    if file_path in file_records
+                ],
+            }
+        )
+    request_payload = {
+        "template_name": template.name,
+        "file_map": normalized_file_map,
+        "paragraphs": paragraph_snapshot,
+    }
+
    document = Document(
        template_id=template.id,
        name=f"{template.name}-{datetime.now().strftime('%Y%m%d%H%M%S')}",
@@ -259,6 +357,7 @@ async def generate_full(body: GenerateFullRequest, db: AsyncSession = Depends(ge
        status="generating",
        file_path="",
        error="",
+        request_payload_json=json.dumps(request_payload, ensure_ascii=False),
    )
    db.add(document)
    await db.flush()
@@ -87,7 +87,11 @@ class GenerateTestRequest(BaseModel):

 class GenerateFullRequest(BaseModel):
    template_id: int
-    file_map: dict[str, str] = {}  # paragraph_id -> file_path
+    file_map: dict[str, list[str]] = {}  # paragraph_id -> file_paths
+
+
+class ReferenceFileUpdate(BaseModel):
+    file_name: str = Field(min_length=1, max_length=255)

 class DocumentOut(BaseModel):
    id: int
@@ -1,6 +1,8 @@
 import csv
 import io
 import json
+import subprocess
+import tempfile
 from pathlib import Path

 import pandas as pd
@@ -38,7 +40,10 @@ def _summarize_csv(content: bytes) -> str:

 def _summarize_excel(content: bytes, suffix: str) -> str:
    excel_buffer = io.BytesIO(content)
-    sheet_map = pd.read_excel(excel_buffer, sheet_name=None) if suffix == ".xlsx" else pd.read_excel(excel_buffer, sheet_name=None, engine="xlrd")
+    if suffix in {".xlsx", ".xlsm"}:
+        sheet_map = pd.read_excel(excel_buffer, sheet_name=None)
+    else:
+        sheet_map = pd.read_excel(excel_buffer, sheet_name=None, engine="xlrd")
    parts: list[str] = []
    for sheet_name, dataframe in list(sheet_map.items())[:5]:
        preview = dataframe.head(10).fillna("").astype(str)
@@ -47,6 +52,21 @@ def _summarize_excel(content: bytes, suffix: str) -> str:
    return "\n".join(parts)[:5000]


+def _summarize_doc(content: bytes) -> str:
+    with tempfile.NamedTemporaryFile(suffix=".doc") as temp_file:
+        temp_file.write(content)
+        temp_file.flush()
+        result = subprocess.run(
+            ["textutil", "-convert", "txt", "-stdout", temp_file.name],
+            capture_output=True,
+            check=False,
+        )
+    if result.returncode != 0:
+        stderr = result.stderr.decode("utf-8", errors="ignore").strip()
+        return f"旧版 Word 文件解析失败：{stderr or 'textutil 无法提取正文'}"
+    return _decode_text(result.stdout)[:4000]
+
+
 def _summarize_pdf(content: bytes) -> str:
    if PdfReader is None:
        return "当前环境未安装 PDF 文本解析依赖，无法提取 PDF 正文。"
@@ -68,18 +88,18 @@ def summarize_file_bytes(file_name: str, content: bytes) -> str:
    if suffix == ".docx":
        return _summarize_docx(content)
    if suffix == ".doc":
-        return "当前暂不支持直接解析 .doc 旧版 Word 文件正文，建议先另存为 .docx 后再上传。"
+        return _summarize_doc(content)
    if suffix == ".pdf":
        return _summarize_pdf(content)
    return f"暂不支持解析该文件内容：{file_name}"


-def summarize_minio_files(file_paths: list[str]) -> list[dict]:
+def summarize_minio_files(file_paths: list[str], file_name_mapping: dict[str, str] | None = None) -> list[dict]:
    summaries: list[dict] = []
    for file_path in file_paths:
        bucket, object_name = split_bucket_path(file_path)
        content = download_object_bytes(bucket, object_name)
-        file_name = Path(object_name).name
+        file_name = (file_name_mapping or {}).get(file_path) or Path(object_name).name
        summaries.append(
            {
                "file_name": file_name,
@@ -12,6 +12,7 @@ from models.generation_log import GenerationLog
 from models.paragraph import Paragraph
 from models.template import Template
 from services.ai_service import call_ai
+from services.file_summary import summarize_minio_files

 generation_progress: dict[int, dict] = {}
 generation_cancel_flags: dict[int, bool] = {}
@@ -90,6 +91,13 @@ async def run_generation(document_id: int, template_id: int):
        )
        paragraphs = result.scalars().all()
        total = len(paragraphs)
+        request_payload = {}
+        if document.request_payload_json:
+            try:
+                request_payload = json.loads(document.request_payload_json)
+            except Exception:
+                request_payload = {}
+        file_map = request_payload.get("file_map", {}) if isinstance(request_payload, dict) else {}
        update_progress(document_id, status="generating", total=total, done=0, percent=0, message="开始生成...")

        done_count = 0
@@ -114,11 +122,25 @@ async def run_generation(document_id: int, template_id: int):
                    model = await get_effective_model(paragraph)
                    model_id = model.id if model is not None else paragraph.model_id
                    try:
+                        selected_file_paths = file_map.get(str(paragraph.id), [])
+                        file_summaries = []
+                        if selected_file_paths:
+                            file_name_mapping = {}
+                            for paragraph_item in request_payload.get("paragraphs", []):
+                                for selected_file in paragraph_item.get("selected_files", []):
+                                    file_path = selected_file.get("file_path")
+                                    if file_path in selected_file_paths:
+                                        file_name_mapping[file_path] = selected_file.get("file_name")
+                            file_summaries = await asyncio.to_thread(
+                                summarize_minio_files,
+                                selected_file_paths,
+                                file_name_mapping,
+                            )
                        if model is None:
                            content = build_mock_content(paragraph)
                        else:
                            setattr(paragraph, "enable_reasoning", bool(model.enable_reasoning))
-                            result_data = await call_ai(paragraph, model)
+                            result_data = await call_ai(paragraph, model, file_summaries)
                            content = result_data.content
                        status = "success"
                        error_message = ""
@@ -68,3 +68,7 @@ def upload_bytes(
        len(content),
        content_type=content_type,
    )
+
+
+def delete_object(bucket: str, object_name: str):
+    minio_client.remove_object(bucket, object_name)