feat: 查标第 1 步——上传投标分册 → 检查 → 逐条处理 → 复查沿用
后端 - check 模块：检查预览、发起、进度、取消、记录；六个阶段（准备投标文件、响应匹配、规则、证据校验、 状态沿用、汇总）；问题列表、处理、撤销、批量处理（只限提示级）。 - 每条清单项的检查结论 check_item_result（C9，迁移 0005）：「已响应」也带投标原文出处； 没有校验通过的证据不能判已响应（C3）。级别从清单项算，不让模型定；说明不写整改建议。 - 响应匹配先只有第三层：内存 BM25 检索候选句 + check.match v1 提示词，检索不到不调模型， 漏答记 missing_answer，模型临时错误只影响那一批。 - 计费：免费版不记流水；专业版 24 小时内复查免扣（记一行 0）；expected_charge 不一致时拒绝不扣点； 失败、取消退回；投标文件读不出文字按输入问题失败、不扣点。 - 投标分册：类型（volume_kind）和版本；电子投标平台加密格式在上传时拦下。 - 解析流水线前几个阶段拆到 document/pipeline/normalize.py，解析和检查共用。 - ops 模块：反馈接口；致命问题确认无误的说明自动进误报反馈（U4）。 - SK1：Valkey 客户端短超时、SSE 连接池上限。SK2：CI 加 Gotenberg、直接跑 init-roles.sh。 前端 - 上传投标文件并检查页、查标页（进度、覆盖范围、按级别筛选、逐条处理和行内说明、撤销、原文抽屉、 已处理、本次已解决）；项目头抽成公共组件，加「查标」标签页；工作台「下一步」覆盖查标阶段。 契约、文档：openapi.yaml 加 item-results、volume_kind、coverage 字段；data-model、api-design §10、 架构 §8 / §13（SK14）、AGENTS.md 同步。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
98
server/src/bid_doctor/document/pipeline/normalize.py
Normal file
98
server/src/bid_doctor/document/pipeline/normalize.py
Normal file
@@ -0,0 +1,98 @@
|
||||
"""解析流水线的前几个阶段：格式核对 → 转换 → 文字层 → OCR 补全（架构 §3.5）。
|
||||
|
||||
解析任务（招标文件）和检查批次（投标文件，m3-plan §2 prepare_bid）共用。阶段的进度、取消由调用方的 on_stage 负责。
|
||||
TODO(S5)：OCR 还没接，没有文字层的页 text_source = "empty"。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core.crypto import open_object, seal_object, sha256
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.pipeline.text_layer import Page
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
|
||||
# text_layer v2：页眉页脚带单独成句，句子编号和 v1 不同（同一份规范化 PDF 在 v1、v2 下的编号不能混用）
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v4", "sentence_split": "s3-v4"}
|
||||
OCR_REQUIRED_RATIO = 0.5 # 超过一半的页没有文字层：招标文件按「需要 OCR」失败
|
||||
|
||||
OnStage = Callable[[str], Awaitable[None]]
|
||||
|
||||
|
||||
async def pages(
|
||||
ctx: TenantContext, document_id: UUID, on_stage: OnStage, *, reuse: bool, require_text: bool
|
||||
) -> list[Page]:
|
||||
"""返回文档的文字层。reuse=True 时，已有同一版本的文字层就直接读出来（投标分册每个版本只解析一次）。
|
||||
|
||||
require_text：没有文字层的页太多时报 ocr_required（招标文件）。
|
||||
投标文件不报，扫描页由检查按「请人工核对」处理（C10）。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == document_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
if reuse and doc.text_layer_key and (doc.pipeline_profile or {}).get("text_layer") == PROFILE["text_layer"]:
|
||||
raw = open_object(dk, storage().get(doc.text_layer_key), doc.text_layer_key)
|
||||
return [Page.from_json(p) for p in json.loads(raw)]
|
||||
|
||||
await on_stage("detect") # 上传完成时已识别过格式；这里只核对原件没被替换
|
||||
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
|
||||
if sha256(original) != doc.original_sha256:
|
||||
raise AppError("file_corrupted")
|
||||
|
||||
await on_stage("convert")
|
||||
if doc.normalized_key is None:
|
||||
pdf = await convert.to_pdf(original, doc.detected_format)
|
||||
key = f"{doc_prefix(doc)}/normalized.pdf"
|
||||
storage().put(key, seal_object(dk, pdf, key))
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
|
||||
d.pipeline_profile = PROFILE
|
||||
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
|
||||
else:
|
||||
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
|
||||
|
||||
await on_stage("text_layer")
|
||||
result = text_layer.extract(str(doc.id), pdf)
|
||||
text_key = f"{doc_prefix(doc)}/text.json"
|
||||
storage().put(
|
||||
text_key, seal_object(dk, json.dumps([p.to_json() for p in result], ensure_ascii=False).encode(), text_key)
|
||||
)
|
||||
async with tenant_tx(ctx) as s:
|
||||
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等：先清残留
|
||||
for p in result:
|
||||
s.add(
|
||||
DocumentPage(
|
||||
tenant_id=ctx.tenant_id,
|
||||
document_id=doc.id,
|
||||
page_no=p.page_no,
|
||||
width_pt=p.width,
|
||||
height_pt=p.height,
|
||||
rotation=p.rotation,
|
||||
text_source=p.text_source,
|
||||
char_count=p.char_count,
|
||||
sentence_count=len(p.sentences),
|
||||
)
|
||||
)
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.text_layer_key, d.page_count, d.status = text_key, len(result), "ready"
|
||||
d.pipeline_profile = PROFILE
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(result) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
|
||||
await on_stage("ocr")
|
||||
empty = sum(p.text_source == "empty" for p in result)
|
||||
if require_text and result and empty / len(result) > OCR_REQUIRED_RATIO:
|
||||
raise AppError("ocr_required")
|
||||
return result
|
||||
@@ -8,28 +8,24 @@ TODO(结构识别)：章节、表格、分包识别还没做，现在只抽实
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import select
|
||||
|
||||
from bid_doctor.ai.providers.base import TransientError
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.checklist import extract
|
||||
from bid_doctor.checklist import service as checklist
|
||||
from bid_doctor.core import events, keys
|
||||
from bid_doctor.core.crypto import open_object, seal_object, sha256
|
||||
from bid_doctor.core import events
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
from bid_doctor.document.models import ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import normalize, text_layer
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import extracted as project_extracted
|
||||
from bid_doctor.project import service as projects
|
||||
@@ -45,8 +41,6 @@ PROGRESS = {
|
||||
"verify": 92,
|
||||
"merge": 95,
|
||||
}
|
||||
# text_layer v2：页眉页脚带单独成句，句子编号和 v1 不同（同一份规范化 PDF 在 v1、v2 下的编号不能混用）
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v4", "sentence_split": "s3-v4"}
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
@@ -113,58 +107,11 @@ async def _stages(ctx: TenantContext, job_id: UUID) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == job.document_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
|
||||
await _stage(ctx, job_id, "detect") # 上传完成时已识别过格式；这里只核对原件没被替换
|
||||
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
|
||||
if sha256(original) != doc.original_sha256:
|
||||
raise AppError("file_corrupted")
|
||||
async def on_stage(stage: str) -> None:
|
||||
await _stage(ctx, job_id, stage)
|
||||
|
||||
await _stage(ctx, job_id, "convert")
|
||||
if doc.normalized_key is None:
|
||||
pdf = await convert.to_pdf(original, doc.detected_format)
|
||||
key = f"{doc_prefix(doc)}/normalized.pdf"
|
||||
storage().put(key, seal_object(dk, pdf, key))
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
|
||||
d.pipeline_profile = PROFILE
|
||||
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
|
||||
else:
|
||||
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
|
||||
|
||||
await _stage(ctx, job_id, "text_layer")
|
||||
pages = text_layer.extract(str(doc.id), pdf)
|
||||
text_key = f"{doc_prefix(doc)}/text.json"
|
||||
storage().put(
|
||||
text_key, seal_object(dk, json.dumps([p.to_json() for p in pages], ensure_ascii=False).encode(), text_key)
|
||||
)
|
||||
async with tenant_tx(ctx) as s:
|
||||
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等：先清残留
|
||||
for p in pages:
|
||||
s.add(
|
||||
DocumentPage(
|
||||
tenant_id=ctx.tenant_id,
|
||||
document_id=doc.id,
|
||||
page_no=p.page_no,
|
||||
width_pt=p.width,
|
||||
height_pt=p.height,
|
||||
rotation=p.rotation,
|
||||
text_source=p.text_source,
|
||||
char_count=p.char_count,
|
||||
sentence_count=len(p.sentences),
|
||||
)
|
||||
)
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.text_layer_key, d.page_count, d.status = text_key, len(pages), "ready"
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(pages) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
|
||||
await _stage(ctx, job_id, "ocr")
|
||||
empty = sum(p.text_source == "empty" for p in pages)
|
||||
if pages and empty / len(pages) > 0.5:
|
||||
raise AppError("ocr_required")
|
||||
pages = await normalize.pages(ctx, doc.id, on_stage, reuse=False, require_text=True)
|
||||
|
||||
await _stage(ctx, job_id, "structure")
|
||||
body = text_layer.body_sentences(pages)
|
||||
|
||||
Reference in New Issue
Block a user