feat: 查标第 1 步——上传投标分册 → 检查 → 逐条处理 → 复查沿用

后端
- check 模块:检查预览、发起、进度、取消、记录;六个阶段(准备投标文件、响应匹配、规则、证据校验、
  状态沿用、汇总);问题列表、处理、撤销、批量处理(只限提示级)。
- 每条清单项的检查结论 check_item_result(C9,迁移 0005):「已响应」也带投标原文出处;
  没有校验通过的证据不能判已响应(C3)。级别从清单项算,不让模型定;说明不写整改建议。
- 响应匹配先只有第三层:内存 BM25 检索候选句 + check.match v1 提示词,检索不到不调模型,
  漏答记 missing_answer,模型临时错误只影响那一批。
- 计费:免费版不记流水;专业版 24 小时内复查免扣(记一行 0);expected_charge 不一致时拒绝不扣点;
  失败、取消退回;投标文件读不出文字按输入问题失败、不扣点。
- 投标分册:类型(volume_kind)和版本;电子投标平台加密格式在上传时拦下。
- 解析流水线前几个阶段拆到 document/pipeline/normalize.py,解析和检查共用。
- ops 模块:反馈接口;致命问题确认无误的说明自动进误报反馈(U4)。
- SK1:Valkey 客户端短超时、SSE 连接池上限。SK2:CI 加 Gotenberg、直接跑 init-roles.sh。

前端
- 上传投标文件并检查页、查标页(进度、覆盖范围、按级别筛选、逐条处理和行内说明、撤销、原文抽屉、
  已处理、本次已解决);项目头抽成公共组件,加「查标」标签页;工作台「下一步」覆盖查标阶段。

契约、文档:openapi.yaml 加 item-results、volume_kind、coverage 字段;data-model、api-design §10、
架构 §8 / §13(SK14)、AGENTS.md 同步。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
2026-11-04 11:48:10 +08:00
parent 6ab42959df
commit e499de7486
56 changed files with 3735 additions and 183 deletions

View File

@@ -33,6 +33,8 @@ class DocumentIn(BaseModel):
upload_id: UUID
role: Literal["tender", "addendum", "clarification", "bid_volume"]
volume_name: str | None = Field(default=None, max_length=40)
volume_kind: Literal["whole", "qualification", "commercial", "technical", "price", "other"] | None = None
replaces_document_id: UUID | None = None
class ParseIn(BaseModel):
@@ -84,7 +86,16 @@ async def list_documents(project_id: UUID, ctx: Ctx) -> dict[str, Any]:
@router.post("/projects/{project_id}/documents", status_code=201)
async def create_document(project_id: UUID, body: DocumentIn, ctx: Ctx) -> dict[str, Any]:
async with tenant_tx(ctx) as s:
doc = await service.create(s, ctx, project_id, body.upload_id, body.role, body.volume_name)
doc = await service.create(
s,
ctx,
project_id,
body.upload_id,
body.role,
body.volume_name,
volume_kind=body.volume_kind,
replaces=body.replaces_document_id,
)
return await service.doc_to_api(s, doc)

View File

@@ -16,7 +16,21 @@ from bid_doctor.document.pipeline.text_layer import Sentence
LABELS = {"tender": "招标文件", "addendum": "补遗", "clarification": "澄清", "bid_volume": "投标文件"}
def document_label(role: str, seq_no: int) -> str:
VOLUME_KINDS = {
"whole": "投标文件",
"qualification": "资格证明分册",
"commercial": "商务分册",
"technical": "技术分册",
"price": "报价分册",
"other": "投标分册",
}
def document_label(role: str, seq_no: int, volume_name: str | None = None, volume_kind: str | None = None) -> str:
"""「招标文件」「补遗 1」;投标分册是「技术分册 v2」,用户填了分册名时用分册名。"""
if role == "bid_volume":
name = volume_name or VOLUME_KINDS.get(volume_kind or "whole", "投标文件")
return f"{name} v{seq_no}" if seq_no > 1 else name
return f"{LABELS[role]} {seq_no}" if role in ("addendum", "clarification") else LABELS.get(role, role)
@@ -68,8 +82,10 @@ async def to_api_many(s: AsyncSession, ids: list[UUID]) -> dict[UUID, dict[str,
if not ids:
return {}
rows = await s.execute(
select(Citation, SourceDocument.role, SourceDocument.seq_no)
select(
Citation, SourceDocument.role, SourceDocument.seq_no, SourceDocument.volume_name, SourceDocument.volume_kind
)
.join(SourceDocument, SourceDocument.id == Citation.document_id)
.where(Citation.id.in_(ids))
)
return {c.id: to_api(c, document_label(role, seq)) for c, role, seq in rows.tuples()}
return {c.id: to_api(c, document_label(role, seq, name, kind)) for c, role, seq, name, kind in rows.tuples()}

View File

@@ -12,6 +12,8 @@ from bid_doctor.core.errors import AppError
OLE_MAGIC = bytes.fromhex("D0CF11E0A1B11AE1")
EXTENSIONS = {"pdf": {"pdf"}, "docx": {"docx", "doc", "wps"}, "doc": {"doc", "wps"}, "ofd": {"ofd"}}
# 电子投标平台的专有 / 加密格式(FR-M3-01、C8):只认有样本或公开资料确认过的写法,见过新的再补
PLATFORM_FORMATS = {"kedt": "苏采云(江苏政府采购)"}
@dataclass(frozen=True, slots=True)
@@ -23,6 +25,10 @@ class Detected:
def detect(data: bytes, filename: str) -> Detected:
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
if platform := PLATFORM_FORMATS.get(ext):
raise AppError(
"bid_file_platform_encrypted", f"这是{platform}平台的文件,请上传加密前的 Word 或 PDF", platform=platform
)
if data.startswith(b"%PDF"):
fmt, pages = "pdf", _pdf_pages(data)
elif data.startswith(b"PK\x03\x04"):
@@ -52,13 +58,17 @@ def _pdf_pages(data: bytes) -> int:
def _zip_kind(data: bytes) -> str:
try:
names = set(zipfile.ZipFile(io.BytesIO(data)).namelist())
infos = zipfile.ZipFile(io.BytesIO(data)).infolist()
except zipfile.BadZipFile as e:
raise AppError("file_corrupted") from e
names = {i.filename for i in infos}
if "word/document.xml" in names:
return "docx"
if "OFD.xml" in names:
return "ofd"
if "EncryptionInfo" in names or "EncryptedPackage" in names:
raise AppError("file_encrypted")
if infos and all(i.flag_bits & 0x1 for i in infos if not i.is_dir()):
# 条目全部加密的压缩包:电子投标客户端导出的加密投标文件常是这种(07 F3)
raise AppError("bid_file_platform_encrypted")
raise AppError("file_format_unsupported", "支持 PDF、Word(DOCX / DOC)和 WPS 文件")

View File

@@ -15,6 +15,7 @@ class SourceDocument(Base):
role: Mapped[str]
seq_no: Mapped[int]
volume_name: Mapped[str | None]
volume_kind: Mapped[str | None]
lineage_id: Mapped[UuidCol]
original_filename: Mapped[str]
detected_format: Mapped[str]

View File

@@ -0,0 +1,98 @@
"""解析流水线的前几个阶段:格式核对 → 转换 → 文字层 → OCR 补全(架构 §3.5)。
解析任务(招标文件)和检查批次(投标文件,m3-plan §2 prepare_bid)共用。阶段的进度、取消由调用方的 on_stage 负责。
TODO(S5):OCR 还没接,没有文字层的页 text_source = "empty"。
"""
from __future__ import annotations
import json
from collections.abc import Awaitable, Callable
from uuid import UUID
from sqlalchemy import delete, select
from bid_doctor.billing import service as billing
from bid_doctor.core import keys
from bid_doctor.core.crypto import open_object, seal_object, sha256
from bid_doctor.core.db import TenantContext, tenant_tx
from bid_doctor.core.errors import AppError
from bid_doctor.core.storage import storage
from bid_doctor.document.models import DocumentPage, SourceDocument
from bid_doctor.document.pipeline import convert, text_layer
from bid_doctor.document.pipeline.text_layer import Page
from bid_doctor.document.service import doc_prefix
# text_layer v2:页眉页脚带单独成句,句子编号和 v1 不同(同一份规范化 PDF 在 v1、v2 下的编号不能混用)
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v4", "sentence_split": "s3-v4"}
OCR_REQUIRED_RATIO = 0.5 # 超过一半的页没有文字层:招标文件按「需要 OCR」失败
OnStage = Callable[[str], Awaitable[None]]
async def pages(
ctx: TenantContext, document_id: UUID, on_stage: OnStage, *, reuse: bool, require_text: bool
) -> list[Page]:
"""返回文档的文字层。reuse=True 时,已有同一版本的文字层就直接读出来(投标分册每个版本只解析一次)。
require_text:没有文字层的页太多时报 ocr_required(招标文件)。
投标文件不报,扫描页由检查按「请人工核对」处理(C10)。"""
async with tenant_tx(ctx) as s:
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == document_id))).scalar_one()
dk = await keys.data_key(s, ctx, doc.project_id)
if reuse and doc.text_layer_key and (doc.pipeline_profile or {}).get("text_layer") == PROFILE["text_layer"]:
raw = open_object(dk, storage().get(doc.text_layer_key), doc.text_layer_key)
return [Page.from_json(p) for p in json.loads(raw)]
await on_stage("detect") # 上传完成时已识别过格式;这里只核对原件没被替换
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
if sha256(original) != doc.original_sha256:
raise AppError("file_corrupted")
await on_stage("convert")
if doc.normalized_key is None:
pdf = await convert.to_pdf(original, doc.detected_format)
key = f"{doc_prefix(doc)}/normalized.pdf"
storage().put(key, seal_object(dk, pdf, key))
async with tenant_tx(ctx) as s:
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
d.pipeline_profile = PROFILE
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
else:
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
await on_stage("text_layer")
result = text_layer.extract(str(doc.id), pdf)
text_key = f"{doc_prefix(doc)}/text.json"
storage().put(
text_key, seal_object(dk, json.dumps([p.to_json() for p in result], ensure_ascii=False).encode(), text_key)
)
async with tenant_tx(ctx) as s:
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等:先清残留
for p in result:
s.add(
DocumentPage(
tenant_id=ctx.tenant_id,
document_id=doc.id,
page_no=p.page_no,
width_pt=p.width,
height_pt=p.height,
rotation=p.rotation,
text_source=p.text_source,
char_count=p.char_count,
sentence_count=len(p.sentences),
)
)
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
d.text_layer_key, d.page_count, d.status = text_key, len(result), "ready"
d.pipeline_profile = PROFILE
_, plan = await billing.current_plan(s, ctx)
if len(result) > plan.limits["max_pages"]:
raise AppError("plan_limit_exceeded")
await on_stage("ocr")
empty = sum(p.text_source == "empty" for p in result)
if require_text and result and empty / len(result) > OCR_REQUIRED_RATIO:
raise AppError("ocr_required")
return result

View File

@@ -8,28 +8,24 @@ TODO(结构识别):章节、表格、分包识别还没做,现在只抽实
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from uuid import UUID
import httpx
from sqlalchemy import delete, select
from sqlalchemy import select
from bid_doctor.ai.providers.base import TransientError
from bid_doctor.billing import service as billing
from bid_doctor.checklist import extract
from bid_doctor.checklist import service as checklist
from bid_doctor.core import events, keys
from bid_doctor.core.crypto import open_object, seal_object, sha256
from bid_doctor.core import events
from bid_doctor.core.db import TenantContext, tenant_tx
from bid_doctor.core.errors import AppError
from bid_doctor.core.ids import uuid7
from bid_doctor.core.storage import storage
from bid_doctor.document.models import DocumentPage, ParseJob, SourceDocument
from bid_doctor.document.pipeline import convert, text_layer
from bid_doctor.document.service import doc_prefix
from bid_doctor.document.models import ParseJob, SourceDocument
from bid_doctor.document.pipeline import normalize, text_layer
from bid_doctor.notification.models import Notification
from bid_doctor.project import extracted as project_extracted
from bid_doctor.project import service as projects
@@ -45,8 +41,6 @@ PROGRESS = {
"verify": 92,
"merge": 95,
}
# text_layer v2:页眉页脚带单独成句,句子编号和 v1 不同(同一份规范化 PDF 在 v1、v2 下的编号不能混用)
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v4", "sentence_split": "s3-v4"}
class Cancelled(Exception):
@@ -113,58 +107,11 @@ async def _stages(ctx: TenantContext, job_id: UUID) -> dict[str, Any]:
async with tenant_tx(ctx) as s:
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == job.document_id))).scalar_one()
dk = await keys.data_key(s, ctx, doc.project_id)
await _stage(ctx, job_id, "detect") # 上传完成时已识别过格式;这里只核对原件没被替换
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
if sha256(original) != doc.original_sha256:
raise AppError("file_corrupted")
async def on_stage(stage: str) -> None:
await _stage(ctx, job_id, stage)
await _stage(ctx, job_id, "convert")
if doc.normalized_key is None:
pdf = await convert.to_pdf(original, doc.detected_format)
key = f"{doc_prefix(doc)}/normalized.pdf"
storage().put(key, seal_object(dk, pdf, key))
async with tenant_tx(ctx) as s:
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
d.pipeline_profile = PROFILE
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
else:
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
await _stage(ctx, job_id, "text_layer")
pages = text_layer.extract(str(doc.id), pdf)
text_key = f"{doc_prefix(doc)}/text.json"
storage().put(
text_key, seal_object(dk, json.dumps([p.to_json() for p in pages], ensure_ascii=False).encode(), text_key)
)
async with tenant_tx(ctx) as s:
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等:先清残留
for p in pages:
s.add(
DocumentPage(
tenant_id=ctx.tenant_id,
document_id=doc.id,
page_no=p.page_no,
width_pt=p.width,
height_pt=p.height,
rotation=p.rotation,
text_source=p.text_source,
char_count=p.char_count,
sentence_count=len(p.sentences),
)
)
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
d.text_layer_key, d.page_count, d.status = text_key, len(pages), "ready"
_, plan = await billing.current_plan(s, ctx)
if len(pages) > plan.limits["max_pages"]:
raise AppError("plan_limit_exceeded")
await _stage(ctx, job_id, "ocr")
empty = sum(p.text_source == "empty" for p in pages)
if pages and empty / len(pages) > 0.5:
raise AppError("ocr_required")
pages = await normalize.pages(ctx, doc.id, on_stage, reuse=False, require_text=True)
await _stage(ctx, job_id, "structure")
body = text_layer.body_sentences(pages)

View File

@@ -33,22 +33,42 @@ def doc_prefix(doc: SourceDocument) -> str:
async def create(
s: AsyncSession, ctx: TenantContext, project_id: UUID, upload_id: UUID, role: str, volume_name: str | None
s: AsyncSession,
ctx: TenantContext,
project_id: UUID,
upload_id: UUID,
role: str,
volume_name: str | None,
*,
volume_kind: str | None = None,
replaces: UUID | None = None,
) -> SourceDocument:
"""挂到项目上。投标分册重新上传时带 replaces,生成同一分册的新版本(FR-M3-01)。"""
p = await projects.get(s, ctx, project_id)
projects.ensure_editable(p)
doc_id = uuid7()
lineage, seq = doc_id, 0
if role == "bid_volume":
volume_kind = volume_kind or "whole"
seq = 1
if replaces is not None:
prev = await get(s, replaces)
if prev.project_id != project_id or prev.role != "bid_volume":
raise AppError("not_found")
lineage = prev.lineage_id
seq = await _next_version(s, lineage)
volume_name = volume_name or prev.volume_name
volume_kind = prev.volume_kind if volume_kind == "whole" and prev.volume_kind else volume_kind
elif role != "tender":
seq = (
await s.execute(
select(func.coalesce(func.max(SourceDocument.seq_no), 0) + 1).where(
SourceDocument.project_id == project_id, SourceDocument.role == role
)
)
).scalar_one()
meta, data = await uploads.take(ctx, upload_id)
assert meta.detected_format is not None
seq = (
await s.execute(
select(func.coalesce(func.max(SourceDocument.seq_no), -1) + 1).where(
SourceDocument.project_id == project_id, SourceDocument.role == role
)
)
).scalar_one()
if role == "tender":
seq = 0
doc_id = uuid7()
doc = SourceDocument(
id=doc_id,
tenant_id=ctx.tenant_id,
@@ -56,7 +76,8 @@ async def create(
role=role,
seq_no=seq,
volume_name=volume_name,
lineage_id=doc_id,
volume_kind=volume_kind if role == "bid_volume" else None,
lineage_id=lineage,
original_filename=meta.filename,
detected_format=meta.detected_format,
size_bytes=meta.size_bytes,
@@ -76,7 +97,7 @@ async def create(
"source_document",
doc.id,
project_id=project_id,
after={"role": role, "filename": meta.filename},
after={"role": role, "filename": meta.filename, "version": seq if role == "bid_volume" else None},
)
await s.flush()
await s.refresh(doc)
@@ -84,6 +105,34 @@ async def create(
return doc
async def _next_version(s: AsyncSession, lineage_id: UUID) -> int:
return (
await s.execute(
select(func.coalesce(func.max(SourceDocument.seq_no), 0) + 1).where(SourceDocument.lineage_id == lineage_id)
)
).scalar_one()
async def latest_bid_volumes(s: AsyncSession, project_id: UUID) -> list[SourceDocument]:
"""每个投标分册的最新版本(检查默认用它们,CheckPreview.bid_documents)。"""
docs = (
(
await s.execute(
select(SourceDocument)
.where(SourceDocument.project_id == project_id, SourceDocument.role == "bid_volume")
.order_by(SourceDocument.created_at)
)
)
.scalars()
.all()
)
latest: dict[UUID, SourceDocument] = {}
for d in docs:
if d.lineage_id not in latest or d.seq_no > latest[d.lineage_id].seq_no:
latest[d.lineage_id] = d
return list(latest.values())
async def get(s: AsyncSession, document_id: UUID) -> SourceDocument:
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == document_id))).scalar_one_or_none()
if doc is None:
@@ -189,14 +238,24 @@ async def doc_to_api(s: AsyncSession, doc: SourceDocument) -> dict[str, Any]:
select(ParseJob).where(ParseJob.document_id == doc.id).order_by(ParseJob.attempt_no.desc()).limit(1)
)
).scalar_one_or_none()
newer = None
if doc.role == "bid_volume":
newer = (
await s.execute(
select(SourceDocument.id)
.where(SourceDocument.lineage_id == doc.lineage_id, SourceDocument.seq_no > doc.seq_no)
.limit(1)
)
).scalar_one_or_none()
return {
"id": doc.id,
"role": doc.role,
"label": document_label(doc.role, doc.seq_no),
"label": document_label(doc.role, doc.seq_no, doc.volume_name, doc.volume_kind),
"seq_no": doc.seq_no,
"volume_name": doc.volume_name,
"volume_kind": doc.volume_kind,
"lineage_id": doc.lineage_id,
"is_latest_version": True,
"is_latest_version": newer is None,
"original_filename": doc.original_filename,
"detected_format": doc.detected_format,
"size_bytes": doc.size_bytes,
@@ -210,18 +269,18 @@ async def doc_to_api(s: AsyncSession, doc: SourceDocument) -> dict[str, Any]:
async def summaries(s: AsyncSession, project_ids: list[UUID]) -> dict[UUID, dict[str, Any]]:
"""工作台用:每个项目的招标文件数、正在跑的解析。"""
out: dict[UUID, dict[str, Any]] = {pid: {"tenders": 0, "job": None} for pid in project_ids}
"""工作台用:每个项目的招标文件数、投标分册数、正在跑的解析。"""
out: dict[UUID, dict[str, Any]] = {pid: {"tenders": 0, "bids": 0, "job": None} for pid in project_ids}
if not project_ids:
return out
for pid, n in (
for pid, role, n in (
await s.execute(
select(SourceDocument.project_id, func.count())
.where(SourceDocument.project_id.in_(project_ids), SourceDocument.role == "tender")
.group_by(SourceDocument.project_id)
select(SourceDocument.project_id, SourceDocument.role, func.count())
.where(SourceDocument.project_id.in_(project_ids), SourceDocument.role.in_(("tender", "bid_volume")))
.group_by(SourceDocument.project_id, SourceDocument.role)
)
).tuples():
out[pid]["tenders"] = n
out[pid]["tenders" if role == "tender" else "bids"] = n
for job in (
await s.execute(
select(ParseJob).where(ParseJob.project_id.in_(project_ids), ParseJob.status.in_(("queued", "running")))