feat(ocr): 扫描件与图片识别——OCR 进文字层、资质库上传识别、大图浏览
计划见 docs/architecture/vision-plan.md(VS1–VS13,docs/decisions.md)。 - 验证:OCR(RapidOCR + PP-OCRv6 small,CPU)和 DeepSeek Flash 看图在国动投标文件 和 7 张证书上初测(tech-validation §7,代码 spikes/s5-ocr/) - OCR 服务 services/ocr(镜像、Compose 服务):整页转正(含 180°)、红章附近按红色通道重读 - 扫描页和插图识别后并入文字层 v6,句子标 source=ocr;出处标「OCR」(迁移 0013); 依据识别文字的问题降级,规则不查扫描附件;国动端到端见 check-eval §16 - 资质库上传识别:添加资质对话框最上面传证书照片,识别后填表,只预填和 OCR 文字 对得上的字段;识别值留痕、照片挂成附件(迁移 0014,提示词 vision.cert_fields v1) - 资质库改为查看详情(只读键值 + 附件缩略图),附件用大图浏览控件看,不开新标签页 (设计规范 §7.23);PDF 附件按页出预览图 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
19
server/migrations/versions/0013_citation_ocr.py
Normal file
19
server/migrations/versions/0013_citation_ocr.py
Normal file
@@ -0,0 +1,19 @@
|
||||
"""出处标出「识别文字」(vision-plan 第 1 步):原句来自扫描页或插图的 OCR 时,界面在出处上标出来,提醒可能有错字。
|
||||
|
||||
Revision ID: 0013
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0013"
|
||||
down_revision = "0012"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.get_bind().exec_driver_sql("ALTER TABLE citation ADD COLUMN ocr boolean NOT NULL DEFAULT false")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.get_bind().exec_driver_sql("ALTER TABLE citation DROP COLUMN ocr")
|
||||
60
server/migrations/versions/0014_qualification_scan.py
Normal file
60
server/migrations/versions/0014_qualification_scan.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""资质库上传识别(vision-plan 第 2 步):上传证书扫描件 → OCR + 视觉模型识别 → 用户核对后保存成资质。
|
||||
|
||||
- qualification_scan:一次识别。原件按企业级密钥加密存对象存储;result 是识别结果(每张证书的字段、引用的 OCR 行、
|
||||
校验结论)。没保存的识别 24 小时后连同原件删除(worker 定时任务);
|
||||
- qualification.holder:证书主体(企业名、著作权人、持证人);
|
||||
- qualification.ai_values:保存时的识别值快照(识别值和用户最终值都留痕,同清单项的 ai_values);
|
||||
- ai_call.job_type 加 qualification_scan。
|
||||
|
||||
Revision ID: 0014
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "0014"
|
||||
down_revision = "0013"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.get_bind().exec_driver_sql("""
|
||||
ALTER TABLE qualification ADD COLUMN holder text;
|
||||
ALTER TABLE qualification ADD COLUMN ai_values jsonb;
|
||||
|
||||
CREATE TABLE qualification_scan (
|
||||
id uuid PRIMARY KEY,
|
||||
tenant_id uuid NOT NULL REFERENCES tenant (id),
|
||||
filename text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
sha256 bytea NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
media_type text NOT NULL,
|
||||
status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'running', 'succeeded', 'failed')),
|
||||
error_code text,
|
||||
result jsonb,
|
||||
created_by uuid NOT NULL REFERENCES user_account (id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz,
|
||||
UNIQUE (tenant_id, id)
|
||||
);
|
||||
CREATE INDEX ON qualification_scan (tenant_id, created_at);
|
||||
|
||||
ALTER TABLE qualification_scan ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE qualification_scan FORCE ROW LEVEL SECURITY;
|
||||
CREATE POLICY tenant_isolation ON qualification_scan
|
||||
USING (tenant_id = current_tenant()) WITH CHECK (tenant_id = current_tenant());
|
||||
|
||||
ALTER TABLE ai_call DROP CONSTRAINT ai_call_job_type_check;
|
||||
ALTER TABLE ai_call ADD CONSTRAINT ai_call_job_type_check
|
||||
CHECK (job_type IN ('parse_job', 'check_run', 'qualification_scan'));
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.get_bind().exec_driver_sql("""
|
||||
-- ai_call 只追加、不能删行:job_type 的约束保持三种取值
|
||||
DROP TABLE qualification_scan;
|
||||
ALTER TABLE qualification DROP COLUMN ai_values;
|
||||
ALTER TABLE qualification DROP COLUMN holder;
|
||||
""")
|
||||
@@ -38,4 +38,6 @@ class TransientError(Exception):
|
||||
class Provider(Protocol):
|
||||
name: str
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str, task: str) -> Completion: ...
|
||||
async def complete_json(
|
||||
self, system: str, user: str, capability: str, task: str, images: list[bytes] | None = None
|
||||
) -> Completion: ...
|
||||
|
||||
@@ -196,6 +196,50 @@ def _check_match(user: str) -> dict[str, Any]:
|
||||
return {"results": results}
|
||||
|
||||
|
||||
OCR_LINE = re.compile(r"^(L\d+) (.*)$")
|
||||
CERT_LABELS = {
|
||||
"cert_no": r"证书编号|证书号",
|
||||
"holder": r"企业名称|持有人|著作权人|申请企业|姓\s*名",
|
||||
"level": r"等级|级别",
|
||||
"issuer": r"发证机构|颁发机构",
|
||||
"issued_on": r"发证日期|批准日期",
|
||||
"valid_from": r"有效期起",
|
||||
"valid_to": r"有效期至",
|
||||
}
|
||||
|
||||
|
||||
def _cert_fields(user: str) -> dict[str, Any]:
|
||||
"""证书识别:标题行(含「证书」二字)当名称,「证书编号:」「有效期至:」这类写法取冒号后面的值;打了「■」的当遮挡。"""
|
||||
lines = [m.groups() for raw in user.splitlines() if (m := OCR_LINE.match(raw.strip()))]
|
||||
fields: list[dict[str, Any]] = []
|
||||
masked: list[str] = []
|
||||
title = next(((sid, t) for sid, t in lines if t.endswith("证书") or ("证书" in t and ":" not in t)), None)
|
||||
if title:
|
||||
fields.append({"field": "name", "value": title[1], "lines": [title[0]]})
|
||||
for sid, t in lines:
|
||||
for field, label in CERT_LABELS.items():
|
||||
m = re.match(rf"^(?:{label})[::]\s*(.+)$", t)
|
||||
if not m:
|
||||
continue
|
||||
if "■" in m.group(1):
|
||||
masked.append(field)
|
||||
else:
|
||||
fields.append({"field": field, "value": m.group(1).strip(), "lines": [sid]})
|
||||
if not fields:
|
||||
return {"certificates": []}
|
||||
name = title[1] if title else ""
|
||||
category = (
|
||||
"software_copyright"
|
||||
if "著作权" in name
|
||||
else "system_cert"
|
||||
if "体系" in name
|
||||
else "personnel"
|
||||
if "职业" in name
|
||||
else "enterprise_qualification"
|
||||
)
|
||||
return {"certificates": [{"category": category, "fields": fields, "masked": masked}]}
|
||||
|
||||
|
||||
TASKS: dict[str, Callable[[list[tuple[str, str, str, str]]], dict[str, Any]]] = {
|
||||
"extract.substantive": _substantive,
|
||||
"extract.qualification": _qualification,
|
||||
@@ -209,9 +253,13 @@ TASKS: dict[str, Callable[[list[tuple[str, str, str, str]]], dict[str, Any]]] =
|
||||
class FakeProvider:
|
||||
name = "fake"
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str, task: str) -> Completion:
|
||||
del system, capability
|
||||
if task == "check.match":
|
||||
async def complete_json(
|
||||
self, system: str, user: str, capability: str, task: str, images: list[bytes] | None = None
|
||||
) -> Completion:
|
||||
del system, capability, images
|
||||
if task == "vision.cert_fields":
|
||||
data = _cert_fields(user)
|
||||
elif task == "check.match":
|
||||
data = _check_match(user)
|
||||
else:
|
||||
data = TASKS.get(task, lambda _lines: {"items": []})(_lines(user))
|
||||
|
||||
@@ -7,7 +7,9 @@ TODO(路由表):按能力配候选列表和降级、按「供应商 × 租户
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -18,16 +20,25 @@ from bid_doctor.core.config import get_settings
|
||||
class OpenAICompatProvider:
|
||||
name = "openai_compat"
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str, task: str) -> Completion:
|
||||
async def complete_json(
|
||||
self, system: str, user: str, capability: str, task: str, images: list[bytes] | None = None
|
||||
) -> Completion:
|
||||
"""images:JPEG 图片,按 OpenAI 兼容的 image_url(base64 data URL)放在 user 消息里,不给模型可访问的外链。"""
|
||||
del task
|
||||
s = get_settings()
|
||||
model = s.ai_models.get(capability) or s.ai_model
|
||||
content: str | list[dict[str, Any]] = user
|
||||
if images:
|
||||
content = [{"type": "text", "text": user}] + [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64," + base64.b64encode(x).decode()}}
|
||||
for x in images
|
||||
]
|
||||
body = {
|
||||
"model": model,
|
||||
"temperature": 0,
|
||||
"response_format": {"type": "json_object"},
|
||||
"max_tokens": s.ai_max_output_tokens,
|
||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": content}],
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(base_url=s.ai_base_url, timeout=httpx.Timeout(180, connect=10)) as c:
|
||||
@@ -48,6 +59,8 @@ class OpenAICompatProvider:
|
||||
incomplete = choice.get("finish_reason") == "length" or near_limit(usage.get("completion_tokens"))
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and len(data) == 2 and isinstance(data.get("content"), dict):
|
||||
data = data["content"] # 看图时偶尔把结果包一层 {"type": "json_object", "content": {...}}(S5 初测)
|
||||
except json.JSONDecodeError:
|
||||
# 以前当成「没有结果」:输出被截断时,这一整块的条目会悄悄丢掉(2026-11-03 审计发现,输出正好 8192 token)
|
||||
data, incomplete = {}, True
|
||||
|
||||
@@ -34,9 +34,18 @@ def provider() -> Provider:
|
||||
|
||||
|
||||
async def run_json(
|
||||
ctx: TenantContext, *, job_type: str, job_id: UUID, task: str, prompt_id: str, version: int, user: str
|
||||
ctx: TenantContext,
|
||||
*,
|
||||
job_type: str,
|
||||
job_id: UUID,
|
||||
task: str,
|
||||
prompt_id: str,
|
||||
version: int,
|
||||
user: str,
|
||||
images: list[bytes] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""外部内容(招标文件)只放在 user 的数据区;系统提示词声明数据区里的指令一律不执行。"""
|
||||
"""外部内容(招标文件)只放在 user 的数据区;系统提示词声明数据区里的指令一律不执行。
|
||||
images:看图的任务(vision)一起发的图片;输入哈希把图片算进去,图片本身不存(架构 §3.6 不存原文)。"""
|
||||
p = prompts.get(prompt_id, version)
|
||||
prov = provider()
|
||||
started = time.monotonic()
|
||||
@@ -44,7 +53,7 @@ async def run_json(
|
||||
error: TransientError | None = None
|
||||
for attempt in range(RETRIES + 1):
|
||||
try:
|
||||
result = await prov.complete_json(p.body, user, p.capability, task)
|
||||
result = await prov.complete_json(p.body, user, p.capability, task, images)
|
||||
error = None
|
||||
break
|
||||
except TransientError as e: # 只对临时错误重试
|
||||
@@ -64,7 +73,7 @@ async def run_json(
|
||||
prompt_id=p.id,
|
||||
prompt_version=p.version,
|
||||
prompt_hash=p.sha256,
|
||||
input_hash=sha256(user),
|
||||
input_hash=sha256(user + "".join(sha256(x).hex() for x in images or [])),
|
||||
output_hash=sha256(result.raw) if result else None,
|
||||
tokens_in=result.tokens_in if result else None,
|
||||
tokens_out=result.tokens_out if result else None,
|
||||
|
||||
@@ -11,6 +11,7 @@ from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from bid_doctor.check.match import Result
|
||||
from bid_doctor.document.pipeline.text_layer import Sentence
|
||||
|
||||
FATAL_CATEGORIES = {"qualification", "invalidity"}
|
||||
REASONS = {
|
||||
@@ -45,6 +46,20 @@ def severity(item: dict[str, Any]) -> str:
|
||||
return "hint"
|
||||
|
||||
|
||||
LOWER = {"fatal": "important", "important": "hint", "hint": "hint"}
|
||||
OCR_NOTE = "依据的投标原文是扫描件或插图上识别出的文字,可能有识别错误,请对照原图核对。"
|
||||
|
||||
|
||||
def ocr_based(evidence: list[tuple[Any, Sentence]]) -> bool:
|
||||
return any(x.source == "ocr" for _, x in evidence)
|
||||
|
||||
|
||||
def soften(severity: str, explanation: str, *, numeric: bool) -> tuple[str, str]:
|
||||
"""依据是 OCR 文字的问题降一级;报价数字(大小写、合计、超限价)降到提示(决策 VS4:OCR 和视觉模型读数一致才保留,
|
||||
视觉复核还没做)。识别错一个字就可能是误报,不能给致命。"""
|
||||
return ("hint" if numeric else LOWER[severity]), f"{explanation}{OCR_NOTE}"
|
||||
|
||||
|
||||
def _pages(pages: list[int]) -> str:
|
||||
return "、".join(f"第 {p} 页" for p in pages)
|
||||
|
||||
@@ -115,7 +130,13 @@ def _spec(result: Result, item: dict[str, Any]) -> Spec | None:
|
||||
if result.verdict == "scanned":
|
||||
total = result.detail.get("scanned_total") or 0
|
||||
where = result.detail.get("scanned_where")
|
||||
# 只说扫描件在哪几页,不说「材料可能在扫描件里」:扫描件上是不是这份材料,没有 OCR 分不出来
|
||||
# 只说扫描件在哪几页,不说「材料可能在扫描件里」:扫描件上是不是这份材料,没有页面分类分不清
|
||||
if result.detail.get("scanned_ocr"):
|
||||
text = (
|
||||
f"投标文件里没有找到这份材料。{where}是扫描件(共 {total} 页),"
|
||||
"系统识别了上面的文字,没有认出这份材料,但识别可能有错漏;材料如果附在这些页上,请人工核对,没有附就是缺了。"
|
||||
)
|
||||
return Spec("manual_review", "hint", f"请人工核对(没有找到):{name}", text, base)
|
||||
text = (
|
||||
f"投标文件文字部分没有找到这份材料。{where}是扫描件(共 {total} 页),系统读不出上面的文字;"
|
||||
"材料如果附在这些页上,请人工核对,没有附就是缺了。"
|
||||
|
||||
@@ -64,6 +64,7 @@ class BidDoc:
|
||||
sentences: list[Sentence]
|
||||
scanned_pages: list[int] = field(default_factory=list[int]) # 没有文字层的页(扫描件),C10
|
||||
header_footer: list[Sentence] = field(default_factory=list[Sentence]) # 页眉页脚(规则检查看页眉里的项目名称)
|
||||
ocr_pages: list[int] = field(default_factory=list[int]) # 扫描页里识别出了文字的(scanned_pages 的子集)
|
||||
|
||||
|
||||
Batch = list[tuple[UUID, dict[str, Any], dict[str, tuple["BidDoc", Sentence]]]] # 一次调用判断的清单项
|
||||
@@ -110,7 +111,8 @@ def scanned_fallback(
|
||||
results: dict[UUID, Result], items: dict[UUID, dict[str, Any]], modes: dict[UUID, Any], docs: list[BidDoc]
|
||||
) -> None:
|
||||
"""扫描件兜底(C10):要附证明材料的要求没找到,而投标文件里有扫描件页时,不报「未找到响应」,
|
||||
改成「可能在扫描件页上,请人工核对」。没有 OCR 时,扫描件里有没有这份材料分不出来。"""
|
||||
改成「请人工核对扫描件」。扫描页识别了文字也一样:识别可能有错漏,扫描件上是什么材料要等页面分类
|
||||
(vision-plan 第 3 步)才分得清。"""
|
||||
pages = [f"{d.label} 第 {p} 页" for d in docs for p in d.scanned_pages]
|
||||
if not pages:
|
||||
return
|
||||
@@ -121,7 +123,13 @@ def scanned_fallback(
|
||||
if r is None or r.verdict != "not_found" or not (d.evidence or cats & {"qualification", "scoring"}):
|
||||
continue
|
||||
r.verdict, r.evidence_status, r.reason_code = "scanned", "none", None
|
||||
r.detail = {**r.detail, "scanned_pages": pages[:30], "scanned_total": len(pages), "scanned_where": where}
|
||||
r.detail = {
|
||||
**r.detail,
|
||||
"scanned_pages": pages[:30],
|
||||
"scanned_total": len(pages),
|
||||
"scanned_where": where,
|
||||
"scanned_ocr": all(set(d.scanned_pages) <= set(d.ocr_pages) for d in docs),
|
||||
}
|
||||
|
||||
|
||||
def page_ranges(pages: list[int]) -> str:
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from decimal import Decimal
|
||||
@@ -37,6 +38,7 @@ from bid_doctor.document import service as documents
|
||||
from bid_doctor.document.citations import document_label
|
||||
from bid_doctor.document.models import SourceDocument
|
||||
from bid_doctor.document.pipeline import normalize, text_layer
|
||||
from bid_doctor.document.pipeline.ocr import OcrUnavailable
|
||||
from bid_doctor.document.pipeline.text_layer import norm
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import service as projects
|
||||
@@ -68,6 +70,9 @@ async def run(tenant_id: UUID, run_id: UUID) -> None:
|
||||
await _finish(ctx, run_id, "cancelled", "cancelled", "cancelled")
|
||||
except AppError as e:
|
||||
await _finish(ctx, run_id, "failed", "input", e.code)
|
||||
except OcrUnavailable as e:
|
||||
log.warning("check run %s ocr unavailable: %s", run_id, e)
|
||||
await _finish(ctx, run_id, "failed", "infra", "ocr_unavailable")
|
||||
except (TransientError, httpx.HTTPError) as e:
|
||||
log.warning("check run %s infra error: %s", run_id, e)
|
||||
code = "model_unavailable" if isinstance(e, TransientError) else "converter_unavailable"
|
||||
@@ -131,8 +136,9 @@ async def _stages(ctx: TenantContext, run_id: UUID) -> None:
|
||||
no,
|
||||
label,
|
||||
text_layer.body_sentences(pages),
|
||||
[p.page_no for p in pages if p.text_source == "empty"],
|
||||
[p.page_no for p in pages if p.scanned],
|
||||
[x for p in pages for x in p.sentences if x.location == "header_footer"],
|
||||
[p.page_no for p in pages if p.text_source == "ocr"],
|
||||
)
|
||||
)
|
||||
pages_info.append(
|
||||
@@ -141,7 +147,11 @@ async def _stages(ctx: TenantContext, run_id: UUID) -> None:
|
||||
"label": label,
|
||||
"pages": len(pages),
|
||||
"pages_without_text": [p.page_no for p in pages if p.text_source == "empty"],
|
||||
"chars": sum(p.char_count for p in pages),
|
||||
"ocr_pages": [p.page_no for p in pages if p.text_source == "ocr"],
|
||||
"ocr_figures": sum(len(p.figures) for p in pages if p.text_source == "mixed"),
|
||||
"chars": sum(p.char_count + p.ocr_chars for p in pages),
|
||||
# 文字页不到三成:整份是扫描件(纸质投标扫描),格式文件也在扫描页上,规则要看识别文字
|
||||
"scan_only": sum(not p.scanned for p in pages) < 0.3 * len(pages),
|
||||
}
|
||||
)
|
||||
if sum(x["chars"] for x in pages_info) < MIN_BID_CHARS:
|
||||
@@ -214,7 +224,11 @@ async def _stages(ctx: TenantContext, run_id: UUID) -> None:
|
||||
qual_hits, qual_cov = qualification.run(lot_items, results, bid_docs, library, facts.deadline)
|
||||
|
||||
await _stage(ctx, run_id, "rules")
|
||||
hits, rule_cov = rules.run(facts, bid_docs, tender_sents)
|
||||
hits, rule_cov = rules.run(facts, _rule_docs(bid_docs, pages_info), tender_sents)
|
||||
if any(not x["scan_only"] and (x["ocr_pages"] or x["ocr_figures"]) for x in pages_info):
|
||||
rule_cov["not_checked"].append(
|
||||
"扫描附件和插图上的识别文字不做报价形式、形式要件的规则检查:附件多是历史合同、证书这类别人的文件"
|
||||
)
|
||||
covered = {
|
||||
(d.label, x.page_no, norm(x.text))
|
||||
for res in results.values()
|
||||
@@ -309,6 +323,19 @@ async def _facts(s: AsyncSession, ctx: TenantContext, r: CheckRun, snapshot: dic
|
||||
)
|
||||
|
||||
|
||||
def _rule_docs(docs: list[match.BidDoc], pages_info: list[dict[str, Any]]) -> list[match.BidDoc]:
|
||||
"""规则检查(报价形式、形式要件)查的是投标人自己填写的格式文件。扫描附件(历史合同、劳动合同、证书)和插图上的
|
||||
识别文字不查:国动真实投标文件开了 OCR 以后,劳动合同模板的空格、历史合同的价款和「另行协商」条款报出了 21 个
|
||||
噪音(vision-plan 第 1 步)。整份都是扫描件时格式文件也在扫描页上,照查(问题按 OCR 降级)。"""
|
||||
scan_only = {x["id"]: x["scan_only"] for x in pages_info}
|
||||
return [
|
||||
d
|
||||
if scan_only.get(str(d.id))
|
||||
else dataclasses.replace(d, sentences=[x for x in d.sentences if x.source == "text"])
|
||||
for d in docs
|
||||
]
|
||||
|
||||
|
||||
async def _write_rules(
|
||||
s: AsyncSession,
|
||||
ctx: TenantContext,
|
||||
@@ -320,6 +347,9 @@ async def _write_rules(
|
||||
"""规则问题:挂投标原文的句子;有依据的(限价、有效期要求、项目名称)再挂招标侧出处,资质问题挂清单项自己的出处。
|
||||
只按资质库记录判断的资质问题没有投标侧出处,界面显示「依据:资质库记录」(detail 里存记录快照,QL2)。"""
|
||||
for h in hits:
|
||||
severity, explanation = h.severity, h.explanation
|
||||
if findings.ocr_based(h.evidence):
|
||||
severity, explanation = findings.soften(severity, explanation, numeric=h.code.startswith("PRICE."))
|
||||
bid_cites = []
|
||||
for doc_id in dict.fromkeys(d.id for d, _ in h.evidence):
|
||||
sents = [x for d, x in h.evidence if d.id == doc_id]
|
||||
@@ -333,9 +363,9 @@ async def _write_rules(
|
||||
rule_code=h.code,
|
||||
match_key=f"rule:{h.code}:{h.target}",
|
||||
type=h.type,
|
||||
severity=h.severity,
|
||||
severity=severity,
|
||||
title=h.title[:200],
|
||||
explanation=h.explanation,
|
||||
explanation=explanation,
|
||||
detector="rule",
|
||||
detail=h.detail,
|
||||
checks_passed=[],
|
||||
@@ -408,6 +438,9 @@ async def _write(
|
||||
spec = findings.spec(res, item)
|
||||
if spec is None:
|
||||
continue
|
||||
severity, explanation = spec.severity, spec.explanation
|
||||
if findings.ocr_based(res.evidence):
|
||||
severity, explanation = findings.soften(severity, explanation, numeric=False)
|
||||
f = Finding(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
@@ -416,9 +449,9 @@ async def _write(
|
||||
checklist_item_id=item_id,
|
||||
match_key=f"item:{item_id}",
|
||||
type=spec.type,
|
||||
severity=spec.severity,
|
||||
severity=severity,
|
||||
title=spec.title[:200],
|
||||
explanation=spec.explanation,
|
||||
explanation=explanation,
|
||||
detector="ai",
|
||||
detail={k: v for k, v in spec.detail.items() if k != "search_scope"},
|
||||
search_scope={"documents": res.detail.get("search_scope") or [], "sections": []}
|
||||
@@ -556,6 +589,8 @@ def _coverage(
|
||||
reused = sum(bool(x.detail.get("reused_from")) for x in results.values())
|
||||
unverified = sum(x.evidence_status == "unverified" for x in results.values())
|
||||
scanned = sum(len(x["pages_without_text"]) for x in pages_info)
|
||||
ocr_pages = sum(len(x.get("ocr_pages") or []) for x in pages_info)
|
||||
ocr_figures = sum(x.get("ocr_figures") or 0 for x in pages_info)
|
||||
checked = [f"响应完整性:正式清单 {len(items)} 条,逐条在投标文件里找响应"]
|
||||
not_checked = []
|
||||
if pending:
|
||||
@@ -568,8 +603,15 @@ def _coverage(
|
||||
not_checked.append(f"{over_budget} 条超出本次检查的预算没有判断(投标文件特别大),已列为「请人工核对」")
|
||||
if reused:
|
||||
checked.append(f"其中 {reused} 条和上次检查送去判断的内容完全相同,沿用上次的结论")
|
||||
if ocr_pages or ocr_figures:
|
||||
what = "、".join(
|
||||
x
|
||||
for x in (f"{ocr_pages} 页扫描件" if ocr_pages else "", f"{ocr_figures} 张插图" if ocr_figures else "")
|
||||
if x
|
||||
)
|
||||
checked.append(f"{what}上的文字是识别出来的(OCR),一起参与检查;识别可能有错漏,依据识别文字的问题降一级")
|
||||
if scanned:
|
||||
not_checked.append(f"{scanned} 页没有文字层(扫描件),这些页上的内容没有检查")
|
||||
not_checked.append(f"{scanned} 页扫描件识别不出文字,这些页上的内容没有检查")
|
||||
for reason, n in skipped.items():
|
||||
not_checked.append(f"{applicability.SKIP_LABELS[reason]} {n} 条:不在投标文件里逐条找响应")
|
||||
labels = {"technical": "技术要求", "commercial": "商务要求"}
|
||||
|
||||
@@ -44,11 +44,12 @@ LOCAL_TZ = ZoneInfo("Asia/Shanghai") # 投标截止时间按北京时间取日
|
||||
RULE_SET_VERSION = "2026.11-r5"
|
||||
WAIVE_WINDOW = timedelta(hours=24)
|
||||
ERROR_MESSAGES = {
|
||||
"bid_text_missing": "投标文件里几乎读不出文字(可能都是扫描件),扫描件识别还没有上线",
|
||||
"bid_text_missing": "投标文件里几乎读不出文字(可能是很模糊的扫描件),请换一份清晰的文件",
|
||||
"file_corrupted": "投标文件无法转换,可能已加密或损坏",
|
||||
"plan_limit_exceeded": "投标文件页数超出套餐上限",
|
||||
"checklist_empty": "正式清单还没有条目",
|
||||
"converter_unavailable": "文档转换服务暂时不可用",
|
||||
"ocr_unavailable": "扫描件识别服务暂时不可用",
|
||||
"model_unavailable": "模型服务暂时不可用",
|
||||
"internal_error": "系统出错了",
|
||||
}
|
||||
|
||||
@@ -51,6 +51,13 @@ class Settings(BaseSettings):
|
||||
# 每次调用的输出上限(token)。输出用到 92% 以上就按截断处理、拆小重跑(ai/providers/base.near_limit)
|
||||
ai_max_output_tokens: int = 8192
|
||||
|
||||
# OCR(vision-plan §4.1):扫描页、插图交给 OCR 服务(services/ocr)。
|
||||
# none = 不识别,扫描页按没有文字处理(dev / 测试默认)
|
||||
ocr_provider: Literal["none", "http"] = "none"
|
||||
ocr_url: str = "http://127.0.0.1:8029"
|
||||
ocr_concurrency: int = 4 # 同时发给 OCR 服务的页数
|
||||
ocr_dpi: int = 200
|
||||
|
||||
# 专业版对公转账的收款信息(决策 M6-6):不进代码;没配时开通申请里显示「(待定)」,生产环境必须配
|
||||
payee_name: str = ""
|
||||
payee_bank: str = ""
|
||||
@@ -79,6 +86,8 @@ class Settings(BaseSettings):
|
||||
raise ValueError("生产环境不能使用 dev 短信通道")
|
||||
if self.ai_provider == "fake":
|
||||
raise ValueError("生产环境不能使用假模型")
|
||||
if self.ocr_provider == "none":
|
||||
raise ValueError("生产环境要接 OCR 服务(BD_OCR_PROVIDER=http、BD_OCR_URL)")
|
||||
if not self.cookie_secure:
|
||||
raise ValueError("生产环境必须开启 cookie_secure")
|
||||
if not self.ops_allowed_ips or any(c in ("0.0.0.0/0", "::/0") for c in self.ops_allowed_ips):
|
||||
|
||||
@@ -55,10 +55,11 @@ CODES: dict[str, tuple[int, str]] = {
|
||||
"citation_required": (422, "请补上出处,或者声明「人工确认、无出处」"),
|
||||
"note_required": (422, "请填写说明"),
|
||||
"bulk_only_hint": (422, "批量处理只限「提示」级问题"),
|
||||
"bid_text_missing": (422, "投标文件里几乎读不出文字(可能都是扫描件),扫描件识别还没有上线"),
|
||||
"bid_text_missing": (422, "投标文件里几乎读不出文字(可能是很模糊的扫描件),请换一份清晰的文件"),
|
||||
"sentence_not_found": (422, "选中的句子不在这份文件里,或不在同一页"),
|
||||
"ocr_required": (422, "这份文件大部分页面没有文字(可能是扫描件),扫描件识别还没有上线"),
|
||||
"ocr_required": (422, "这份文件大部分页面识别不出文字(可能是很模糊的扫描件),请换一份清晰的文件"),
|
||||
"rate_limited": (429, "操作太频繁,请稍后再试"),
|
||||
"ocr_unavailable": (503, "扫描件识别服务暂时不可用,可以先手动填写"),
|
||||
"internal_error": (500, "系统出错了,请稍后再试"),
|
||||
}
|
||||
|
||||
|
||||
@@ -71,5 +71,9 @@ def qualification_prefix(tenant_id: object, qualification_id: object) -> str:
|
||||
return f"t/{tenant_id}/q/{qualification_id}"
|
||||
|
||||
|
||||
def qualification_scan_prefix(tenant_id: object, scan_id: object) -> str:
|
||||
return f"t/{tenant_id}/qs/{scan_id}"
|
||||
|
||||
|
||||
def tenant_prefix(tenant_id: object) -> str:
|
||||
return f"t/{tenant_id}"
|
||||
|
||||
@@ -54,6 +54,7 @@ async def from_sentences(
|
||||
quote="".join(x.text for x in same),
|
||||
created_by_kind=kind,
|
||||
checks_passed=checks,
|
||||
ocr=any(x.source == "ocr" for x in same),
|
||||
)
|
||||
s.add(c)
|
||||
out.append(c)
|
||||
@@ -74,6 +75,7 @@ def to_api(c: Citation, label: str) -> dict[str, Any]:
|
||||
"quote": c.quote,
|
||||
"quads": c.quads,
|
||||
"created_by_kind": c.created_by_kind,
|
||||
"ocr": c.ocr,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -43,11 +43,11 @@ def detect(data: bytes, filename: str) -> Detected:
|
||||
return Detected(format=fmt, page_count=pages, extension_mismatch=ext not in EXTENSIONS.get(fmt, set()))
|
||||
|
||||
|
||||
ATTACHMENT_EXTENSIONS = {"pdf": {"pdf"}, "jpeg": {"jpg", "jpeg"}, "png": {"png"}}
|
||||
ATTACHMENT_EXTENSIONS = {"pdf": {"pdf"}, "jpeg": {"jpg", "jpeg"}, "png": {"png"}, "webp": {"webp"}}
|
||||
|
||||
|
||||
def detect_attachment(data: bytes, filename: str) -> Detected:
|
||||
"""资质附件(证书扫描件):PDF、JPG、PNG,同样按文件头认。"""
|
||||
"""资质附件(证书扫描件):PDF、JPG、PNG、WEBP(网上和手机里常见),同样按文件头认。"""
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
if data.startswith(b"%PDF"):
|
||||
fmt, pages = "pdf", _pdf_pages(data)
|
||||
@@ -55,8 +55,10 @@ def detect_attachment(data: bytes, filename: str) -> Detected:
|
||||
fmt, pages = "jpeg", 1
|
||||
elif data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
fmt, pages = "png", 1
|
||||
elif data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
fmt, pages = "webp", 1
|
||||
else:
|
||||
raise AppError("file_format_unsupported", "证书附件支持 PDF、JPG、PNG")
|
||||
raise AppError("file_format_unsupported", "证书附件支持 PDF、JPG、PNG、WEBP")
|
||||
return Detected(format=fmt, page_count=pages, extension_mismatch=ext not in ATTACHMENT_EXTENSIONS[fmt])
|
||||
|
||||
|
||||
|
||||
@@ -86,3 +86,4 @@ class Citation(Base):
|
||||
quote: Mapped[str]
|
||||
created_by_kind: Mapped[str]
|
||||
checks_passed: Mapped[TextArray]
|
||||
ocr: Mapped[bool] = mapped_column(default=False) # 原句是 OCR 识别出来的(扫描页、插图),可能有错字
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""解析流水线的前几个阶段:格式核对 → 转换 → 文字层 → OCR 补全(架构 §3.5)。
|
||||
|
||||
解析任务(招标文件)和检查批次(投标文件,m3-plan §2 prepare_bid)共用。阶段的进度、取消由调用方的 on_stage 负责。
|
||||
TODO(S5):OCR 还没接,没有文字层的页 text_source = "empty"。
|
||||
OCR(vision-plan §4.1):BD_OCR_PROVIDER=http 时,扫描页和大图交给 OCR 服务,识别结果并入文字层(pipeline/ocr.py);
|
||||
没接 OCR 时扫描页 text_source = "empty"。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,14 +20,21 @@ from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.pipeline import convert, ocr, text_layer
|
||||
from bid_doctor.document.pipeline.text_layer import Page
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
|
||||
# text_layer v2:页眉页脚带单独成句,句子编号和 v1 不同(同一份规范化 PDF 在 v1、v2 下的编号不能混用)
|
||||
# text_layer v5:每页记图片占比(扫描件判断),句子和编号和 v4 相同;投标分册下次检查时重新抽一遍
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v5", "sentence_split": "s3-v4"}
|
||||
OCR_REQUIRED_RATIO = 0.5 # 超过一半的页没有文字层:招标文件按「需要 OCR」失败
|
||||
# text_layer v6:记大图位置,OCR 句子接在原有句子后面(原有句子的编号不变)
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v6", "sentence_split": "s3-v4"}
|
||||
OCR_REQUIRED_RATIO = 0.5 # 识别之后仍有一半以上的页没有文字:招标文件按「识别不出文字」失败
|
||||
|
||||
|
||||
def profile() -> dict[str, str]:
|
||||
"""文字层的生成方式。OCR 开没开、用的哪一版,也算在内:先没开 OCR 抽的文字层,开了以后要重抽。"""
|
||||
return {**PROFILE, "ocr": ocr.PROFILE if ocr.enabled() else "none"}
|
||||
|
||||
|
||||
OnStage = Callable[[str], Awaitable[None]]
|
||||
|
||||
@@ -36,12 +44,14 @@ async def pages(
|
||||
) -> list[Page]:
|
||||
"""返回文档的文字层。reuse=True 时,已有同一版本的文字层就直接读出来(投标分册每个版本只解析一次)。
|
||||
|
||||
require_text:没有文字层的页太多时报 ocr_required(招标文件)。
|
||||
投标文件不报,扫描页由检查按「请人工核对」处理(C10)。"""
|
||||
require_text:识别之后仍没有文字的页太多时报 ocr_required(招标文件)。
|
||||
投标文件不报,没有文字的扫描页由检查按「请人工核对」处理(C10)。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == document_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
if reuse and doc.text_layer_key and (doc.pipeline_profile or {}).get("text_layer") == PROFILE["text_layer"]:
|
||||
current = profile()
|
||||
stored = doc.pipeline_profile or {}
|
||||
if reuse and doc.text_layer_key and all(stored.get(k) == current[k] for k in ("text_layer", "ocr")):
|
||||
raw = open_object(dk, storage().get(doc.text_layer_key), doc.text_layer_key)
|
||||
return [Page.from_json(p) for p in json.loads(raw)]
|
||||
|
||||
@@ -58,13 +68,21 @@ async def pages(
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
|
||||
d.pipeline_profile = PROFILE
|
||||
d.pipeline_profile = current
|
||||
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
|
||||
else:
|
||||
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
|
||||
|
||||
await on_stage("text_layer")
|
||||
result = text_layer.extract(str(doc.id), pdf)
|
||||
async with tenant_tx(ctx) as s:
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(result) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
|
||||
await on_stage("ocr")
|
||||
if ocr.enabled():
|
||||
await ocr.fill(str(doc.id), pdf, result)
|
||||
text_key = f"{doc_prefix(doc)}/text.json"
|
||||
storage().put(
|
||||
text_key, seal_object(dk, json.dumps([p.to_json() for p in result], ensure_ascii=False).encode(), text_key)
|
||||
@@ -81,18 +99,14 @@ async def pages(
|
||||
height_pt=p.height,
|
||||
rotation=p.rotation,
|
||||
text_source=p.text_source,
|
||||
char_count=p.char_count,
|
||||
char_count=p.char_count + p.ocr_chars,
|
||||
sentence_count=len(p.sentences),
|
||||
)
|
||||
)
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.text_layer_key, d.page_count, d.status = text_key, len(result), "ready"
|
||||
d.pipeline_profile = PROFILE
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(result) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
d.pipeline_profile = current
|
||||
|
||||
await on_stage("ocr")
|
||||
empty = sum(p.text_source == "empty" for p in result)
|
||||
if require_text and result and empty / len(result) > OCR_REQUIRED_RATIO:
|
||||
raise AppError("ocr_required")
|
||||
|
||||
296
server/src/bid_doctor/document/pipeline/ocr.py
Normal file
296
server/src/bid_doctor/document/pipeline/ocr.py
Normal file
@@ -0,0 +1,296 @@
|
||||
"""OCR 补全(架构 §3.5、vision-plan §4.1):扫描页整页识别,文字页里的大图只识别图片区域,识别结果切句后并入文字层。
|
||||
|
||||
- 渲染在 worker 里做(pypdfium2,200 dpi),图片交给 OCR 服务(services/ocr):它负责转正、红章附近按红色通道重读;
|
||||
- OCR 行换算回 PDF 坐标;和文字层已有文字重叠的行去掉(扫描页上方的文字说明、图上叠的文字);
|
||||
- 切句沿用文字层的规则:句末标点断句,没写到右边的行(段落末行)断句,同一行里离得远的几段(表格)各自成句;
|
||||
- OCR 句接在这一页原有句子后面编号,原有编号不变;句子带 source = "ocr" 和置信度(最低的那一行)。
|
||||
OCR 服务连不上、出错,按基础设施错误处理(OcrUnavailable,任务失败、退回额度),不悄悄跳过。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import itertools
|
||||
import math
|
||||
import re
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.document.pipeline.text_layer import BAND, CLOSERS, END, PAGE_NO, Page, Sentence, norm
|
||||
|
||||
TIMEOUT = httpx.Timeout(120, connect=5)
|
||||
PROFILE = "rapidocr-ppocrv6s-v1" # 识别方式变了(模型、切句)要改,投标分册下次检查时重新识别
|
||||
render_lock = threading.Lock() # pdfium 不是线程安全的:渲染串行,识别并发(资质识别也用这把锁)
|
||||
|
||||
|
||||
class OcrUnavailable(Exception):
|
||||
"""OCR 服务不可用:基础设施错误,退回额度。"""
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return get_settings().ocr_provider == "http"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Task:
|
||||
page: Page
|
||||
region: tuple[float, float, float, float] | None # None = 整页;否则是图片区域(PDF 坐标)
|
||||
|
||||
|
||||
def tasks(pages: list[Page]) -> list[Task]:
|
||||
out: list[Task] = []
|
||||
for p in pages:
|
||||
if p.scanned:
|
||||
out.append(Task(p, None))
|
||||
else:
|
||||
out.extend(Task(p, (f[0], f[1], f[2], f[3])) for f in p.figures)
|
||||
return out
|
||||
|
||||
|
||||
def _render(pdf: bytes, page_no: int, region: tuple[float, float, float, float] | None, dpi: int) -> bytes:
|
||||
with render_lock:
|
||||
doc = pdfium.PdfDocument(pdf)
|
||||
try:
|
||||
img = doc[page_no - 1].render(scale=dpi / 72).to_pil().convert("RGB") # pyright: ignore[reportArgumentType]
|
||||
finally:
|
||||
doc.close()
|
||||
if region:
|
||||
s = dpi / 72
|
||||
img = img.crop((int(region[0] * s), int(region[1] * s), int(region[2] * s), int(region[3] * s)))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", quality=92)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
async def recognize(client: httpx.AsyncClient, image: bytes, *, full_page: bool) -> dict[str, Any]:
|
||||
"""调 OCR 服务。测试里替换这个函数。"""
|
||||
try:
|
||||
r = await client.post(
|
||||
"/v1/ocr", params={"full_page": "true" if full_page else "false"}, content=image, timeout=TIMEOUT
|
||||
)
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPError as e:
|
||||
raise OcrUnavailable(str(e)) from e
|
||||
return r.json()
|
||||
|
||||
|
||||
async def fill(doc_id: str, pdf: bytes, pages: list[Page]) -> None:
|
||||
"""给扫描页、插图补上 OCR 句子(原地修改 pages)。"""
|
||||
todo = tasks(pages)
|
||||
if not todo:
|
||||
return
|
||||
s = get_settings()
|
||||
sem = asyncio.Semaphore(s.ocr_concurrency)
|
||||
|
||||
async def one(client: httpx.AsyncClient, t: Task) -> tuple[Task, dict[str, Any]]:
|
||||
async with sem:
|
||||
image = await asyncio.to_thread(_render, pdf, t.page.page_no, t.region, s.ocr_dpi)
|
||||
return t, await recognize(client, image, full_page=t.region is None)
|
||||
|
||||
async with httpx.AsyncClient(base_url=s.ocr_url) as client:
|
||||
results = await asyncio.gather(*(one(client, t) for t in todo))
|
||||
for t, res in results:
|
||||
origin = (t.region[0], t.region[1]) if t.region else (0.0, 0.0)
|
||||
add_sentences(doc_id, t.page, res, origin, s.ocr_dpi / 72, full_page=t.region is None)
|
||||
|
||||
|
||||
# —— 切句 ——
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class OcrLine:
|
||||
text: str
|
||||
score: float
|
||||
box: list[tuple[float, float]] # 四个点,PDF 坐标;顺序是文字方向上的左上、右上、右下、左下
|
||||
up: tuple[float, float, float, float] # 转正后的外接矩形(像素),排阅读顺序用
|
||||
|
||||
@property
|
||||
def height(self) -> float:
|
||||
return max(1.0, self.up[3] - self.up[1])
|
||||
|
||||
@property
|
||||
def vertical(self) -> bool:
|
||||
"""竖排的框(页边骑缝章上的字、竖排标签):高明显大于宽。"""
|
||||
return len(self.text) >= 2 and self.height > 1.5 * (self.up[2] - self.up[0])
|
||||
|
||||
|
||||
def _overlap(a: tuple[float, float, float, float], b: list[float]) -> float:
|
||||
w = min(a[2], b[2]) - max(a[0], b[0])
|
||||
h = min(a[3], b[3]) - max(a[1], b[1])
|
||||
return w * h if w > 0 and h > 0 else 0.0
|
||||
|
||||
|
||||
def _bbox(pts: list[tuple[float, float]]) -> tuple[float, float, float, float]:
|
||||
xs, ys = [p[0] for p in pts], [p[1] for p in pts]
|
||||
return min(xs), min(ys), max(xs), max(ys)
|
||||
|
||||
|
||||
STAMP_TEXT_SCORE = 0.9 # 印章范围里置信度低于这个的行,多半是章上环形排列的字(「京长卿」「X」)
|
||||
|
||||
|
||||
def _skew(box: list[tuple[float, float]]) -> float:
|
||||
"""行偏离水平或竖直的角度(度):章上沿圆弧排的字是斜的。横放的页在原图里是竖行,按离最近的坐标轴算。"""
|
||||
(x0, y0), (x1, y1) = box[0], box[1]
|
||||
a = abs(math.degrees(math.atan2(y1 - y0, x1 - x0))) % 90
|
||||
return min(a, 90 - a)
|
||||
|
||||
|
||||
def _lines(page: Page, res: dict[str, Any], origin: tuple[float, float], scale: float) -> list[OcrLine]:
|
||||
taken = [q for x in page.sentences if x.source == "text" for q in x.quads]
|
||||
stamps = [
|
||||
[origin[0] + v / scale if i % 2 == 0 else origin[1] + v / scale for i, v in enumerate(b)]
|
||||
for b in res.get("stamps") or []
|
||||
]
|
||||
out: list[OcrLine] = []
|
||||
for ln in res.get("lines") or []:
|
||||
text = str(ln["text"]).strip()
|
||||
if not text:
|
||||
continue
|
||||
box = [(origin[0] + p[0] / scale, origin[1] + p[1] / scale) for p in ln["box"]]
|
||||
bb = _bbox(box)
|
||||
area = max(1e-6, (bb[2] - bb[0]) * (bb[3] - bb[1]))
|
||||
if sum(_overlap(bb, q) for q in taken) / area > 0.5:
|
||||
continue # 文字层里已经有了
|
||||
if any(_overlap(bb, st) / area > 0.8 for st in stamps) and (
|
||||
float(ln["score"]) < STAMP_TEXT_SCORE or len(text) <= 3 or _skew(box) > 8
|
||||
):
|
||||
continue # 整行落在印章里,又短、又斜或认不准:章上沿圆弧排的字。正文里被章压住的值会伸出章外,不受影响
|
||||
out.append(OcrLine(text, float(ln["score"]), box, tuple(ln["upright"]))) # type: ignore[arg-type]
|
||||
return out
|
||||
|
||||
|
||||
def _median(xs: list[float]) -> float:
|
||||
xs = sorted(xs)
|
||||
return xs[len(xs) // 2] if xs else 1.0
|
||||
|
||||
|
||||
def _rows(lines: list[OcrLine]) -> list[list[OcrLine]]:
|
||||
"""按转正后的坐标排成行:竖直中心相差不到半个行高的算同一行,行内从左到右。
|
||||
行高取这一行的中位数和新行自己的较小值;特别高的竖框(页边骑缝章上的字、竖排字)单独成行:
|
||||
国动 p104 页边一个 197 像素高的「秀20118」把 6 行正文并成了一行。"""
|
||||
rows: list[list[OcrLine]] = []
|
||||
for ln in sorted(lines, key=lambda x: (x.up[1] + x.up[3]) / 2):
|
||||
cy = (ln.up[1] + ln.up[3]) / 2
|
||||
last = rows[-1] if rows else None
|
||||
if last and not ln.vertical and not any(x.vertical for x in last):
|
||||
ry = sum((x.up[1] + x.up[3]) / 2 for x in last) / len(last)
|
||||
if abs(cy - ry) < 0.5 * min(ln.height, _median([x.height for x in last])):
|
||||
last.append(ln)
|
||||
continue
|
||||
rows.append([ln])
|
||||
return [sorted(r, key=lambda x: x.up[0]) for r in rows]
|
||||
|
||||
|
||||
Piece = tuple[OcrLine, int, int] # 一行里的 [起, 止) 字符
|
||||
|
||||
|
||||
def _fragments(rows: list[list[OcrLine]], *, full_page: bool) -> list[tuple[list[OcrLine], bool]]:
|
||||
"""行片段:同一行里挨得近的(「甲方:」和后面的值)合成一段,离得远的(表格各格)分开。
|
||||
返回 (片段里的行, 这一段结束时句子是否必须结束)。插图(架构图、界面截图)里的字是一个个标签,每行各自成句。"""
|
||||
if not full_page:
|
||||
return [([x], True) for r in rows for x in r]
|
||||
normal = [x for r in rows for x in r if not x.vertical]
|
||||
typical = _median([x.height for x in normal])
|
||||
right = max((x.up[2] for x in normal), default=0.0)
|
||||
out: list[tuple[list[OcrLine], bool]] = []
|
||||
for i, row in enumerate(rows):
|
||||
if row[0].vertical:
|
||||
out.append((row, True))
|
||||
continue
|
||||
below = next((r for r in rows[i + 1 :] if not r[0].vertical), None)
|
||||
# 和下一行之间空得多(封面、表单的一行行填写项),不是同一段:国动 p135 封面的甲方、乙方、签订日期连成了一句
|
||||
apart = below is None or min(x.up[1] for x in below) - max(x.up[3] for x in row) > 0.6 * typical
|
||||
frags: list[list[OcrLine]] = [[row[0]]]
|
||||
for a, b in itertools.pairwise(row):
|
||||
if b.up[0] - a.up[2] > 1.5 * max(a.height, b.height):
|
||||
frags.append([b])
|
||||
else:
|
||||
frags[-1].append(b)
|
||||
table_like = len(frags) > 1
|
||||
for f in frags:
|
||||
last = f[-1]
|
||||
short = last.up[2] < right - 2 * typical # 没写到右边:段落末行(按正文行高算,大标题的框高不算)
|
||||
out.append((f, table_like or short or apart))
|
||||
return out
|
||||
|
||||
|
||||
HEADING = re.compile(r"^(第[一二三四五六七八九十百\d]+[章节条]|[一二三四五六七八九十]+[、..]|\d+(\.\d+)*[、..]\D)")
|
||||
|
||||
|
||||
def _sentences(frags: list[tuple[list[OcrLine], bool]]) -> list[list[Piece]]:
|
||||
groups: list[list[Piece]] = []
|
||||
cur: list[Piece] = []
|
||||
for i, (lines, must_end) in enumerate(frags):
|
||||
if cur and (HEADING.match(lines[0].text) or lines[0].vertical):
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
for ln in lines:
|
||||
start, ended = 0, False
|
||||
for k, ch in enumerate(ln.text):
|
||||
if ended and ch not in CLOSERS and ch not in END:
|
||||
cur.append((ln, start, k))
|
||||
groups.append(cur)
|
||||
cur, start, ended = [], k, False
|
||||
if ch in END:
|
||||
ended = True
|
||||
if start < len(ln.text):
|
||||
cur.append((ln, start, len(ln.text)))
|
||||
if ended:
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
nxt = frags[i + 1][0][0] if i + 1 < len(frags) else None
|
||||
if cur and (must_end or nxt is None or HEADING.match(nxt.text)):
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
if cur:
|
||||
groups.append(cur)
|
||||
return [g for g in groups if any(ln.text[a:b].strip() for ln, a, b in g)]
|
||||
|
||||
|
||||
def _quad(ln: OcrLine, a: int, b: int) -> list[float]:
|
||||
"""一行里第 a~b 个字的位置:沿文字方向按字数比例插值(OCR 只给整行的框)。"""
|
||||
n = max(1, len(ln.text))
|
||||
p0, p1, p2, p3 = ln.box
|
||||
|
||||
def at(u: tuple[float, float], v: tuple[float, float], t: float) -> tuple[float, float]:
|
||||
return (u[0] + (v[0] - u[0]) * t, u[1] + (v[1] - u[1]) * t)
|
||||
|
||||
t0, t1 = a / n, b / n
|
||||
x0, y0, x1, y1 = _bbox([at(p0, p1, t0), at(p0, p1, t1), at(p3, p2, t0), at(p3, p2, t1)])
|
||||
return [round(x0, 2), round(y0, 2), round(x1, 2), round(y1, 2)]
|
||||
|
||||
|
||||
def add_sentences(
|
||||
doc_id: str, page: Page, res: dict[str, Any], origin: tuple[float, float], scale: float, *, full_page: bool
|
||||
) -> None:
|
||||
lines = _lines(page, res, origin, scale)
|
||||
if not lines:
|
||||
return
|
||||
seq = max((x.seq for x in page.sentences), default=0)
|
||||
added = 0
|
||||
for group in _sentences(_fragments(_rows(lines), full_page=full_page)):
|
||||
text = "".join(ln.text[a:b] for ln, a, b in group).strip()
|
||||
quads = [_quad(ln, a, b) for ln, a, b in group]
|
||||
top, bottom = min(q[1] for q in quads), max(q[3] for q in quads)
|
||||
band = bottom <= page.height * BAND or top >= page.height * (1 - BAND)
|
||||
seq += 1
|
||||
page.sentences.append(
|
||||
Sentence(
|
||||
key=f"{doc_id}:p{page.page_no}:s{seq}",
|
||||
page_no=page.page_no,
|
||||
seq=seq,
|
||||
text=text,
|
||||
quads=quads,
|
||||
location="header_footer" if full_page and band and PAGE_NO.match(norm(text)) else "body",
|
||||
source="ocr",
|
||||
conf=round(min(ln.score for ln, _, _ in group), 3),
|
||||
)
|
||||
)
|
||||
added += len(norm(text))
|
||||
page.ocr_chars += added
|
||||
@@ -3,7 +3,7 @@
|
||||
- 每个阶段结束写检查点;重跑时已完成的阶段直接跳过(规范化 PDF 只生成一次,本来也不能重写)。
|
||||
- 阶段之间检查取消标记。
|
||||
- 结局由错误类别决定扣不扣点:基础设施错误、输入问题、取消 → 退回;成功 → 结算(架构 §3.6 的错误分类表)。
|
||||
TODO(结构识别):章节、表格、分包识别还没做,现在只抽实质性条款。TODO(S5):OCR 还没接,没有文字层的页直接报「需要 OCR」。
|
||||
TODO(结构识别):章节、表格、分包识别还没做。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -26,6 +26,7 @@ from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.document.models import ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import normalize, text_layer
|
||||
from bid_doctor.document.pipeline.ocr import OcrUnavailable
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import extracted as project_extracted
|
||||
from bid_doctor.project import service as projects
|
||||
@@ -61,6 +62,9 @@ async def run(tenant_id: UUID, job_id: UUID) -> None:
|
||||
await _finish(ctx, job_id, "cancelled", "cancelled", None)
|
||||
except AppError as e:
|
||||
await _finish(ctx, job_id, "failed", "input", e.code)
|
||||
except OcrUnavailable as e:
|
||||
log.warning("parse job %s ocr unavailable: %s", job_id, e)
|
||||
await _finish(ctx, job_id, "failed", "infra", "ocr_unavailable")
|
||||
except (TransientError, httpx.HTTPError) as e:
|
||||
log.warning("parse job %s infra error: %s", job_id, e)
|
||||
code = "model_unavailable" if isinstance(e, TransientError) else "converter_unavailable"
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
4. 句子不跨页。编号 <文档>:p<页>:s<页内序号>;规范化 PDF 不再重新生成,所以编号稳定。
|
||||
格式:斜体看字体名或字形倾斜;下划线看字下方的横线;加粗看字体名(S2:斜体且下划线按字符 100% 一致)。
|
||||
页眉页脚带里的页码和重复行单独成句(location = header_footer)。
|
||||
扫描页、插图里的文字由 OCR 补进来(pipeline/ocr.py):句子带 source = "ocr" 和置信度,接在这一页原有句子后面编号。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -73,6 +74,8 @@ class Sentence:
|
||||
location: str = "body"
|
||||
open_end: bool = False # 写到页面末尾还没结束(没有句末标点、末行写满),可能接续到下一页
|
||||
cell: list[float] | None = None # 在有框线的表格里:所在单元格 [x0, top, x1, bottom](查标按单元格分行)
|
||||
source: str = "text" # text:PDF 文字层;ocr:扫描页、插图上识别出的文字(可能有错字)
|
||||
conf: float | None = None # OCR 置信度(这一句用到的各行里最低的)
|
||||
|
||||
@property
|
||||
def marked(self) -> str:
|
||||
@@ -98,6 +101,7 @@ class Sentence:
|
||||
"location": self.location,
|
||||
"open_end": self.open_end,
|
||||
**({"cell": self.cell} if self.cell else {}),
|
||||
**({"source": self.source, "conf": self.conf} if self.source != "text" else {}),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -112,6 +116,8 @@ class Sentence:
|
||||
location=d["location"],
|
||||
open_end=d.get("open_end", False),
|
||||
cell=d.get("cell"),
|
||||
source=d.get("source", "text"),
|
||||
conf=d.get("conf"),
|
||||
)
|
||||
|
||||
|
||||
@@ -124,17 +130,24 @@ class Page:
|
||||
char_count: int
|
||||
sentences: list[Sentence]
|
||||
image_cover: float = 0.0 # 图片占页面的比例(text_layer v5 起有;更早的文字层读出来是 0)
|
||||
figures: list[list[float]] = field(default_factory=list[list[float]]) # 大图的位置 [x0, top, x1, bottom](v6 起)
|
||||
ocr_chars: int = 0 # OCR 补进来的字数(v6 起)
|
||||
|
||||
@property
|
||||
def scanned(self) -> bool:
|
||||
"""扫描件页。扫描件常带一行文字说明(「合同扫描件第 2 页(共 15 页)」「2.1 营业执照」),只看字数会漏掉
|
||||
(国动真实投标文件 60 多页,check-eval §14):图片占两成以上、不到 60 字,或占四成半以上、不到 120 字,
|
||||
也算扫描件;带插图的正文页字多,不算。"""
|
||||
n, cover = self.char_count, self.image_cover
|
||||
return n < 20 or (cover >= 0.2 and n < 60) or (cover >= 0.45 and n < 120)
|
||||
|
||||
@property
|
||||
def text_source(self) -> str:
|
||||
"""扫描件页算「没有文字层」(empty)。扫描件常带一行文字说明
|
||||
(「合同扫描件第 2 页(共 15 页)」「2.1 营业执照」),只看字数会漏掉(国动真实投标文件 60 多页,
|
||||
check-eval §14):图片占两成以上、不到 60 字,或占四成半以上、不到 120 字,也算扫描件;
|
||||
带插图的正文页字多,不算。"""
|
||||
n, cover = self.char_count, self.image_cover
|
||||
if n < 20 or (cover >= 0.2 and n < 60) or (cover >= 0.45 and n < 120):
|
||||
return "empty"
|
||||
return "text"
|
||||
"""text:文字层;ocr:扫描页,文字是 OCR 识别出来的;mixed:文字页,插图里的字由 OCR 补;
|
||||
empty:扫描页且识别不出字。"""
|
||||
if self.scanned:
|
||||
return "ocr" if self.ocr_chars >= 10 else "empty"
|
||||
return "mixed" if self.ocr_chars else "text"
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
@@ -144,6 +157,8 @@ class Page:
|
||||
"rotation": self.rotation,
|
||||
"char_count": self.char_count,
|
||||
"image_cover": self.image_cover,
|
||||
**({"figures": self.figures} if self.figures else {}),
|
||||
**({"ocr_chars": self.ocr_chars} if self.ocr_chars else {}),
|
||||
"sentences": [s.to_json() for s in self.sentences],
|
||||
}
|
||||
|
||||
@@ -157,6 +172,8 @@ class Page:
|
||||
char_count=d["char_count"],
|
||||
sentences=[Sentence.from_json(d["page_no"], s) for s in d["sentences"]],
|
||||
image_cover=d.get("image_cover", 0.0),
|
||||
figures=d.get("figures", []),
|
||||
ocr_chars=d.get("ocr_chars", 0),
|
||||
)
|
||||
|
||||
|
||||
@@ -348,21 +365,40 @@ def _page(doc_id: str, page: Any, frags: list[list[dict[str, Any]]], infos: list
|
||||
char_count=len(chars),
|
||||
sentences=sentences,
|
||||
image_cover=_image_cover(page),
|
||||
figures=_figures(page),
|
||||
)
|
||||
|
||||
|
||||
def _clipped(page: Any) -> list[tuple[float, float, float, float]]:
|
||||
w, h = float(page.width), float(page.height)
|
||||
out = []
|
||||
for im in page.images:
|
||||
x0, top = max(0.0, float(im["x0"])), max(0.0, float(im["top"]))
|
||||
x1, bottom = min(w, float(im["x1"])), min(h, float(im["bottom"]))
|
||||
if x1 > x0 and bottom > top:
|
||||
out.append((x0, top, x1, bottom))
|
||||
return out
|
||||
|
||||
|
||||
def _image_cover(page: Any) -> float:
|
||||
"""图片占页面面积的比例(重叠的不去重,封顶 1),判断扫描件用。"""
|
||||
area = float(page.width) * float(page.height)
|
||||
total = 0.0
|
||||
for im in page.images:
|
||||
w = min(float(page.width), float(im["x1"])) - max(0.0, float(im["x0"]))
|
||||
h = min(float(page.height), float(im["bottom"])) - max(0.0, float(im["top"]))
|
||||
if w > 0 and h > 0:
|
||||
total += w * h
|
||||
total = sum((x1 - x0) * (b - t) for x0, t, x1, b in _clipped(page))
|
||||
return round(min(total / area, 1.0), 3) if area else 0.0
|
||||
|
||||
|
||||
FIGURE_MIN = 0.05 # 占页面 5% 以上的图片才做 OCR:更小的是 logo、图标、二维码
|
||||
|
||||
|
||||
def _figures(page: Any) -> list[list[float]]:
|
||||
area = float(page.width) * float(page.height)
|
||||
return [
|
||||
[round(v, 2) for v in box]
|
||||
for box in _clipped(page)
|
||||
if (box[2] - box[0]) * (box[3] - box[1]) >= FIGURE_MIN * area
|
||||
]
|
||||
|
||||
|
||||
def extract(doc_id: str, pdf: bytes) -> list[Page]:
|
||||
with pdfplumber.open(io.BytesIO(pdf)) as doc:
|
||||
pages = [(p, *_fragments([c for c in p.chars if str(c["text"]).strip()], _cells(p))) for p in doc.pages]
|
||||
|
||||
@@ -202,9 +202,10 @@ async def get_job(s: AsyncSession, job_id: UUID) -> ParseJob:
|
||||
ERROR_MESSAGES = {
|
||||
"file_corrupted": "这份文件无法转换,可能已加密或损坏",
|
||||
"file_encrypted": "文件已加密,请上传未加密的版本",
|
||||
"ocr_required": "这份文件大部分页面没有文字层(可能是扫描件),扫描件识别还没有上线",
|
||||
"ocr_required": "这份文件大部分页面识别不出文字(可能是很模糊的扫描件),请换一份清晰的文件",
|
||||
"plan_limit_exceeded": "页数超出套餐上限",
|
||||
"converter_unavailable": "文档转换服务暂时不可用",
|
||||
"ocr_unavailable": "扫描件识别服务暂时不可用",
|
||||
"model_unavailable": "模型服务暂时不可用",
|
||||
"internal_error": "系统出错了",
|
||||
}
|
||||
|
||||
@@ -19,22 +19,34 @@ router = APIRouter(tags=["qualifications"])
|
||||
Kind = Literal["certificate", "system_cert", "software_copyright", "other"]
|
||||
|
||||
|
||||
class ScanRef(BaseModel):
|
||||
"""从识别结果保存:识别任务编号和第几张证书(从 0 数)。"""
|
||||
|
||||
scan_id: UUID
|
||||
index: int = Field(ge=0)
|
||||
|
||||
|
||||
class QualificationIn(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
kind: Kind
|
||||
level: str | None = Field(default=None, max_length=40)
|
||||
cert_no: str | None = Field(default=None, max_length=80)
|
||||
issuer: str | None = Field(default=None, max_length=120)
|
||||
holder: str | None = Field(default=None, max_length=120)
|
||||
valid_from: date | None = None
|
||||
valid_to: date | None = None # 空表示长期有效
|
||||
scan: ScanRef | None = None
|
||||
|
||||
def data(self) -> dict[str, Any]:
|
||||
d = self.model_dump()
|
||||
for k in ("level", "cert_no", "issuer"):
|
||||
d = self.model_dump(exclude={"scan"})
|
||||
for k in ("level", "cert_no", "issuer", "holder"):
|
||||
d[k] = (d[k] or "").strip() or None
|
||||
d["name"] = d["name"].strip()
|
||||
return d
|
||||
|
||||
def scan_ref(self) -> tuple[UUID, int] | None:
|
||||
return (self.scan.scan_id, self.scan.index) if self.scan else None
|
||||
|
||||
|
||||
class AttachIn(BaseModel):
|
||||
upload_id: UUID
|
||||
@@ -51,13 +63,13 @@ async def list_qualifications(
|
||||
@router.post("/qualifications", status_code=201)
|
||||
async def create_qualification(body: QualificationIn, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.to_api(s, await service.create(s, ctx, body.data()))
|
||||
return await service.to_api(s, await service.create(s, ctx, body.data(), body.scan_ref()))
|
||||
|
||||
|
||||
@router.put("/qualifications/{qualification_id}")
|
||||
async def update_qualification(qualification_id: UUID, body: QualificationIn, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.to_api(s, await service.update(s, ctx, qualification_id, body.data()))
|
||||
return await service.to_api(s, await service.update(s, ctx, qualification_id, body.data(), body.scan_ref()))
|
||||
|
||||
|
||||
@router.delete("/qualifications/{qualification_id}", status_code=204)
|
||||
@@ -77,12 +89,58 @@ async def get_qualification_file_url(qualification_id: UUID, file_id: UUID, ctx:
|
||||
return await service.file_url(s, ctx, qualification_id, file_id)
|
||||
|
||||
|
||||
@router.get("/qualifications/{qualification_id}/files/{file_id}/previews")
|
||||
async def get_qualification_file_previews(qualification_id: UUID, file_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
"""附件的预览图(图片一张,PDF 每页一张),给详情和大图浏览用,不在新标签页打开原件。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.file_previews(s, ctx, qualification_id, file_id)
|
||||
|
||||
|
||||
@router.get("/qualification-file-previews/{tenant_id}/{file_id}/{page}", include_in_schema=False)
|
||||
async def qualification_file_preview(tenant_id: UUID, file_id: UUID, page: int, exp: int, sig: str) -> Response:
|
||||
if not verify_download(f"{tenant_id}:{file_id}:{page}:qualification_preview", exp, sig):
|
||||
raise AppError("not_found")
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
data = await service.preview_image(s, ctx, file_id, page)
|
||||
return Response(data, media_type="image/jpeg", headers={"Cache-Control": "private, max-age=300"})
|
||||
|
||||
|
||||
@router.delete("/qualifications/{qualification_id}/files/{file_id}", status_code=204)
|
||||
async def delete_qualification_file(qualification_id: UUID, file_id: UUID, ctx: Ctx) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
await service.delete_file(s, ctx, qualification_id, file_id)
|
||||
|
||||
|
||||
@router.post("/qualification-scans", status_code=201)
|
||||
async def create_qualification_scan(body: AttachIn, ctx: Ctx) -> dict[str, Any]:
|
||||
"""上传证书扫描件识别(vision-plan 第 2 步):后台识别,前端轮询 GET 拿结果。不扣额度。"""
|
||||
return await service.create_scan(ctx, body.upload_id)
|
||||
|
||||
|
||||
@router.get("/qualification-scans/{scan_id}")
|
||||
async def get_qualification_scan(scan_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.get_scan(s, ctx, scan_id)
|
||||
|
||||
|
||||
@router.delete("/qualification-scans/{scan_id}", status_code=204)
|
||||
async def delete_qualification_scan(scan_id: UUID, ctx: Ctx) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
await service.delete_scan(s, ctx, scan_id)
|
||||
|
||||
|
||||
@router.get("/qualification-scan-pages/{tenant_id}/{scan_id}/{page}", include_in_schema=False)
|
||||
async def qualification_scan_page(tenant_id: UUID, scan_id: UUID, page: int, exp: int, sig: str) -> Response:
|
||||
"""识别时的页面图(签名地址,<img> 用)。和识别用的是同一张图,高亮框的坐标对得上。"""
|
||||
if not verify_download(f"{tenant_id}:{scan_id}:{page}:qualification_scan", exp, sig):
|
||||
raise AppError("not_found")
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
data = await service.scan_page(s, ctx, scan_id, page)
|
||||
return Response(data, media_type="image/jpeg", headers={"Cache-Control": "private, max-age=300"})
|
||||
|
||||
|
||||
@router.get("/qualification-files/{tenant_id}/{file_id}", include_in_schema=False)
|
||||
async def download(tenant_id: UUID, file_id: UUID, exp: int, sig: str) -> Response:
|
||||
"""签名下载(本地磁盘驱动用)。不带鉴权头,靠短时签名。"""
|
||||
|
||||
@@ -2,9 +2,10 @@ from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy.orm import Mapped
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, UuidCol
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, Json, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
@@ -21,6 +22,8 @@ class Qualification(Base):
|
||||
issuer: Mapped[str | None]
|
||||
valid_from: Mapped[date | None]
|
||||
valid_to: Mapped[date | None] # 空 = 长期有效
|
||||
holder: Mapped[str | None] # 证书主体:企业名、著作权人(vision-plan 第 2 步)
|
||||
ai_values: Mapped[Json | None] # 从识别结果保存时的识别值快照:识别值和用户最终值都留痕
|
||||
created_by: Mapped[UuidCol]
|
||||
created_at: Mapped[DbTs]
|
||||
updated_at: Mapped[DbTs]
|
||||
@@ -38,3 +41,23 @@ class QualificationFile(Base):
|
||||
sha256: Mapped[Bytes]
|
||||
size_bytes: Mapped[int]
|
||||
created_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class QualificationScan(Base):
|
||||
"""一次证书识别(vision-plan 第 2 步):原件按企业级密钥加密;result 是识别出的证书和字段。
|
||||
没保存的 24 小时后删除。"""
|
||||
|
||||
__tablename__ = "qualification_scan"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
filename: Mapped[str]
|
||||
object_key: Mapped[str]
|
||||
sha256: Mapped[Bytes]
|
||||
size_bytes: Mapped[int]
|
||||
media_type: Mapped[str]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
error_code: Mapped[str | None]
|
||||
result: Mapped[Json | None]
|
||||
created_by: Mapped[UuidCol]
|
||||
created_at: Mapped[DbTs]
|
||||
finished_at: Mapped[Ts | None]
|
||||
|
||||
309
server/src/bid_doctor/qualification/recognize.py
Normal file
309
server/src/bid_doctor/qualification/recognize.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""资质库上传识别(vision-plan 第 2 步,tech-validation §7.3):证书扫描件 → OCR → 视觉模型 → 校验 → 给用户核对。
|
||||
|
||||
- 每页图片先交给 OCR 服务(转正、红章重读),再把转正后的图 + 带编号的 OCR 行交给视觉模型(vision.cert_fields),
|
||||
按证书逐张输出类别和字段,每个字段引用 OCR 行;
|
||||
- 校验(vision-plan §2):字段值要在所引各行拼接后的文字里(日期按年月日比);所引行的 OCR 置信度低于 0.9 的
|
||||
(马赛克、模糊)标「识别不确定」;对不上的标「不一致」。这些都不自动填,只作建议给用户看(红线 5:不编造);
|
||||
- 不推算有效期:证书只写了「五年」时给一个按发证日期推算的建议,用户点了才填;
|
||||
- 不扣额度(决策 VS5),每个企业每天限 SCAN_DAILY_LIMIT 次。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
import pypdfium2 as pdfium
|
||||
from PIL import Image, ImageOps
|
||||
from sqlalchemy import select
|
||||
|
||||
from bid_doctor.ai import service as ai
|
||||
from bid_doctor.ai.providers.base import TransientError
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import open_object
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.pipeline import ocr
|
||||
from bid_doctor.identity.models import Tenant
|
||||
from bid_doctor.qualification.models import Qualification, QualificationScan
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
PROMPT = ("vision.cert_fields", 1)
|
||||
MAX_PAGES = 10 # PDF 只识别前 10 页(证书汇编再长就分开传)
|
||||
MAX_SIDE = 4000 # 图片长边超过这个先缩小(手机原图 4000~8000 像素,OCR 用不着)
|
||||
LLM_SIDE = 1600 # 给视觉模型的图长边(S5 初测用的尺寸)
|
||||
LOW_SCORE = 0.9 # 所引 OCR 行的置信度低于这个:识别不确定(马赛克、模糊,tech-validation §7.3)
|
||||
DATE_FIELDS = ("issued_on", "valid_from", "valid_to")
|
||||
# 识别出的类别 → 资质库的类型。人员证书、产品证书、营业执照不参与查标的资质匹配(qualification-plan §3 排除词),
|
||||
# 仍可保存,归「其他」
|
||||
KINDS = {
|
||||
"enterprise_qualification": "certificate",
|
||||
"system_cert": "system_cert",
|
||||
"software_copyright": "software_copyright",
|
||||
}
|
||||
|
||||
|
||||
# —— 图片 ——
|
||||
|
||||
|
||||
def page_images(data: bytes, media_type: str) -> list[Image.Image]:
|
||||
"""原件 → 每页一张图(RGB)。图片按 EXIF 转正(手机拍的照片);PDF 按 OCR 的分辨率渲染前 MAX_PAGES 页。
|
||||
识别和界面预览都用它,框的坐标才对得上。"""
|
||||
if media_type == "application/pdf":
|
||||
with ocr.render_lock:
|
||||
doc = pdfium.PdfDocument(data)
|
||||
try:
|
||||
scale = get_settings().ocr_dpi / 72
|
||||
return [
|
||||
doc[i].render(scale=scale).to_pil().convert("RGB") # pyright: ignore[reportArgumentType]
|
||||
for i in range(min(len(doc), MAX_PAGES))
|
||||
]
|
||||
finally:
|
||||
doc.close()
|
||||
img = ImageOps.exif_transpose(Image.open(io.BytesIO(data))).convert("RGB")
|
||||
if max(img.size) > MAX_SIDE:
|
||||
s = MAX_SIDE / max(img.size)
|
||||
img = img.resize((round(img.width * s), round(img.height * s)))
|
||||
return [img]
|
||||
|
||||
|
||||
def jpeg(img: Image.Image, long_side: int | None = None) -> bytes:
|
||||
if long_side and max(img.size) > long_side:
|
||||
s = long_side / max(img.size)
|
||||
img = img.resize((round(img.width * s), round(img.height * s)))
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", quality=90)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# —— 校验 ——
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
s = "".join(str(s).split())
|
||||
for a, b in (("(", "("), (")", ")"), (":", ":"), (",", ","), ("[", "["), ("]", "]"), ("—", "-"), ("-", "-")):
|
||||
s = s.replace(a, b)
|
||||
return s.upper()
|
||||
|
||||
|
||||
def parse_day(s: str) -> date | None:
|
||||
"""「2019年11月09日」「2025-8-27」「2024.02.19」→ 日期;年月日要齐。"""
|
||||
nums = [int(x) for x in re.findall(r"\d+", str(s))]
|
||||
if len(nums) < 3 or not 1900 < nums[0] < 2100:
|
||||
return None
|
||||
try:
|
||||
return date(nums[0], nums[1], nums[2])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
NOT_NAMES = {"甲方", "乙方", "丙方", "本公司", "本单位", "申请人", "持证人", "申请企业", "委托人", "受托人"}
|
||||
|
||||
|
||||
def check_field(field: str, value: str, refs: list[str], lines: dict[str, dict[str, Any]], category: str) -> str:
|
||||
"""ok:值在所引各行拼接的文字里;date_ok:日期的年月日和所引行一致;low_conf:所引行置信度低(打码、模糊);
|
||||
mismatch:所引行对不上(OCR 认错、或模型读错);bad_line:引了不存在的行;bad_format:值的格式不像这个字段。"""
|
||||
if not refs or any(r not in lines for r in refs):
|
||||
return "bad_line"
|
||||
v = norm(value)
|
||||
joined = norm("".join(lines[r]["text"] for r in refs))
|
||||
if field in DATE_FIELDS:
|
||||
d = parse_day(v)
|
||||
if d is None:
|
||||
return "bad_format"
|
||||
status = "ok" if v in joined else "date_ok" if d == parse_day(joined) else "mismatch"
|
||||
else:
|
||||
if not v:
|
||||
return "bad_format"
|
||||
if field == "holder" and v in NOT_NAMES:
|
||||
return "bad_format" # 「甲方」「本公司」不是名称(S5 初测:用人单位 = 甲方)
|
||||
if field in ("cert_no", "reg_no") and not re.search(r"[0-9A-Z]", v):
|
||||
return "bad_format"
|
||||
status = "ok" if v in joined else "mismatch"
|
||||
if status != "mismatch" and min(lines[r]["score"] for r in refs) < LOW_SCORE:
|
||||
return "low_conf"
|
||||
return status
|
||||
|
||||
|
||||
CN_NUM = {"一": 1, "二": 2, "两": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9, "十": 10}
|
||||
|
||||
|
||||
def validity_years(text: str) -> int | None:
|
||||
m = re.search(r"(\d+|[一二两三四五六七八九十])\s*年", text)
|
||||
if not m:
|
||||
return None
|
||||
g = m.group(1)
|
||||
return int(g) if g.isdigit() else CN_NUM[g]
|
||||
|
||||
|
||||
def suggest_valid_to(issued: date | None, text: str | None) -> date | None:
|
||||
"""证书只写了有效期长度(「有效期:五年」):发证日期 + N 年 − 1 天。只作建议,用户点了才填。"""
|
||||
years = validity_years(text or "")
|
||||
if not issued or not years:
|
||||
return None
|
||||
try:
|
||||
end = issued.replace(year=issued.year + years)
|
||||
except ValueError: # 2 月 29 日
|
||||
end = issued.replace(year=issued.year + years, day=28)
|
||||
return end - timedelta(days=1)
|
||||
|
||||
|
||||
def _boxes(refs: list[str], lines: dict[str, dict[str, Any]]) -> list[list[float]]:
|
||||
out = []
|
||||
for r in refs:
|
||||
pts = lines[r]["box"]
|
||||
xs, ys = [p[0] for p in pts], [p[1] for p in pts]
|
||||
out.append([round(min(xs), 1), round(min(ys), 1), round(max(xs), 1), round(max(ys), 1)])
|
||||
return out
|
||||
|
||||
|
||||
def build(
|
||||
page: int, raw: dict[str, Any], lines: dict[str, dict[str, Any]], tenant_name: str, existing: list[Qualification]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""模型输出 → 给界面的证书列表:字段带校验结论和高亮框,加上类型、建议的名称、提醒。"""
|
||||
out = []
|
||||
for c in raw.get("certificates") or []:
|
||||
if not isinstance(c, dict):
|
||||
continue
|
||||
category = str(c.get("category") or "other")
|
||||
fields: dict[str, dict[str, Any]] = {}
|
||||
for f in c.get("fields") or []:
|
||||
if not isinstance(f, dict) or not f.get("field") or f.get("field") in fields:
|
||||
continue
|
||||
refs = f.get("lines") or []
|
||||
refs = [refs] if isinstance(refs, str) else [str(x) for x in refs]
|
||||
field, value = str(f["field"]), str(f.get("value") or "").strip()
|
||||
entry: dict[str, Any] = {"value": value, "check": check_field(field, value, refs, lines, category)}
|
||||
entry["boxes"] = _boxes([r for r in refs if r in lines], lines)
|
||||
if field in DATE_FIELDS and (d := parse_day(value)):
|
||||
entry["day"] = d.isoformat()
|
||||
fields[field] = entry
|
||||
masked = [str(x) for x in c.get("masked") or [] if str(x) not in fields]
|
||||
out.append(_card(page, category, fields, masked, tenant_name, existing))
|
||||
return out
|
||||
|
||||
|
||||
def _ok(fields: dict[str, dict[str, Any]], k: str) -> str | None:
|
||||
f = fields.get(k)
|
||||
return f["value"] if f and f["check"] in ("ok", "date_ok") else None
|
||||
|
||||
|
||||
def _card(
|
||||
page: int,
|
||||
category: str,
|
||||
fields: dict[str, dict[str, Any]],
|
||||
masked: list[str],
|
||||
tenant_name: str,
|
||||
existing: list[Qualification],
|
||||
) -> dict[str, Any]:
|
||||
notes: list[dict[str, Any]] = []
|
||||
if category in ("personnel", "product", "business_license"):
|
||||
notes.append({"code": category})
|
||||
holder = _ok(fields, "holder")
|
||||
if holder and category not in ("personnel",) and tenant_name and norm(holder) != norm(tenant_name):
|
||||
notes.append({"code": "holder_mismatch", "holder": holder, "tenant": tenant_name})
|
||||
# 名称:证书标题后面带上软件名、专业,免得同类证书叫一个名字(查标按名称认证书类别,标题要保留)
|
||||
title = (fields.get("name") or {}).get("value") or ""
|
||||
extra = (fields.get("software") or fields.get("specialty") or {}).get("value")
|
||||
name = f"{title}({extra})" if title and extra and extra not in title else title or extra or ""
|
||||
# 软著用登记号(查询、投标文件里写的都是它)
|
||||
no_key = "reg_no" if category == "software_copyright" and "reg_no" in fields else "cert_no"
|
||||
suggest: dict[str, str] = {}
|
||||
issued = parse_day(_ok(fields, "issued_on") or "")
|
||||
if "valid_to" not in fields and (to := suggest_valid_to(issued, (fields.get("validity_text") or {}).get("value"))):
|
||||
suggest["valid_to"] = to.isoformat()
|
||||
no = (fields.get(no_key) or {}).get("value")
|
||||
same = next((q for q in existing if no and q.cert_no and norm(q.cert_no) == norm(no)), None)
|
||||
return {
|
||||
"page": page,
|
||||
"category": category,
|
||||
"kind": KINDS.get(category, "other"),
|
||||
"name": name,
|
||||
"cert_no_field": no_key,
|
||||
"fields": fields,
|
||||
"masked": masked,
|
||||
"suggest": suggest,
|
||||
"notes": notes,
|
||||
"existing": {"id": str(same.id), "name": same.name} if same else None,
|
||||
}
|
||||
|
||||
|
||||
# —— 任务 ——
|
||||
|
||||
|
||||
async def run(tenant_id: UUID, scan_id: UUID) -> None:
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
scan = (await s.execute(select(QualificationScan).where(QualificationScan.id == scan_id))).scalar_one_or_none()
|
||||
if scan is None or scan.status not in ("queued", "running"):
|
||||
return
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=scan.created_by, actor_name="系统")
|
||||
scan.status = "running"
|
||||
try:
|
||||
result = await _recognize(ctx, scan_id)
|
||||
except AppError as e:
|
||||
await _finish(ctx, scan_id, "failed", e.code)
|
||||
except ocr.OcrUnavailable as e:
|
||||
log.warning("qualification scan %s ocr unavailable: %s", scan_id, e)
|
||||
await _finish(ctx, scan_id, "failed", "ocr_unavailable")
|
||||
except (TransientError, httpx.HTTPError) as e:
|
||||
log.warning("qualification scan %s model unavailable: %s", scan_id, e)
|
||||
await _finish(ctx, scan_id, "failed", "model_unavailable")
|
||||
except Exception:
|
||||
log.exception("qualification scan %s crashed", scan_id)
|
||||
await _finish(ctx, scan_id, "failed", "internal_error")
|
||||
else:
|
||||
await _finish(ctx, scan_id, "succeeded", None, result)
|
||||
|
||||
|
||||
async def _finish(
|
||||
ctx: TenantContext, scan_id: UUID, status: str, code: str | None, result: dict[str, Any] | None = None
|
||||
) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
scan = (await s.execute(select(QualificationScan).where(QualificationScan.id == scan_id))).scalar_one()
|
||||
scan.status, scan.error_code, scan.result, scan.finished_at = status, code, result, datetime.now(UTC)
|
||||
|
||||
|
||||
async def _recognize(ctx: TenantContext, scan_id: UUID) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
scan = (await s.execute(select(QualificationScan).where(QualificationScan.id == scan_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx)
|
||||
tenant = (await s.execute(select(Tenant).where(Tenant.id == ctx.tenant_id))).scalar_one()
|
||||
existing = list((await s.execute(select(Qualification))).scalars())
|
||||
data = open_object(dk, storage().get(scan.object_key), scan.object_key)
|
||||
images = await asyncio.to_thread(page_images, data, scan.media_type)
|
||||
pages: list[dict[str, Any]] = []
|
||||
certs: list[dict[str, Any]] = []
|
||||
async with httpx.AsyncClient(base_url=get_settings().ocr_url) as client:
|
||||
for no, img in enumerate(images, start=1):
|
||||
res = await ocr.recognize(client, await asyncio.to_thread(jpeg, img), full_page=True)
|
||||
rot = int(res.get("rotation") or 0)
|
||||
pages.append({"page": no, "width": img.width, "height": img.height, "rotation": rot})
|
||||
lines = {f"L{i + 1}": x for i, x in enumerate(res.get("lines") or [])}
|
||||
if not lines:
|
||||
continue
|
||||
upright = img.rotate(rot, expand=True) if rot else img
|
||||
text = "\n".join(f"{k} {v['text']}" for k, v in lines.items())
|
||||
try:
|
||||
raw = await ai.run_json(
|
||||
ctx,
|
||||
job_type="qualification_scan",
|
||||
job_id=scan_id,
|
||||
task=PROMPT[0],
|
||||
prompt_id=PROMPT[0],
|
||||
version=PROMPT[1],
|
||||
user=f"<数据区>\nOCR 文字行:\n{text}\n</数据区>",
|
||||
images=[await asyncio.to_thread(jpeg, upright, LLM_SIDE)],
|
||||
)
|
||||
except ai.OutputIncomplete as e: # 一页证书的输出不长,截断时已经解析出来的部分照用,字段照样逐个校验
|
||||
raw = e.data
|
||||
certs += build(no, raw, lines, tenant.name, existing)
|
||||
return {"pages": pages, "certificates": certs, "prompt": f"{PROMPT[0]} v{PROMPT[1]}"}
|
||||
@@ -3,38 +3,57 @@
|
||||
- 只有用户自己录入的资质(红线 5);不扣额度;增删改和附件操作都写审计;
|
||||
- 附件按企业级数据密钥加密(DM4),对象键 t/<tenant>/q/<资质>/<附件>;
|
||||
- 删除是硬删,连同附件对象一起删,不做撤销(QL6:撤销要软删除,和「彻底删除」冲突);
|
||||
- late_for:进行中的项目里,投标截止日晚于「有效期至」的项目,到期按每个项目的截止日判断,不按今天。
|
||||
- late_for:进行中的项目里,投标截止日晚于「有效期至」的项目,到期按每个项目的截止日判断,不按今天;
|
||||
- 上传识别(vision-plan 第 2 步):传证书扫描件 → 后台识别(recognize.py)→ 用户逐张核对后保存。从识别结果保存时,
|
||||
识别值记在 ai_values,原件挂成附件;没保存的识别 24 小时后删除(建下一次识别时顺手清)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, date, datetime
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.audit import service as audit
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core import keys, queue
|
||||
from bid_doctor.core.authz import Permission, require
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import open_object, seal_object, sign_download
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import qualification_prefix, storage
|
||||
from bid_doctor.core.storage import qualification_prefix, qualification_scan_prefix, storage
|
||||
from bid_doctor.document import uploads
|
||||
from bid_doctor.document.pipeline import ocr
|
||||
from bid_doctor.project.models import Project
|
||||
from bid_doctor.qualification.models import Qualification, QualificationFile
|
||||
from bid_doctor.qualification import recognize
|
||||
from bid_doctor.qualification.models import Qualification, QualificationFile, QualificationScan
|
||||
|
||||
LOCAL_TZ = ZoneInfo("Asia/Shanghai")
|
||||
ACTIVE_STATUSES = ("tracking", "drafting", "checked") # 进行中的项目:还没递交、没归档
|
||||
FIELDS = ("name", "kind", "level", "cert_no", "issuer", "valid_from", "valid_to")
|
||||
MEDIA_TYPES = {b"%PDF": "application/pdf", b"\xff\xd8\xff": "image/jpeg", b"\x89PNG": "image/png"}
|
||||
FIELDS = ("name", "kind", "level", "cert_no", "issuer", "holder", "valid_from", "valid_to")
|
||||
SCAN_TASK = "recognize_qualification"
|
||||
SCAN_DAILY_LIMIT = 100 # 每个企业每天最多识别的次数(不扣额度,防刷,决策 VS5)
|
||||
SCAN_TTL = timedelta(hours=24)
|
||||
SCAN_ERRORS = {
|
||||
"ocr_unavailable": "扫描件识别服务暂时不可用,可以先手动填写",
|
||||
"model_unavailable": "模型服务暂时不可用,请稍后重试,或先手动填写",
|
||||
"file_corrupted": "文件已损坏,请换一个文件",
|
||||
"file_encrypted": "文件已加密,请上传未加密的版本",
|
||||
"internal_error": "识别出错了,请重试,或先手动填写",
|
||||
}
|
||||
MEDIA_TYPES = {
|
||||
b"%PDF": "application/pdf",
|
||||
b"\xff\xd8\xff": "image/jpeg",
|
||||
b"\x89PNG": "image/png",
|
||||
b"RIFF": "image/webp",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -109,6 +128,7 @@ async def to_api(
|
||||
"id": q.id,
|
||||
**{f: getattr(q, f) for f in FIELDS},
|
||||
"files": [{"id": f.id, "filename": f.filename, "size_bytes": f.size_bytes} for f in files],
|
||||
"recognized": q.ai_values is not None,
|
||||
"late_for": late,
|
||||
"created_at": q.created_at,
|
||||
"updated_at": q.updated_at,
|
||||
@@ -135,18 +155,29 @@ async def records(s: AsyncSession) -> list[Record]:
|
||||
return [Record(q.id, q.name, q.kind, q.level, q.cert_no, q.valid_to) for q in rows]
|
||||
|
||||
|
||||
async def create(s: AsyncSession, ctx: TenantContext, data: dict[str, Any]) -> Qualification:
|
||||
async def create(
|
||||
s: AsyncSession, ctx: TenantContext, data: dict[str, Any], scan: tuple[UUID, int] | None = None
|
||||
) -> Qualification:
|
||||
"""scan:从识别结果保存(识别任务编号,第几张证书):记下识别值,原件挂成附件。"""
|
||||
require(ctx, Permission.QUALIFICATION_EDIT)
|
||||
_check(data)
|
||||
q = Qualification(id=uuid7(), tenant_id=ctx.tenant_id, created_by=ctx.user_id, **data)
|
||||
s.add(q)
|
||||
await s.flush()
|
||||
await audit.record(s, ctx, "qualification.create", "qualification", q.id, after=_values(q))
|
||||
source = await _from_scan(s, ctx, q, scan) if scan else None
|
||||
await audit.record(s, ctx, "qualification.create", "qualification", q.id, after={**_values(q), **_src(source)})
|
||||
await s.refresh(q)
|
||||
return q
|
||||
|
||||
|
||||
async def update(s: AsyncSession, ctx: TenantContext, qualification_id: UUID, data: dict[str, Any]) -> Qualification:
|
||||
async def update(
|
||||
s: AsyncSession,
|
||||
ctx: TenantContext,
|
||||
qualification_id: UUID,
|
||||
data: dict[str, Any],
|
||||
scan: tuple[UUID, int] | None = None,
|
||||
) -> Qualification:
|
||||
"""scan:用新识别的证书更新已有的一条(换证),新原件也挂上,旧附件保留。"""
|
||||
require(ctx, Permission.QUALIFICATION_EDIT)
|
||||
_check(data)
|
||||
q = await get(s, qualification_id)
|
||||
@@ -154,11 +185,40 @@ async def update(s: AsyncSession, ctx: TenantContext, qualification_id: UUID, da
|
||||
for k, v in data.items():
|
||||
setattr(q, k, v)
|
||||
await s.flush()
|
||||
await audit.record(s, ctx, "qualification.update", "qualification", q.id, before=before, after=_values(q))
|
||||
source = await _from_scan(s, ctx, q, scan) if scan else None
|
||||
await audit.record(
|
||||
s, ctx, "qualification.update", "qualification", q.id, before=before, after={**_values(q), **_src(source)}
|
||||
)
|
||||
await s.refresh(q)
|
||||
return q
|
||||
|
||||
|
||||
def _src(source: str | None) -> dict[str, Any]:
|
||||
return {"from_scan": source} if source else {}
|
||||
|
||||
|
||||
async def _from_scan(s: AsyncSession, ctx: TenantContext, q: Qualification, ref: tuple[UUID, int]) -> str:
|
||||
scan = await _scan(s, ref[0])
|
||||
certs = (scan.result or {}).get("certificates") or []
|
||||
if scan.status != "succeeded" or not 0 <= ref[1] < len(certs):
|
||||
raise AppError("validation_failed", "识别结果已过期或不存在,请重新上传")
|
||||
c = certs[ref[1]]
|
||||
q.ai_values = {
|
||||
"scan_id": str(scan.id),
|
||||
"filename": scan.filename,
|
||||
"prompt": (scan.result or {}).get("prompt"),
|
||||
"category": c.get("category"),
|
||||
"fields": {k: {"value": v.get("value"), "check": v.get("check")} for k, v in (c.get("fields") or {}).items()},
|
||||
"masked": c.get("masked") or [],
|
||||
}
|
||||
files = (await s.execute(select(QualificationFile.sha256).where(QualificationFile.qualification_id == q.id))).all()
|
||||
if scan.sha256 not in {bytes(x) for (x,) in files}:
|
||||
dk = await keys.data_key(s, ctx)
|
||||
await _add_file(s, ctx, q, scan.filename, open_object(dk, storage().get(scan.object_key), scan.object_key))
|
||||
await s.flush()
|
||||
return str(scan.id)
|
||||
|
||||
|
||||
async def delete(s: AsyncSession, ctx: TenantContext, qualification_id: UUID) -> None:
|
||||
require(ctx, Permission.QUALIFICATION_EDIT)
|
||||
q = await get(s, qualification_id)
|
||||
@@ -177,28 +237,32 @@ async def attach(ctx: TenantContext, qualification_id: UUID, upload_id: UUID) ->
|
||||
raise AppError("validation_failed", "这个上传不是资质附件")
|
||||
async with tenant_tx(ctx) as s:
|
||||
q = await get(s, qualification_id)
|
||||
dk = await keys.data_key(s, ctx)
|
||||
f = QualificationFile(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
qualification_id=q.id,
|
||||
object_key="",
|
||||
filename=meta.filename,
|
||||
sha256=hashlib.sha256(data).digest(),
|
||||
size_bytes=len(data),
|
||||
)
|
||||
f.object_key = f"{qualification_prefix(ctx.tenant_id, q.id)}/{f.id}"
|
||||
storage().put(f.object_key, seal_object(dk, data, f.object_key))
|
||||
s.add(f)
|
||||
await s.flush()
|
||||
await audit.record(
|
||||
s, ctx, "qualification_file.attach", "qualification", q.id, after={"file": str(f.id), "name": f.filename}
|
||||
)
|
||||
await _add_file(s, ctx, q, meta.filename, data)
|
||||
out = await to_api(s, q)
|
||||
await uploads.discard(ctx, upload_id)
|
||||
return out
|
||||
|
||||
|
||||
async def _add_file(s: AsyncSession, ctx: TenantContext, q: Qualification, filename: str, data: bytes) -> None:
|
||||
dk = await keys.data_key(s, ctx)
|
||||
f = QualificationFile(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
qualification_id=q.id,
|
||||
object_key="",
|
||||
filename=filename,
|
||||
sha256=hashlib.sha256(data).digest(),
|
||||
size_bytes=len(data),
|
||||
)
|
||||
f.object_key = f"{qualification_prefix(ctx.tenant_id, q.id)}/{f.id}"
|
||||
storage().put(f.object_key, seal_object(dk, data, f.object_key))
|
||||
s.add(f)
|
||||
await s.flush()
|
||||
await audit.record(
|
||||
s, ctx, "qualification_file.attach", "qualification", q.id, after={"file": str(f.id), "name": f.filename}
|
||||
)
|
||||
|
||||
|
||||
async def _file(s: AsyncSession, qualification_id: UUID, file_id: UUID) -> QualificationFile:
|
||||
f = (
|
||||
await s.execute(
|
||||
@@ -236,6 +300,34 @@ async def file_url(s: AsyncSession, ctx: TenantContext, qualification_id: UUID,
|
||||
return {"url": url, "expires_at": datetime.fromtimestamp(exp, UTC), "sha256": f.sha256.hex()}
|
||||
|
||||
|
||||
async def file_previews(s: AsyncSession, ctx: TenantContext, qualification_id: UUID, file_id: UUID) -> dict[str, Any]:
|
||||
"""附件的预览图(详情里的缩略图和大图浏览用):图片一张,PDF 每页一张(最多前 10 页,和识别同一套渲染)。
|
||||
用短时签名地址:<img> 带不了鉴权头。"""
|
||||
f = await _file(s, qualification_id, file_id)
|
||||
dk = await keys.data_key(s, ctx)
|
||||
data = open_object(dk, storage().get(f.object_key), f.object_key)
|
||||
images = recognize.page_images(data, media_type(data))
|
||||
base, ttl = get_settings().public_base_url, get_settings().download_url_ttl_s
|
||||
pages = []
|
||||
for no, img in enumerate(images, start=1):
|
||||
sig, exp = sign_download(f"{ctx.tenant_id}:{f.id}:{no}:qualification_preview", ttl)
|
||||
url = f"{base}/api/v1/qualification-file-previews/{ctx.tenant_id}/{f.id}/{no}?exp={exp}&sig={sig}"
|
||||
pages.append({"page": no, "width": img.width, "height": img.height, "url": url})
|
||||
return {"file_id": f.id, "filename": f.filename, "pages": pages}
|
||||
|
||||
|
||||
async def preview_image(s: AsyncSession, ctx: TenantContext, file_id: UUID, page: int) -> bytes:
|
||||
f = (await s.execute(select(QualificationFile).where(QualificationFile.id == file_id))).scalar_one_or_none()
|
||||
if f is None:
|
||||
raise AppError("not_found")
|
||||
dk = await keys.data_key(s, ctx)
|
||||
data = open_object(dk, storage().get(f.object_key), f.object_key)
|
||||
images = recognize.page_images(data, media_type(data))
|
||||
if not 1 <= page <= len(images):
|
||||
raise AppError("not_found")
|
||||
return recognize.jpeg(images[page - 1], 2400)
|
||||
|
||||
|
||||
async def file_content(s: AsyncSession, ctx: TenantContext, file_id: UUID) -> tuple[QualificationFile, bytes, str]:
|
||||
f = (await s.execute(select(QualificationFile).where(QualificationFile.id == file_id))).scalar_one_or_none()
|
||||
if f is None:
|
||||
@@ -244,3 +336,112 @@ async def file_content(s: AsyncSession, ctx: TenantContext, file_id: UUID) -> tu
|
||||
data = open_object(dk, storage().get(f.object_key), f.object_key)
|
||||
media = next((t for magic, t in MEDIA_TYPES.items() if data.startswith(magic)), "application/octet-stream")
|
||||
return f, data, media
|
||||
|
||||
|
||||
# —— 上传识别(vision-plan 第 2 步) ——
|
||||
|
||||
|
||||
def media_type(data: bytes) -> str:
|
||||
return next((t for magic, t in MEDIA_TYPES.items() if data.startswith(magic)), "application/octet-stream")
|
||||
|
||||
|
||||
async def create_scan(ctx: TenantContext, upload_id: UUID) -> dict[str, Any]:
|
||||
"""收下证书扫描件并排队识别。不扣额度,每个企业每天限 SCAN_DAILY_LIMIT 次;OCR 没开时直接说明,可以手动填写。"""
|
||||
require(ctx, Permission.QUALIFICATION_EDIT)
|
||||
if not ocr.enabled():
|
||||
raise AppError("ocr_unavailable")
|
||||
meta, data = await uploads.take(ctx, upload_id)
|
||||
if meta.purpose != "qualification_file":
|
||||
raise AppError("validation_failed", "这个上传不是资质附件")
|
||||
async with tenant_tx(ctx) as s:
|
||||
await _purge_scans(s, ctx)
|
||||
since = datetime.now(UTC) - SCAN_TTL
|
||||
n = (await s.execute(select(func.count()).where(QualificationScan.created_at >= since))).scalar_one()
|
||||
if n >= SCAN_DAILY_LIMIT:
|
||||
raise AppError("rate_limited", f"每个企业每天最多识别 {SCAN_DAILY_LIMIT} 次证书,可以先手动填写")
|
||||
dk = await keys.data_key(s, ctx)
|
||||
scan = QualificationScan(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
filename=meta.filename,
|
||||
object_key="",
|
||||
sha256=hashlib.sha256(data).digest(),
|
||||
size_bytes=len(data),
|
||||
media_type=media_type(data),
|
||||
created_by=ctx.user_id,
|
||||
)
|
||||
scan.object_key = f"{qualification_scan_prefix(ctx.tenant_id, scan.id)}/original"
|
||||
storage().put(scan.object_key, seal_object(dk, data, scan.object_key))
|
||||
s.add(scan)
|
||||
await s.flush()
|
||||
await queue.enqueue(s, SCAN_TASK, {"tenant_id": str(ctx.tenant_id), "scan_id": str(scan.id)})
|
||||
await s.refresh(scan)
|
||||
out = scan_api(ctx, scan)
|
||||
await uploads.discard(ctx, upload_id)
|
||||
return out
|
||||
|
||||
|
||||
async def _scan(s: AsyncSession, scan_id: UUID) -> QualificationScan:
|
||||
scan = (await s.execute(select(QualificationScan).where(QualificationScan.id == scan_id))).scalar_one_or_none()
|
||||
if scan is None:
|
||||
raise AppError("not_found")
|
||||
return scan
|
||||
|
||||
|
||||
async def get_scan(s: AsyncSession, ctx: TenantContext, scan_id: UUID) -> dict[str, Any]:
|
||||
return scan_api(ctx, await _scan(s, scan_id))
|
||||
|
||||
|
||||
def scan_api(ctx: TenantContext, scan: QualificationScan) -> dict[str, Any]:
|
||||
"""识别结果。页面预览图用短时签名地址(<img> 带不了鉴权头);框的坐标是这一页图片的像素,前端按宽高换成比例。"""
|
||||
result = scan.result or {}
|
||||
base = get_settings().public_base_url
|
||||
pages = []
|
||||
for p in result.get("pages") or []:
|
||||
sig, exp = sign_download(
|
||||
f"{ctx.tenant_id}:{scan.id}:{p['page']}:qualification_scan", get_settings().download_url_ttl_s
|
||||
)
|
||||
url = f"{base}/api/v1/qualification-scan-pages/{ctx.tenant_id}/{scan.id}/{p['page']}?exp={exp}&sig={sig}"
|
||||
pages.append({**p, "url": url})
|
||||
error = None
|
||||
if scan.status == "failed":
|
||||
code = scan.error_code or "internal_error"
|
||||
error = {"code": code, "message": SCAN_ERRORS.get(code, SCAN_ERRORS["internal_error"])}
|
||||
return {
|
||||
"id": scan.id,
|
||||
"filename": scan.filename,
|
||||
"status": scan.status,
|
||||
"error": error,
|
||||
"pages": pages,
|
||||
"certificates": result.get("certificates") or [],
|
||||
"created_at": scan.created_at,
|
||||
}
|
||||
|
||||
|
||||
async def scan_page(s: AsyncSession, ctx: TenantContext, scan_id: UUID, page: int) -> bytes:
|
||||
scan = await _scan(s, scan_id)
|
||||
dk = await keys.data_key(s, ctx)
|
||||
data = open_object(dk, storage().get(scan.object_key), scan.object_key)
|
||||
images = recognize.page_images(data, scan.media_type)
|
||||
if not 1 <= page <= len(images):
|
||||
raise AppError("not_found")
|
||||
return recognize.jpeg(images[page - 1], 2000)
|
||||
|
||||
|
||||
async def delete_scan(s: AsyncSession, ctx: TenantContext, scan_id: UUID) -> None:
|
||||
require(ctx, Permission.QUALIFICATION_EDIT)
|
||||
scan = await _scan(s, scan_id)
|
||||
await s.delete(scan)
|
||||
await s.flush()
|
||||
storage().delete_prefix(qualification_scan_prefix(ctx.tenant_id, scan.id))
|
||||
|
||||
|
||||
async def _purge_scans(s: AsyncSession, ctx: TenantContext) -> None:
|
||||
"""没保存的识别留 24 小时(原件是证书扫描件,不长期存在资质以外的地方)。"""
|
||||
old = (
|
||||
await s.execute(select(QualificationScan).where(QualificationScan.created_at < datetime.now(UTC) - SCAN_TTL))
|
||||
).scalars()
|
||||
for scan in list(old):
|
||||
await s.delete(scan)
|
||||
storage().delete_prefix(qualification_scan_prefix(ctx.tenant_id, scan.id))
|
||||
await s.flush()
|
||||
|
||||
@@ -19,7 +19,8 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
TEMPLATE_VERSION = "2026.11-v4" # v2:资质问题的「依据:资质库记录」;v3:出处不重复写文件名;v4:「导出」改称「生成」
|
||||
# v2:资质问题的「依据:资质库记录」;v3:出处不重复写文件名;v4:「导出」改称「生成」;v5:识别文字标「OCR」
|
||||
TEMPLATE_VERSION = "2026.11-v5"
|
||||
TITLE = "投标文件辅助检查报告"
|
||||
FOOTER = "本报告为辅助检查结果,仅供投标前自查参考"
|
||||
DISCLAIMER = (
|
||||
@@ -92,7 +93,7 @@ def _cite(c: dict[str, Any]) -> str:
|
||||
quote = (c.get("quote") or "").strip()
|
||||
quote = quote[:80] + "……" if len(quote) > 80 else quote
|
||||
return f'{_e(c.get("document_label"))} · 第 <span class="fig">{c.get("page_no")}</span> 页' + (
|
||||
f"「{escape(quote)}」" if quote else ""
|
||||
f"「{'OCR ' if c.get('ocr') else ''}{escape(quote)}」" if quote else ""
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,14 @@ async def generate_report(tenant_id: str, job_id: str) -> None:
|
||||
await runner.run(UUID(tenant_id), UUID(job_id))
|
||||
|
||||
|
||||
@app.task(name="recognize_qualification", retry=False)
|
||||
async def recognize_qualification(tenant_id: str, scan_id: str) -> None:
|
||||
"""资质库上传识别(vision-plan 第 2 步)。失败记在识别任务上,用户重传或手动填写。"""
|
||||
from bid_doctor.qualification import recognize
|
||||
|
||||
await recognize.run(UUID(tenant_id), UUID(scan_id))
|
||||
|
||||
|
||||
@app.task(name="purge_project_files", retry=3)
|
||||
async def purge_project_files(tenant_id: str, project_id: str) -> None:
|
||||
"""删除项目以后清对象存储(data-model §6.1)。密钥已经销毁,清不掉也读不出来;失败重试几次。"""
|
||||
|
||||
259
server/tests/test_ocr.py
Normal file
259
server/tests/test_ocr.py
Normal file
@@ -0,0 +1,259 @@
|
||||
"""OCR 补全(vision-plan 第 1 步):扫描件招标文件能解析、出处标 OCR;扫描件投标文件能检查;
|
||||
OCR 服务挂了退回额度;切句规则。
|
||||
|
||||
OCR 服务换成桩:把夹具 PDF 渲染成纯图片 PDF(模拟扫描件),桩按原文件文字层的位置返回「识别结果」
|
||||
(等于一次完美的 OCR)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pypdfium2 as pdfium
|
||||
import pytest
|
||||
|
||||
from bid_doctor.document.pipeline import ocr, text_layer
|
||||
from bid_doctor.document.pipeline.text_layer import Page, Sentence
|
||||
from tests.conftest import FIXTURES, User, balance, run_parse, upload
|
||||
from tests.test_check_flow import run_check
|
||||
|
||||
DPI = 200
|
||||
S = DPI / 72
|
||||
|
||||
|
||||
def scanned(fixture: str) -> tuple[bytes, dict[int, dict[str, Any]]]:
|
||||
"""夹具 → (纯图片 PDF,每页的 OCR 结果)。图片 PDF 的页面尺寸和原文件一样,坐标可以直接对照。"""
|
||||
src = (FIXTURES / fixture).read_bytes()
|
||||
doc = pdfium.PdfDocument(src)
|
||||
images = [doc[i].render(scale=2).to_pil().convert("RGB") for i in range(len(doc))] # pyright: ignore[reportArgumentType]
|
||||
buf = io.BytesIO()
|
||||
images[0].save(buf, "PDF", save_all=True, append_images=images[1:], resolution=144)
|
||||
results: dict[int, dict[str, Any]] = {}
|
||||
for p in text_layer.extract("src", src):
|
||||
lines = []
|
||||
for x in p.sentences:
|
||||
x0, t, x1, b = (
|
||||
min(q[0] for q in x.quads),
|
||||
min(q[1] for q in x.quads),
|
||||
max(q[2] for q in x.quads),
|
||||
max(q[3] for q in x.quads),
|
||||
)
|
||||
box = [[x0 * S, t * S], [x1 * S, t * S], [x1 * S, b * S], [x0 * S, b * S]]
|
||||
lines.append({"box": box, "upright": [x0 * S, t * S, x1 * S, b * S], "text": x.text, "score": 0.98})
|
||||
results[p.page_no] = {"rotation": 0, "lines": lines, "stamps": []}
|
||||
return buf.getvalue(), results
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_ocr(monkeypatch: pytest.MonkeyPatch) -> dict[int, dict[str, Any]]:
|
||||
"""打开 OCR,渲染和识别都换成桩:渲染只记页码,识别按页码返回预先准备的结果。"""
|
||||
pages: dict[int, dict[str, Any]] = {}
|
||||
monkeypatch.setattr(ocr, "enabled", lambda: True)
|
||||
monkeypatch.setattr(ocr, "_render", lambda pdf, page_no, region, dpi: str(page_no).encode())
|
||||
|
||||
async def recognize(client: httpx.AsyncClient, image: bytes, *, full_page: bool) -> dict[str, Any]:
|
||||
return pages.get(int(image.decode()), {"rotation": 0, "lines": [], "stamps": []})
|
||||
|
||||
monkeypatch.setattr(ocr, "recognize", recognize)
|
||||
return pages
|
||||
|
||||
|
||||
async def scanned_tender(u: User, fake: dict[int, dict[str, Any]]) -> tuple[str, str]:
|
||||
pdf, results = scanned("tender-mini.pdf")
|
||||
fake.update(results)
|
||||
project = (await u.post("/projects", json={})).json()
|
||||
done = await upload(u, "tender-scan.pdf", pdf)
|
||||
r = await u.post(f"/projects/{project['id']}/documents", json={"upload_id": done.json()["id"], "role": "tender"})
|
||||
assert r.status_code == 201, r.text
|
||||
return project["id"], r.json()["id"]
|
||||
|
||||
|
||||
async def test_scanned_tender_is_parsed_with_ocr_citations(user: User, fake_ocr: dict[int, dict[str, Any]]) -> None:
|
||||
project_id, doc_id = await scanned_tender(user, fake_ocr)
|
||||
job = await run_parse(user, doc_id)
|
||||
assert job["status"] == "succeeded", job
|
||||
assert job["result"]["items_added"] == 3 # 和文字版的 tender-mini 一样
|
||||
|
||||
items = (await user.get(f"/projects/{project_id}/checklist/items")).json()["items"]
|
||||
formal = [i for i in items if i["zone"] == "formal"]
|
||||
assert len(formal) == 3
|
||||
for it in formal:
|
||||
(c,) = it["citations"]
|
||||
assert c["ocr"] is True and c["quote"].startswith("★") and c["quads"]
|
||||
|
||||
pages = (await user.get(f"/documents/{doc_id}/pages")).json()["items"]
|
||||
assert [p["text_source"] for p in pages] == ["ocr"]
|
||||
|
||||
|
||||
async def test_scanned_tender_without_ocr_still_fails_and_refunds(user: User) -> None:
|
||||
"""没开 OCR(dev 默认):扫描件照旧按「识别不出文字」失败,额度退回。"""
|
||||
pdf, _ = scanned("tender-mini.pdf")
|
||||
project = (await user.post("/projects", json={})).json()
|
||||
done = await upload(user, "tender-scan.pdf", pdf)
|
||||
r = await user.post(f"/projects/{project['id']}/documents", json={"upload_id": done.json()["id"], "role": "tender"})
|
||||
job = await run_parse(user, r.json()["id"])
|
||||
assert job["status"] == "failed" and job["error"]["code"] == "ocr_required" and job["error"]["refunded"]
|
||||
assert "识别不出文字" in job["error"]["message"]
|
||||
|
||||
|
||||
async def test_ocr_unavailable_fails_and_refunds(user: User, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
pdf, _ = scanned("tender-mini.pdf")
|
||||
monkeypatch.setattr(ocr, "enabled", lambda: True)
|
||||
|
||||
async def down(client: httpx.AsyncClient, image: bytes, *, full_page: bool) -> dict[str, Any]:
|
||||
raise ocr.OcrUnavailable("connection refused")
|
||||
|
||||
monkeypatch.setattr(ocr, "recognize", down)
|
||||
project = (await user.post("/projects", json={})).json()
|
||||
done = await upload(user, "tender-scan.pdf", pdf)
|
||||
r = await user.post(f"/projects/{project['id']}/documents", json={"upload_id": done.json()["id"], "role": "tender"})
|
||||
job = await run_parse(user, r.json()["id"])
|
||||
assert job["status"] == "failed" and job["error"]["code"] == "ocr_unavailable" and job["error"]["refunded"]
|
||||
assert await balance(user) == 3
|
||||
|
||||
|
||||
async def test_scanned_bid_is_checked(user: User, fake_ocr: dict[int, dict[str, Any]]) -> None:
|
||||
"""整份扫描的投标文件以前报 bid_text_missing;识别之后照常检查,覆盖范围写明哪些是识别文字。"""
|
||||
project_id, tender_id = await scanned_tender(user, fake_ocr)
|
||||
assert (await run_parse(user, tender_id))["status"] == "succeeded"
|
||||
pdf, results = scanned("bid-mini-v1.pdf")
|
||||
fake_ocr.clear()
|
||||
fake_ocr.update(results)
|
||||
done = await upload(user, "bid-scan.pdf", pdf)
|
||||
body = {"upload_id": done.json()["id"], "role": "bid_volume", "volume_kind": "technical"}
|
||||
bid = (await user.post(f"/projects/{project_id}/documents", json=body)).json()
|
||||
run = await run_check(user, project_id, [bid["id"]])
|
||||
assert run["status"] == "succeeded", run
|
||||
assert any("识别出来的(OCR)" in x for x in run["coverage"]["checked"])
|
||||
results_ = (await user.get(f"/check-runs/{run['id']}/item-results")).json()["items"]
|
||||
responded = [r for r in results_ if r["verdict"] == "responded"]
|
||||
assert responded and all(c["ocr"] for r in responded for c in r["bid_citations"])
|
||||
|
||||
|
||||
# —— 切句(不经过数据库) ——
|
||||
|
||||
|
||||
def _line(text: str, x0: float, top: float, x1: float, h: float = 30, score: float = 0.99) -> dict[str, Any]:
|
||||
return {
|
||||
"box": [[x0, top], [x1, top], [x1, top + h], [x0, top + h]],
|
||||
"upright": [x0, top, x1, top + h],
|
||||
"text": text,
|
||||
"score": score,
|
||||
}
|
||||
|
||||
|
||||
def _page(sentences: list[Sentence] | None = None) -> Page:
|
||||
return Page(page_no=3, width=595, height=842, rotation=0, char_count=0, sentences=sentences or [], image_cover=1.0)
|
||||
|
||||
|
||||
def test_ocr_sentences_follow_text_layer_rules() -> None:
|
||||
page = _page()
|
||||
res = {
|
||||
"rotation": 0,
|
||||
"stamps": [[1000, 1000, 1200, 1200]],
|
||||
"lines": [
|
||||
_line("1.1 项目名称:某某系统建设项目,本项目由甲方负责组织实施并按", 200, 300, 1450),
|
||||
_line("期完成验收。", 200, 340, 500),
|
||||
_line("甲方:", 200, 450, 300),
|
||||
_line("南京某某科技有限公司", 320, 450, 800),
|
||||
_line("序号", 200, 600, 280),
|
||||
_line("名称", 900, 600, 980),
|
||||
_line("京长卿", 1050, 1050, 1150, score=0.999), # 章上的字
|
||||
_line("— 3 —", 700, 2280, 800),
|
||||
],
|
||||
}
|
||||
ocr.add_sentences("doc", page, res, (0.0, 0.0), S, full_page=True)
|
||||
texts = [(x.text, x.location) for x in page.sentences]
|
||||
assert texts == [
|
||||
("1.1 项目名称:某某系统建设项目,本项目由甲方负责组织实施并按期完成验收。", "body"),
|
||||
("甲方:南京某某科技有限公司", "body"),
|
||||
("序号", "body"),
|
||||
("名称", "body"),
|
||||
("— 3 —", "header_footer"),
|
||||
]
|
||||
first = page.sentences[0]
|
||||
assert first.key == "doc:p3:s1" and first.source == "ocr" and first.conf == 0.99 and len(first.quads) == 2
|
||||
assert first.quads[0] == [round(200 / S, 2), round(300 / S, 2), round(1450 / S, 2), round(330 / S, 2)]
|
||||
assert page.ocr_chars > 0 and page.text_source == "ocr"
|
||||
|
||||
|
||||
def test_ocr_keeps_text_layer_numbering_and_drops_duplicates() -> None:
|
||||
"""扫描页上方的文字说明在文字层里已经有了:OCR 认出的同一行去掉;OCR 句接在后面编号。"""
|
||||
caption = Sentence(key="doc:p3:s1", page_no=3, seq=1, text="合同扫描件第 1 页", quads=[[72, 50, 300, 66]])
|
||||
page = _page([caption])
|
||||
res = {
|
||||
"rotation": 0,
|
||||
"lines": [
|
||||
_line("合同扫描件第1页", 72 * S, 50 * S, 300 * S, h=16 * S),
|
||||
_line("软件开发服务合同", 200, 400, 700),
|
||||
],
|
||||
}
|
||||
ocr.add_sentences("doc", page, res, (0.0, 0.0), S, full_page=True)
|
||||
assert [(x.key, x.text, x.source) for x in page.sentences] == [
|
||||
("doc:p3:s1", "合同扫描件第 1 页", "text"),
|
||||
("doc:p3:s2", "软件开发服务合同", "ocr"),
|
||||
]
|
||||
|
||||
|
||||
def test_figure_labels_are_separate_sentences() -> None:
|
||||
"""插图里的字是一个个标签,各自成句,坐标加上图片在页面上的位置。"""
|
||||
page = _page()
|
||||
res = {"rotation": 0, "lines": [_line("业务应用层", 10, 10, 200), _line("统一身份认证", 250, 10, 450)]}
|
||||
ocr.add_sentences("doc", page, res, (100.0, 200.0), S, full_page=False)
|
||||
assert [x.text for x in page.sentences] == ["业务应用层", "统一身份认证"]
|
||||
assert page.sentences[0].quads[0][:2] == [round(100 + 10 / S, 2), round(200 + 10 / S, 2)]
|
||||
assert page.text_source == "ocr" # 这页本来就没有文字(char_count = 0)
|
||||
|
||||
|
||||
def test_rotated_page_uses_upright_order_and_original_coordinates() -> None:
|
||||
"""横放的扫描页:按转正后的坐标排顺序,出处框用原图坐标。"""
|
||||
page = _page()
|
||||
# 原图里是竖排的两行(转正 270°):upright 里上下排列
|
||||
res = {
|
||||
"rotation": 270,
|
||||
"lines": [
|
||||
{
|
||||
"box": [[300, 900], [300, 300], [330, 300], [330, 900]],
|
||||
"upright": [100, 300, 700, 330],
|
||||
"text": "资产负债表",
|
||||
"score": 1.0,
|
||||
},
|
||||
{
|
||||
"box": [[400, 900], [400, 200], [430, 200], [430, 900]],
|
||||
"upright": [100, 400, 800, 430],
|
||||
"text": "单位:元",
|
||||
"score": 1.0,
|
||||
},
|
||||
],
|
||||
}
|
||||
ocr.add_sentences("doc", page, res, (0.0, 0.0), S, full_page=True)
|
||||
assert [x.text for x in page.sentences] == ["资产负债表", "单位:元"]
|
||||
x0, t, x1, b = page.sentences[0].quads[0]
|
||||
assert (x0, x1) == (round(300 / S, 2), round(330 / S, 2)) and b > t
|
||||
|
||||
|
||||
def test_ocr_findings_are_softened() -> None:
|
||||
from bid_doctor.check import findings
|
||||
|
||||
ocr_sent = Sentence(key="d:p1:s9", page_no=1, seq=9, text="壹拾万元", quads=[[0, 0, 1, 1]], source="ocr", conf=0.8)
|
||||
assert findings.ocr_based([(None, ocr_sent)])
|
||||
assert findings.soften("fatal", "大小写不一致。", numeric=True)[0] == "hint"
|
||||
sev, text = findings.soften("fatal", "负偏离未说明。", numeric=False)
|
||||
assert sev == "important" and text.endswith(findings.OCR_NOTE)
|
||||
|
||||
|
||||
def test_rules_skip_ocr_text_on_scanned_attachments() -> None:
|
||||
"""规则只查投标人自己填的格式文件:扫描附件、插图上的识别文字不查;整份扫描的投标文件照查。"""
|
||||
from uuid import uuid4
|
||||
|
||||
from bid_doctor.check import match, runner
|
||||
|
||||
text = Sentence(key="d:p1:s1", page_no=1, seq=1, text="投标函", quads=[[0, 0, 1, 1]])
|
||||
scan = Sentence(key="d:p9:s1", page_no=9, seq=1, text="试用期至___年", quads=[[0, 0, 1, 1]], source="ocr", conf=1.0)
|
||||
doc = match.BidDoc(uuid4(), 1, "投标文件", [text, scan], [9], [], [9])
|
||||
(kept,) = runner._rule_docs([doc], [{"id": str(doc.id), "scan_only": False}]) # pyright: ignore[reportPrivateUsage]
|
||||
assert [x.text for x in kept.sentences] == ["投标函"] and len(doc.sentences) == 2
|
||||
(whole,) = runner._rule_docs([doc], [{"id": str(doc.id), "scan_only": True}]) # pyright: ignore[reportPrivateUsage]
|
||||
assert len(whole.sentences) == 2
|
||||
270
server/tests/test_qualification_scan.py
Normal file
270
server/tests/test_qualification_scan.py
Normal file
@@ -0,0 +1,270 @@
|
||||
"""资质库上传识别(vision-plan 第 2 步):上传 → 识别 → 核对保存;识别值留痕、原件挂成附件;
|
||||
打码、置信度低、对不上的字段不当成确定值;重复证书、主体不一致、只写了有效期长度的提醒;限次;没开 OCR 时说明。
|
||||
|
||||
OCR 服务换成桩(按图片返回预先准备的行),模型用确定性的假模型(按「证书编号:」这类写法取值)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from bid_doctor.document.pipeline import ocr
|
||||
from bid_doctor.qualification import recognize
|
||||
from bid_doctor.qualification import service as qualifications
|
||||
from tests.conftest import User
|
||||
|
||||
|
||||
def png(w: int = 800, h: int = 560) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (w, h), "white").save(buf, "PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def webp() -> bytes:
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (400, 300), "white").save(buf, "WEBP")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def line(text: str, y: float, score: float = 0.99) -> dict[str, Any]:
|
||||
return {
|
||||
"box": [[40, y], [600, y], [600, y + 30], [40, y + 30]],
|
||||
"upright": [40, y, 600, y + 30],
|
||||
"text": text,
|
||||
"score": score,
|
||||
}
|
||||
|
||||
|
||||
CERT = [
|
||||
line("信息系统建设和服务能力等级证书", 20),
|
||||
line("企业名称:测试集成公司", 80),
|
||||
line("证书编号:CS3-3201-2024-0417", 120),
|
||||
line("等级:CS3", 160),
|
||||
line("发证机构:中国电子信息行业联合会", 200),
|
||||
line("发证日期:2024年04月18日", 240),
|
||||
line("有效期至:2028年04月17日", 280),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_ocr(monkeypatch: pytest.MonkeyPatch) -> list[list[dict[str, Any]]]:
|
||||
"""打开 OCR,识别换成桩:按调用顺序依次返回准备好的行(每页一组)。"""
|
||||
queue: list[list[dict[str, Any]]] = []
|
||||
monkeypatch.setattr(ocr, "enabled", lambda: True)
|
||||
|
||||
async def recognize_(client: httpx.AsyncClient, image: bytes, *, full_page: bool) -> dict[str, Any]:
|
||||
assert full_page
|
||||
return {"rotation": 0, "lines": queue.pop(0) if queue else [], "stamps": []}
|
||||
|
||||
monkeypatch.setattr(ocr, "recognize", recognize_)
|
||||
return queue
|
||||
|
||||
|
||||
async def upload(u: User, filename: str, data: bytes) -> str:
|
||||
r = await u.post("/uploads", json={"filename": filename, "size_bytes": len(data), "purpose": "qualification_file"})
|
||||
assert r.status_code == 201, r.text
|
||||
up = r.json()
|
||||
assert (await u.put(f"/uploads/{up['id']}/parts/1", content=data)).status_code == 204
|
||||
done = await u.post(f"/uploads/{up['id']}/complete")
|
||||
assert done.status_code == 200, done.text
|
||||
return up["id"]
|
||||
|
||||
|
||||
async def scan(u: User, filename: str = "cs3.png", data: bytes | None = None) -> dict[str, Any]:
|
||||
r = await u.post("/qualification-scans", json={"upload_id": await upload(u, filename, data or png())})
|
||||
assert r.status_code == 201, r.text
|
||||
assert r.json()["status"] == "queued"
|
||||
await recognize.run(UUID(u.tenant_id), UUID(r.json()["id"])) # 测试进程里直接跑,不经过 worker
|
||||
return (await u.get(f"/qualification-scans/{r.json()['id']}")).json()
|
||||
|
||||
|
||||
async def test_scan_then_save(user: User, fake_ocr: list[list[dict[str, Any]]]) -> None:
|
||||
fake_ocr.append(CERT)
|
||||
s = await scan(user)
|
||||
assert s["status"] == "succeeded" and s["error"] is None
|
||||
(page,) = s["pages"]
|
||||
assert (page["width"], page["height"]) == (800, 560) and "sig=" in page["url"]
|
||||
(c,) = s["certificates"]
|
||||
assert c["category"] == "enterprise_qualification" and c["kind"] == "certificate" and c["page"] == 1
|
||||
f = c["fields"]
|
||||
assert f["cert_no"]["value"] == "CS3-3201-2024-0417" and f["cert_no"]["check"] == "ok"
|
||||
assert f["valid_to"]["day"] == "2028-04-17" and f["valid_to"]["boxes"] == [[40, 280, 600, 310]]
|
||||
assert c["notes"] == [] and c["existing"] is None # 证书主体就是本企业
|
||||
|
||||
# 页面预览图(签名地址,不带鉴权头)
|
||||
img = await user.client.get("http://test" + page["url"].split("8009", 1)[-1])
|
||||
assert img.status_code == 200 and img.headers["content-type"] == "image/jpeg"
|
||||
|
||||
body = {
|
||||
"name": c["name"],
|
||||
"kind": c["kind"],
|
||||
"level": f["level"]["value"],
|
||||
"cert_no": f["cert_no"]["value"],
|
||||
"issuer": f["issuer"]["value"],
|
||||
"holder": f["holder"]["value"],
|
||||
"valid_from": None,
|
||||
"valid_to": f["valid_to"]["day"],
|
||||
"scan": {"scan_id": s["id"], "index": 0},
|
||||
}
|
||||
r = await user.post("/qualifications", json=body)
|
||||
assert r.status_code == 201, r.text
|
||||
q = r.json()
|
||||
assert q["holder"] == "测试集成公司" and q["recognized"] is True and len(q["files"]) == 1
|
||||
assert q["files"][0]["filename"] == "cs3.png"
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.qualification.models import Qualification
|
||||
|
||||
ctx = TenantContext(tenant_id=UUID(user.tenant_id), user_id=UUID(str(user.me["user"]["id"])))
|
||||
async with tenant_tx(ctx) as db:
|
||||
row = (await db.execute(select(Qualification).where(Qualification.id == UUID(q["id"])))).scalar_one()
|
||||
assert row.ai_values is not None and row.ai_values["scan_id"] == s["id"]
|
||||
assert row.ai_values["fields"]["cert_no"] == {"value": "CS3-3201-2024-0417", "check": "ok"}
|
||||
|
||||
# 同一张证书再识别一次:提示资质库里已有编号相同的
|
||||
fake_ocr.append(CERT)
|
||||
again = await scan(user)
|
||||
assert again["certificates"][0]["existing"] == {"id": q["id"], "name": q["name"]}
|
||||
|
||||
# 用新识别结果更新已有的一条:同一个原件不重复挂
|
||||
r = await user.put(f"/qualifications/{q['id']}", json={**body, "scan": {"scan_id": again["id"], "index": 0}})
|
||||
assert r.status_code == 200 and len(r.json()["files"]) == 1
|
||||
|
||||
assert (await user.delete(f"/qualification-scans/{again['id']}")).status_code == 204
|
||||
assert (await user.get(f"/qualification-scans/{again['id']}")).status_code == 404
|
||||
|
||||
|
||||
async def test_masked_low_confidence_and_mismatched_fields(user: User, fake_ocr: list[list[dict[str, Any]]]) -> None:
|
||||
fake_ocr.append(
|
||||
[
|
||||
line("职业技能等级证书", 20),
|
||||
line("姓名:■■■", 80),
|
||||
line("证书编号:FRX11032M897", 120, score=0.71), # 马赛克上认出的字
|
||||
line("发证日期:2024年02月19日", 160),
|
||||
]
|
||||
)
|
||||
(c,) = (await scan(user))["certificates"]
|
||||
assert c["category"] == "personnel" and c["kind"] == "other" and c["notes"] == [{"code": "personnel"}]
|
||||
assert c["masked"] == ["holder"] and "holder" not in c["fields"]
|
||||
assert c["fields"]["cert_no"]["check"] == "low_conf"
|
||||
|
||||
|
||||
async def test_holder_mismatch_and_validity_suggestion(user: User, fake_ocr: list[list[dict[str, Any]]]) -> None:
|
||||
fake_ocr.append(
|
||||
[
|
||||
line("信息安全服务资质证书", 20),
|
||||
line("企业名称:南京某某信息技术有限公司", 80),
|
||||
line("证书编号:CCRC-2025-ISV-SI-0001", 120),
|
||||
line("发证日期:2025-08-27", 160),
|
||||
]
|
||||
)
|
||||
(c,) = (await scan(user))["certificates"]
|
||||
assert c["notes"] == [{"code": "holder_mismatch", "holder": "南京某某信息技术有限公司", "tenant": "测试集成公司"}]
|
||||
assert c["suggest"] == {} # 没写有效期长度,不推算
|
||||
|
||||
|
||||
async def test_scan_needs_ocr_and_is_rate_limited(user: User, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
r = await user.post("/qualification-scans", json={"upload_id": await upload(user, "a.png", png())})
|
||||
assert r.status_code == 503 and r.json()["code"] == "ocr_unavailable"
|
||||
|
||||
monkeypatch.setattr(ocr, "enabled", lambda: True)
|
||||
monkeypatch.setattr(qualifications, "SCAN_DAILY_LIMIT", 1)
|
||||
assert (
|
||||
await user.post("/qualification-scans", json={"upload_id": await upload(user, "a.webp", webp())})
|
||||
).status_code == 201
|
||||
r = await user.post("/qualification-scans", json={"upload_id": await upload(user, "b.png", png())})
|
||||
assert r.status_code == 429 and r.json()["code"] == "rate_limited"
|
||||
|
||||
|
||||
async def test_failed_ocr_is_reported(user: User, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ocr, "enabled", lambda: True)
|
||||
|
||||
async def down(client: httpx.AsyncClient, image: bytes, *, full_page: bool) -> dict[str, Any]:
|
||||
raise ocr.OcrUnavailable("connection refused")
|
||||
|
||||
monkeypatch.setattr(ocr, "recognize", down)
|
||||
s = await scan(user)
|
||||
assert s["status"] == "failed" and s["error"]["code"] == "ocr_unavailable" and s["certificates"] == []
|
||||
|
||||
|
||||
# —— 校验(不经过数据库) ——
|
||||
|
||||
|
||||
def L(text: str, score: float = 0.99) -> dict[str, Any]:
|
||||
return {"text": text, "score": score, "box": [[0, 0], [10, 0], [10, 10], [0, 10]]}
|
||||
|
||||
|
||||
def test_check_field() -> None:
|
||||
lines = {"L1": L("软件名称:建筑工程结构优化设计辅助系统"), "L2": L("V1.0"), "L3": L("登记日期:2024年05月28日")}
|
||||
check = recognize.check_field
|
||||
assert check("software", "建筑工程结构优化设计辅助系统 V1.0", ["L1", "L2"], lines, "software_copyright") == "ok"
|
||||
assert check("issued_on", "2024-5-28", ["L3"], lines, "software_copyright") == "date_ok"
|
||||
assert check("issued_on", "2024年05月29日", ["L3"], lines, "software_copyright") == "mismatch"
|
||||
assert check("issued_on", "五月", ["L3"], lines, "software_copyright") == "bad_format"
|
||||
assert check("cert_no", "X", ["L9"], lines, "software_copyright") == "bad_line"
|
||||
assert check("holder", "甲方", ["L1"], {"L1": L("甲方(用人单位):")}, "personnel") == "bad_format"
|
||||
assert check("holder", "张麒征", ["L1"], {"L1": L("张麒征", 0.69)}, "personnel") == "low_conf"
|
||||
|
||||
|
||||
def test_suggest_valid_to() -> None:
|
||||
assert recognize.suggest_valid_to(date(2025, 8, 27), "五年") == date(2030, 8, 26)
|
||||
assert recognize.suggest_valid_to(date(2024, 2, 29), "3年") == date(2027, 2, 27)
|
||||
assert recognize.suggest_valid_to(None, "五年") is None
|
||||
assert recognize.suggest_valid_to(date(2025, 8, 27), "长期") is None
|
||||
|
||||
|
||||
def test_build_names_and_numbers() -> None:
|
||||
"""名称带上软件名;软著用登记号;两张证书分开。"""
|
||||
lines = {
|
||||
"L1": L("计算机软件著作权登记证书"),
|
||||
"L2": L("软件名称:宇通R系列手机通信软件V1.0"),
|
||||
"L3": L("登记号:2015SR022340"),
|
||||
}
|
||||
raw = {
|
||||
"certificates": [
|
||||
{
|
||||
"category": "software_copyright",
|
||||
"fields": [
|
||||
{"field": "name", "value": "计算机软件著作权登记证书", "lines": ["L1"]},
|
||||
{"field": "software", "value": "宇通R系列手机通信软件V1.0", "lines": ["L2"]},
|
||||
{"field": "reg_no", "value": "2015SR022340", "lines": ["L3"]},
|
||||
],
|
||||
"masked": [],
|
||||
},
|
||||
{"category": "product", "fields": [], "masked": []},
|
||||
]
|
||||
}
|
||||
a, b = recognize.build(1, raw, lines, "某公司", [])
|
||||
assert a["name"] == "计算机软件著作权登记证书(宇通R系列手机通信软件V1.0)" and a["cert_no_field"] == "reg_no"
|
||||
assert a["kind"] == "software_copyright" and b["notes"] == [{"code": "product"}]
|
||||
|
||||
|
||||
async def test_attachment_previews(user: User) -> None:
|
||||
"""附件预览:图片一张、PDF 每页一张;签名地址不带鉴权头也能打开(<img> 用)。"""
|
||||
from tests.test_qualification_flow import attach
|
||||
|
||||
q = (await user.post("/qualifications", json={"name": "ISO 9001", "kind": "system_cert", "cert_no": "Q1"})).json()
|
||||
pdf = io.BytesIO()
|
||||
pages = [Image.new("RGB", (600, 800), "white"), Image.new("RGB", (800, 600), "white")]
|
||||
pages[0].save(pdf, "PDF", save_all=True, append_images=pages[1:])
|
||||
await attach(user, q["id"], "a.png", png())
|
||||
r = await attach(user, q["id"], "b.pdf", pdf.getvalue())
|
||||
files = r.json()["files"]
|
||||
counts = []
|
||||
for f in files:
|
||||
p = (await user.get(f"/qualifications/{q['id']}/files/{f['id']}/previews")).json()
|
||||
counts.append(len(p["pages"]))
|
||||
img = await user.client.get("http://test" + p["pages"][-1]["url"].split("8009", 1)[-1])
|
||||
assert img.status_code == 200 and img.headers["content-type"] == "image/jpeg"
|
||||
assert counts == [1, 2]
|
||||
bad = p["pages"][0]["url"].replace("sig=", "sig=x")
|
||||
assert (await user.client.get("http://test" + bad.split("8009", 1)[-1])).status_code == 404
|
||||
Reference in New Issue
Block a user