feat: 读标补齐五类清单与项目基本信息,修复国动样本暴露的 4 个问题
- 文字层 v2:页眉页脚带单独成句;写到页尾的句子和下一页首句接续(出处两页各一条) - 结构识别(最小版):十进制编号、冒号引出的列表找上级条款;上级条款写明无效的,条目照样认定 - 抽取:资格、评分、日期与递交、技术参数、项目基本信息各一份提示词;实质性条款出 v2 (认上级条款、补「做无效响应处理」等写法);各类取值核对;按首句合并; 拆开被合成一条的列举条款;描述标注方式的句子和表头碎片不当条款;* 号标注 - 项目:解析出的名称、编号、采购人、预算、截止时间、分包写回项目,用户改过的不覆盖 - 前端:清单项显示分值、参数要求、日期、证明材料 - 测试 40 个,新增两页虚构样例 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
BIN
server/tests/fixtures/tender-full.pdf
vendored
Normal file
BIN
server/tests/fixtures/tender-full.pdf
vendored
Normal file
Binary file not shown.
@@ -159,3 +159,44 @@ async def test_objects_are_encrypted_at_rest(user: User) -> None:
|
||||
assert files and not any(p.read_bytes().startswith(b"%PDF") for p in files)
|
||||
plain = (FIXTURES / "tender-mini.pdf").read_bytes()[:64]
|
||||
assert not any(plain in p.read_bytes() for p in files)
|
||||
|
||||
|
||||
async def test_all_categories_and_project_info(user: User) -> None:
|
||||
"""虚构的两页样例:项目基本信息、截止时间、资格、评分、技术参数、上级标题下的无效情形、* 号兜底、页脚不进清单。"""
|
||||
project_id, doc_id = await project_with_tender(user, "tender-full.pdf")
|
||||
job = await run_parse(user, doc_id)
|
||||
assert job["status"] == "succeeded", job
|
||||
|
||||
project = (await user.get(f"/projects/{project_id}")).json()
|
||||
assert project["name"] == "某市智慧政务服务平台建设项目" # 建项目时没填名称:按招标文件自动填
|
||||
assert project["code"] == "TEST-2026-001"
|
||||
assert project["purchaser"] == "某市行政审批局"
|
||||
assert project["legal_regime"] == "gov_procurement"
|
||||
assert project["deadline_at"].startswith("2026-12-01T01:30") # 北京时间 9:30
|
||||
fields = {f["key"]: f for f in project["fields"]}
|
||||
assert fields["budget_total"]["value"] == "450000.00"
|
||||
assert fields["budget_total"]["citation"]["page_no"] == 1
|
||||
|
||||
items = (await user.get(f"/projects/{project_id}/checklist/items")).json()["items"]
|
||||
by_text = {i["requirement_text"]: i for i in items}
|
||||
|
||||
def item(prefix: str) -> dict[str, object]:
|
||||
return next(v for k, v in by_text.items() if k and k.startswith(prefix))
|
||||
|
||||
for clause in ("25.3.1", "25.3.2"): # 本身没写「无效」,靠上级标题认定
|
||||
it = item(clause)
|
||||
assert it["zone"] == "formal" and "invalidity" in it["categories"], it
|
||||
assert item("资格要求")["categories"] == ["qualification"]
|
||||
assert item("技术方案")["spec"]["scoring"]["points"] == "20"
|
||||
assert item("系统支持")["spec"]["tech_param"] == {"param": "系统支持", "op": ">=", "value": "200", "unit": "个"}
|
||||
deadline = item("响应文件递交截止时间")
|
||||
assert deadline["spec"]["schedule"]["date_kind"] == "bid_deadline"
|
||||
star = item("*供应商须提供原厂授权书") # 假模型不认 * 号:规则兜底进待确认
|
||||
assert (star["zone"], star["pending_reason"], star["importance"]) == ("pending", "marker_uncovered", "substantive")
|
||||
assert not any(k and ("—1—" in k or "—2—" in k) for k in by_text) # 页码不混进条款
|
||||
assert not any(k and "斜体且有下划线" in k for k in by_text) # 规则说明本身不是条款
|
||||
|
||||
checklist = (await user.get(f"/projects/{project_id}/checklist")).json()
|
||||
assert {r["marker"] for r in checklist["marking_rules"]} == {"italic_underline", "asterisk"}
|
||||
sentences = (await user.get(f"/documents/{doc_id}/pages/1/sentences")).json()["items"]
|
||||
assert [s["location"] for s in sentences if s["text"] == "—1—"] == ["header_footer"]
|
||||
|
||||
@@ -4,7 +4,17 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from bid_doctor.checklist.extract import check_basis, check_evidence, is_marked
|
||||
from bid_doctor.checklist.extract import (
|
||||
Doc,
|
||||
Extracted,
|
||||
amount_yuan,
|
||||
check_basis,
|
||||
check_evidence,
|
||||
check_value,
|
||||
is_marked,
|
||||
merge_items,
|
||||
parse_at,
|
||||
)
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.events import _sid # pyright: ignore[reportPrivateUsage]
|
||||
from bid_doctor.document.formats import detect
|
||||
@@ -19,28 +29,151 @@ def sent(text: str, runs: list[tuple[int, int, str]] | None = None, seq: int = 1
|
||||
def test_evidence_checks() -> None:
|
||||
s = sent("★投标人须具备有效的信息系统集成服务资质证书。")
|
||||
by_id = {"p1s1": s}
|
||||
assert check_evidence({"sid": "p1s1", "quote": "投标人须具备有效的信息"}, by_id) is None
|
||||
assert check_evidence({"sid": "p9s9", "quote": "投标人须具备有效的信息"}, by_id) == "id"
|
||||
assert check_evidence({"sid": "p1s1", "quote": "投标人须具备"}, by_id) == "quote" # 少于 8 个字
|
||||
assert check_evidence({"sid": "p1s1", "quote": "这句话是模型改写的"}, by_id) == "quote"
|
||||
assert check_evidence({"sid": "p1s1", "quote": "投标人须具备有效的信息"}, by_id) == (None, [s])
|
||||
assert check_evidence({"sid": "p9s9", "quote": "投标人须具备有效的信息"}, by_id)[0] == "id"
|
||||
assert check_evidence({"sid": "p1s1", "quote": "投标人须具备"}, by_id)[0] == "quote" # 少于 8 个字
|
||||
assert check_evidence({"sid": "p1s1", "quote": "这句话是模型改写的"}, by_id)[0] == "quote"
|
||||
|
||||
|
||||
def test_quote_may_run_into_next_sentence() -> None:
|
||||
"""表格、跨页把一句切成两段时,引文可以从这一句接续到紧跟着的下一句,两句都算出处。"""
|
||||
a = sent("支持不少于200个并发", seq=1)
|
||||
b = sent("用户,连续30分钟压力测试成功率不低于99%。", seq=2)
|
||||
by_id = {"p1s1": a, "p1s2": b}
|
||||
assert check_evidence({"sid": "p1s1", "quote": "支持不少于200个并发用户"}, by_id, {a.key: b}) == (None, [a, b])
|
||||
assert check_evidence({"sid": "p1s1", "quote": "支持不少于200个并发用户"}, by_id)[0] == "quote"
|
||||
# 必须从这一句开始,不能整段都在下一句里
|
||||
assert check_evidence({"sid": "p1s1", "quote": "连续30分钟压力测试"}, by_id, {a.key: b})[0] == "quote"
|
||||
|
||||
|
||||
def test_split_listed_conditions() -> None:
|
||||
from bid_doctor.checklist.extract import split_listed
|
||||
|
||||
heading = sent("25.3未实质性响应磋商文件的情形", seq=1)
|
||||
items = [sent("25.3.1未成功解密电子响应文件的。", seq=2), sent("25.3.2未按要求上传电子响应文件的。", seq=3)]
|
||||
d = doc(sentences=[heading, *items], parents={x.key: heading for x in items})
|
||||
merged = Extracted(
|
||||
task="substantive",
|
||||
categories=["invalidity"],
|
||||
summary="未实质性响应的情形",
|
||||
sentences=items,
|
||||
zone="formal",
|
||||
basis="explicit_invalid",
|
||||
importance="substantive",
|
||||
)
|
||||
parts = split_listed(merged, d)
|
||||
assert [p.sentences[0].seq for p in parts] == [2, 3]
|
||||
assert all(p.zone == "formal" for p in parts)
|
||||
|
||||
|
||||
def doc(
|
||||
declared: set[str] | None = None,
|
||||
sentences: list[Sentence] | None = None,
|
||||
parents: dict[str, Sentence] | None = None,
|
||||
) -> Doc:
|
||||
return Doc(sentences=sentences or [], parents=parents or {}, continuations={}, notes=[], declared=declared or set())
|
||||
|
||||
|
||||
def test_basis_check() -> None:
|
||||
star = sent("★服务器内存不低于32GB。")
|
||||
plain = sent("投标人应按要求提供售后服务方案。")
|
||||
iu = sent("投标人须提供原厂授权书。", [(0, 12, "iu")])
|
||||
assert check_basis("star", [star], [star], set())
|
||||
assert not check_basis("star", [plain], [plain], set())
|
||||
assert check_basis("star", [star], [star], doc())
|
||||
assert not check_basis("star", [plain], [plain], doc())
|
||||
# 斜体下划线只在文件声明过时才算(07 F1)
|
||||
assert not check_basis("italic_underline", [iu], [iu], set())
|
||||
assert check_basis("italic_underline", [iu], [iu], {"italic_underline"})
|
||||
assert not check_basis("italic_underline", [iu], [iu], doc())
|
||||
assert check_basis("italic_underline", [iu], [iu], doc({"italic_underline"}))
|
||||
invalid = sent("未提供的,其投标无效。", seq=2)
|
||||
assert check_basis("explicit_invalid", [plain], [plain, invalid], set())
|
||||
assert not check_basis("unknown", [star], [star], set())
|
||||
assert check_basis("explicit_invalid", [plain], [plain, invalid], doc())
|
||||
assert not check_basis("unknown", [star], [star], doc())
|
||||
|
||||
|
||||
def test_invalid_wordings_and_parent_heading() -> None:
|
||||
"""「做无效响应处理」「视为非实质性响应」也算写明无效;列在无效情形标题下的条目,上级标题就是依据。"""
|
||||
for text in ("否则做无效响应处理。", "将被视为非实质性响应。", "其投标将予以拒绝。"):
|
||||
s = sent(text)
|
||||
assert check_basis("explicit_invalid", [s], [s], doc()), text
|
||||
heading = sent("25.3未实质性响应磋商文件的情形", seq=1)
|
||||
item = sent("25.3.5供应商的最后报价超过了采购预算。", seq=9)
|
||||
assert not check_basis("explicit_invalid", [item], [item], doc())
|
||||
assert check_basis("explicit_invalid", [item], [item], doc(parents={item.key: heading}))
|
||||
|
||||
|
||||
def test_marker_talk_is_not_a_clause() -> None:
|
||||
"""国动样本 p67:偏离表表头「序号|实质性要求(斜体且有下划线部分或加*号条款)」被切成碎片,不能当条款。"""
|
||||
frag = sent("号(斜体且有下划线部分或加", [(0, 13, "biu")])
|
||||
assert is_marked(frag, {"italic_underline", "asterisk"}) is None
|
||||
assert not check_basis("italic_underline", [frag], [frag], doc({"italic_underline"}))
|
||||
|
||||
|
||||
def test_asterisk_needs_declaration() -> None:
|
||||
s = sent("*供应商须提供原厂授权书。")
|
||||
assert is_marked(s, set()) is None
|
||||
assert is_marked(s, {"asterisk"}) == "asterisk"
|
||||
assert is_marked(sent("3.*支持国产化CPU。"), {"asterisk"}) == "asterisk"
|
||||
|
||||
|
||||
def test_value_checks() -> None:
|
||||
s = [sent("响应文件递交截止时间:2026年12月1日9时30分(北京时间)。")]
|
||||
assert parse_at("2026-12-01T09:30", s) == ("2026-12-01T09:30:00+08:00", True)
|
||||
assert parse_at("2026-12-02T09:30", s)[1] is False # 日期对不上原文:进待确认
|
||||
assert check_value("200", [sent("系统支持不少于200个并发连接。")])
|
||||
assert not check_value("300", [sent("系统支持不少于200个并发连接。")])
|
||||
assert check_value("0~5", [sent("类似业绩每个得1分,最高5分,满分0~5分。")])
|
||||
assert amount_yuan("45", "万元") == "450000.00"
|
||||
assert amount_yuan("8,000", "元") == "8000.00"
|
||||
|
||||
|
||||
def test_parents_and_continuations() -> None:
|
||||
from bid_doctor.document.pipeline.structure import parents
|
||||
from bid_doctor.document.pipeline.text_layer import continuations
|
||||
|
||||
ss = [
|
||||
sent("25.3未实质性响应磋商文件的情形", seq=1),
|
||||
sent("25.3.1未加盖电子公章的。", seq=2),
|
||||
sent("有下列情形之一的,投标无效:", seq=3),
|
||||
sent("(1)未提交保证金的;", seq=4),
|
||||
sent("26.评审", seq=5),
|
||||
]
|
||||
par = parents(ss)
|
||||
assert par["d:p1:s2"].seq == 1 and par["d:p1:s4"].seq == 3 and "d:p1:s5" not in par
|
||||
|
||||
a = Sentence(key="d:p1:s9", page_no=1, seq=9, text="采购人有权", quads=[], open_end=True)
|
||||
b = Sentence(key="d:p2:s1", page_no=2, seq=1, text="永久使用、修改、维护。", quads=[])
|
||||
h = Sentence(key="d:p2:s1", page_no=2, seq=1, text="11.1供应商应按规定格式填报。", quads=[])
|
||||
assert continuations([a, b]) == {"d:p1:s9": b}
|
||||
assert continuations([a, h]) == {} # 下一页是新的编号条目:不是同一句
|
||||
|
||||
|
||||
def test_merge_items_across_tasks() -> None:
|
||||
s = sent("★服务器内存不低于32GB。")
|
||||
sub = Extracted(
|
||||
task="substantive",
|
||||
categories=["invalidity"],
|
||||
summary="内存要求",
|
||||
sentences=[s],
|
||||
zone="formal",
|
||||
importance="substantive",
|
||||
spec={"invalidity": {"source": "substantive_marker"}},
|
||||
)
|
||||
tech = Extracted(
|
||||
task="tech_param",
|
||||
categories=["tech_param", "invalidity"],
|
||||
summary="内存不低于 32GB",
|
||||
sentences=[s],
|
||||
zone="formal",
|
||||
importance="substantive",
|
||||
spec={"tech_param": {"param": "内存", "op": ">=", "value": "32", "unit": "GB"}},
|
||||
)
|
||||
(m,) = merge_items([sub, tech])
|
||||
assert m.summary == "内存不低于 32GB"
|
||||
assert set(m.categories) == {"tech_param", "invalidity"}
|
||||
assert set(m.spec) == {"tech_param", "invalidity"}
|
||||
|
||||
|
||||
def test_marker_fallback() -> None:
|
||||
assert is_marked(sent("★采用国产化CPU芯片。"), set()) == "star"
|
||||
assert is_marked(sent("带★号的条款为实质性要求。"), set()) is None # 规则说明本身
|
||||
iu = sent("投标人须提供原厂授权书。", [(0, 12, "iu")])
|
||||
assert is_marked(iu, set()) is None
|
||||
assert is_marked(iu, {"italic_underline"}) == "italic_underline"
|
||||
@@ -73,3 +206,13 @@ def test_text_layer_normalizes_radicals_and_keys() -> None:
|
||||
def test_stream_id_compare() -> None:
|
||||
assert _sid("1700000000000-1") > _sid("999-5")
|
||||
assert _sid("bad") == (0, 0)
|
||||
|
||||
|
||||
def test_heading_is_context_not_the_requirement() -> None:
|
||||
from bid_doctor.checklist.extract import without_headings
|
||||
|
||||
heading = sent("25.3未实质性响应磋商文件的情形", seq=1)
|
||||
item = sent("25.3.1未成功解密电子响应文件的。", seq=2)
|
||||
d = doc(sentences=[heading, item], parents={item.key: heading})
|
||||
assert without_headings([heading, item], d) == [item]
|
||||
assert without_headings([heading], d) == [heading] # 只引了标题:保留
|
||||
|
||||
Reference in New Issue
Block a user