feat(check): 响应方式识别(C12)和「这条要不要投标人回答」;评测集纠错
- checklist/response_mode.py:按类别(技术 / 商务)识别招标文件的响应方式(逐条响应 / 默认接受),规则优先; 两种写法冲突按逐条,认不出按逐条并写进覆盖范围。 - check/applicability.py:无效情形(★ / 斜体下划线标注的条款除外)、日期与递交、没要求附材料的评分标准、 无需提供证明或由采购人查询的资格,不逐条找响应,写进覆盖范围;要求附证明材料按原文判断,中标后才交的不算。 - 默认接受的要求没找到响应:一般要求记 implicit_accepted、不出问题;实质性要求记 implicit_review、提示级 人工核对;要附证明材料的不免(迁移 0006)。 - match.py:修同一行单元格又是别的条目候选句时的 KeyError(每条清单项单独记候选句,句子从总表里查)。 - 评测集:注入器同一要求的所有表格行一起改;散文、附件、格式文件里也响应了的要求不挂漏响应类问题; 错位响应必须是另一个参数;复核判不成立的单元不再挂;regen.py 按固定种子重新生成。 开发集 17 份:规矩投标文件报出的问题 639 → 229,误报 21.3 → 12.4 / 100,召回 68% → 65%,致命级 20 → 21 / 32 (docs/architecture/check-eval.md §4)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,7 +9,7 @@
|
||||
1. 没有真实投标文件(真实的国动投标文件只有一份,留作最终检验,见 §4),所以按 S01–S10 的真实招标文件**自己造虚构的投标文件**。
|
||||
2. **规矩的投标文件靠构造保证,不靠复核**:偏离表、响应表、报价表、资格和评分材料清单由脚本从清单生成,技术参数按 `op / value / unit` 算一个满足的值;LLM 只写散文(技术方案、投标函叙述)和按风格改写表格行(截断、同义替换、重新编号),逐行小提示,不整本生成。
|
||||
3. 投标文件对招标文件的**全部要求**响应:「全集」用读标正式清单(`evals/out/runs/fix3/<ID>/items.json`),按「招标侧页码 + 片段」记,不只响应标注集的单元。
|
||||
4. 再用确定性的脚本往里**注入问题**得到马虎的投标文件。注入了什么、注在哪里,脚本同时写进**响应清单**,它就是金标准。**问题只挂在人工复核过的标注单元上**(`evals/gold/<ID>.json`,优先实质性条款和资格条件);`optional` 和复核时补的单元不挂问题。
|
||||
4. 再用确定性的脚本往里**注入问题**得到马虎的投标文件。注入了什么、注在哪里,脚本同时写进**响应清单**,它就是金标准。**问题只挂在复核过的标注单元上**(`evals/gold/<ID>.json`,优先实质性条款和资格条件;复核方式见决策 E1)。`optional` 和复核时补的单元也可以挂,按内容复核(四批复核和用户抽查都是这样做的);**上一次复核判为不成立的单元不再挂问题**。
|
||||
5. 评分只对照响应清单,不用检索、偏离表对齐这些查标自己的逻辑来判断对错。
|
||||
|
||||
## 2. 每条要求怎么才算「响应了」:响应方式
|
||||
|
||||
@@ -41,10 +41,18 @@ UNIT_DOWN = {"TB": ("GB", 1024), "GB": ("MB", 1024), "Gbps": ("Mbps", 1000), "s"
|
||||
class Mutator:
|
||||
def __init__(self, spec: dict[str, Any], vocab: dict[str, Any], rng: random.Random) -> None:
|
||||
self.spec, self.vocab, self.rng = spec, vocab, rng
|
||||
self.vols = spec["volumes"]
|
||||
self.unit_defects: dict[str, dict[str, Any]] = {}
|
||||
self.rule_defects: list[dict[str, Any]] = []
|
||||
self.umbrella: set[str] = set()
|
||||
self.vols = spec["volumes"]
|
||||
self.rejected: set[str] = set() # 以前复核判为不成立的单元(同一招标文件的任何投标文件里),不再挂问题
|
||||
# 在表格以外(技术方案散文、附件、格式文件)也响应了的单元:删表格行、改响应值都不构成问题,不挂这几类
|
||||
self.prose = [
|
||||
{t[i : i + 2] for t in [re.sub(r"\W", "", b["text"])] for i in range(len(t) - 1)}
|
||||
for b in self.blocks
|
||||
if b.get("type") == "para" and b.get("form") == "prose"
|
||||
]
|
||||
self.elsewhere = {u for b in self.blocks if b.get("type") != "table" for u in b.get("responds") or []}
|
||||
|
||||
@property
|
||||
def blocks(self) -> list[dict[str, Any]]:
|
||||
@@ -80,7 +88,7 @@ class Mutator:
|
||||
return out
|
||||
|
||||
def free(self, units: list[str]) -> bool:
|
||||
return bool(units) and not any(u in self.unit_defects for u in units)
|
||||
return bool(units) and not any(u in self.unit_defects or u in self.rejected for u in units)
|
||||
|
||||
def mark(
|
||||
self, units: list[str], defect: str, variant: str, granularity: str = "single", worded: bool = True
|
||||
@@ -103,10 +111,11 @@ class Mutator:
|
||||
chunk = rows[start : start + n]
|
||||
if not all(self.free(r.get("responds") or []) for r in chunk if r.get("responds")):
|
||||
continue
|
||||
if any(set(r.get("responds") or []) & self.elsewhere for r in chunk):
|
||||
continue
|
||||
for r in chunk:
|
||||
self.mark(r.get("responds") or [], "D01", "删连续一段", "block", worded=False)
|
||||
del rows[start : start + n]
|
||||
self.drop_everywhere([u for r in chunk for u in r.get("responds") or []])
|
||||
self.drop_everywhere([u for r in chunk for u in r.get("responds") or []], [(t, r) for r in chunk])
|
||||
return
|
||||
|
||||
def row_ops(self, n: int) -> None:
|
||||
@@ -126,10 +135,43 @@ class Mutator:
|
||||
done += 1
|
||||
break
|
||||
|
||||
def drop_everywhere(self, units: list[str]) -> None:
|
||||
"""漏响应要在所有表里都漏:S04 的技术参数同时列在全量偏离表和 ★ 表里,只删一张表等于没漏。"""
|
||||
def in_prose(self, text: str) -> bool:
|
||||
"""技术方案散文里基本复述了这条要求(字 2-gram 有六成以上出现在同一段里):表格里删掉或改错,散文里还在响应。"""
|
||||
t = re.sub(r"\W", "", text)
|
||||
grams = {t[i : i + 2] for i in range(len(t) - 1)}
|
||||
return len(grams) >= 6 and any(len(grams & g) / len(grams) >= 0.6 for g in self.prose)
|
||||
|
||||
def _req_text(self, t: dict[str, Any], r: dict[str, Any]) -> str:
|
||||
at = self.col(t, "requirement")
|
||||
return re.sub(r"\W", "", r["cells"][at]) if at is not None else ""
|
||||
|
||||
def siblings(self, t: dict[str, Any], r: dict[str, Any]) -> list[tuple[dict[str, Any], dict[str, Any]]]:
|
||||
"""同一条要求在投标文件里的所有表格行:挂同一个标注单元,或者要求原文相同(读标清单常把同一要求抽成几条,
|
||||
S04 的参数同时列在两张表里)。注入的问题要在这些行上一起做,否则别处还留着正确的响应,就不是问题了。"""
|
||||
units, text = set(r.get("responds") or []), self._req_text(t, r)
|
||||
out = []
|
||||
for t2 in self.tables():
|
||||
for r2 in t2["rows"]:
|
||||
if r2 is r:
|
||||
continue
|
||||
if units & set(r2.get("responds") or []) or (len(text) >= 6 and self._req_text(t2, r2) == text):
|
||||
out.append((t2, r2))
|
||||
return out
|
||||
|
||||
def propagate(self, t: dict[str, Any], r: dict[str, Any]) -> None:
|
||||
"""把这一行改过的响应、偏离、说明照样写到同一要求的其他行上。"""
|
||||
for t2, r2 in self.siblings(t, r):
|
||||
for field in ("answer", "deviation", "note"):
|
||||
a, b = self.col(t, field), self.col(t2, field)
|
||||
if a is not None and b is not None:
|
||||
r2["cells"][b] = r["cells"][a]
|
||||
|
||||
def drop_everywhere(self, units: list[str], rows: list[tuple[dict[str, Any], dict[str, Any]]] = ()) -> None:
|
||||
"""漏响应要在所有表里都漏:S04 的技术参数同时列在全量偏离表和 ★ 表里,只删一张表等于没漏;
|
||||
要求原文相同的重复行也一起删。"""
|
||||
drop = {id(r2) for t, r in rows for _, r2 in self.siblings(t, r)} | {id(r) for _, r in rows}
|
||||
for t in self.tables():
|
||||
t["rows"] = [r for r in t["rows"] if not set(r.get("responds") or []) & set(units)]
|
||||
t["rows"] = [r for r in t["rows"] if id(r) not in drop and not set(r.get("responds") or []) & set(units)]
|
||||
|
||||
@staticmethod
|
||||
def col(t: dict[str, Any], field: str) -> int | None:
|
||||
@@ -148,12 +190,14 @@ class Mutator:
|
||||
numeric = tp.get("op") in (">=", "<=", "=") and tp.get("value")
|
||||
if op == "D05" and (numeric or not FUNCTIONAL.search(req_text) or NON_FUNCTIONAL.search(req_text)):
|
||||
return False # 「暂不支持」只套在功能类要求上(数值参数、证书、清单、服务义务不会这么写)
|
||||
if op in ("D01", "D03", "D08", "D09") and (set(units) & self.elsewhere or self.in_prose(req_text)):
|
||||
return False
|
||||
if op == "D01":
|
||||
t["rows"].remove(r)
|
||||
self.drop_everywhere(units)
|
||||
self.drop_everywhere(units, [(t, r)])
|
||||
self.mark(units, "D01", "删一行", worded=False)
|
||||
elif op == "D03":
|
||||
return self._unmet(r, tp, ans, dev)
|
||||
if not self._unmet(r, tp, ans, dev):
|
||||
return False
|
||||
elif op == "D05":
|
||||
param = tp.get("param") or (r["cells"][req][:12] if req is not None else "该功能")
|
||||
r["cells"][ans] = self.phrase("D05", "phrases").format(param=param)
|
||||
@@ -171,11 +215,14 @@ class Mutator:
|
||||
r["cells"][ans] = self.phrase("D08", "phrases")
|
||||
self.mark(units, "D08", r["cells"][ans], worded=False)
|
||||
elif op == "D09":
|
||||
mine = str(tp.get("param") or "")
|
||||
others = [
|
||||
x
|
||||
for _, x in cands
|
||||
if x is not r
|
||||
and (x.get("tp") or {}).get("param")
|
||||
and x["tp"]["param"] != mine # 必须是另一个参数:同名参数(另一台设备的同一指标)的响应可能恰好满足
|
||||
and (not mine or mine not in x["cells"][x.get("anchor", ans)])
|
||||
and x["cells"][x.get("anchor", ans)] != r["cells"][ans]
|
||||
]
|
||||
if not others:
|
||||
@@ -183,6 +230,8 @@ class Mutator:
|
||||
o = self.rng.choice(others)
|
||||
r["cells"][ans] = o["cells"][o.get("anchor", ans)]
|
||||
self.mark(units, "D09", "另一条参数的响应", worded=False)
|
||||
if op != "D01":
|
||||
self.propagate(t, r)
|
||||
return True
|
||||
|
||||
def _unmet(self, r: dict[str, Any], tp: dict[str, Any], ans: int, dev: int | None) -> bool:
|
||||
@@ -205,7 +254,7 @@ class Mutator:
|
||||
if dev is not None and self.rng.random() < 0.5:
|
||||
r["cells"][dev] = "无偏离" # 一半在偏离栏仍写「无偏离」(另一半保持原样,也是无偏离或正偏离)
|
||||
self.mark(r.get("responds") or [], "D03", variant, worded=False)
|
||||
return True
|
||||
return True # 同一要求的其他行由调用方(_apply)同步
|
||||
|
||||
def implicit_deviation(self, n: int) -> None:
|
||||
"""只列偏离的表:挑默认接受的要求写成偏离(D06 负偏离未说明 / D05 写明不支持)。"""
|
||||
@@ -421,6 +470,10 @@ def main() -> None:
|
||||
optional = {u.id for u in units if u.optional}
|
||||
modes = unit_modes(spec) # 注入前算:被删掉的行也要知道它本来该怎么响应
|
||||
m = Mutator(spec, json.loads(VOCAB[a.split].read_text()), random.Random(a.seed)) # noqa: S311 评测数据,不用于安全
|
||||
for f in OUT.glob(f"{tender}-*/manifest.json"):
|
||||
for uid, e in json.loads(f.read_text())["units"].items():
|
||||
if (e.get("review") or {}).get("ok") is False:
|
||||
m.rejected.add(uid) # noqa: S311 评测数据,不用于安全
|
||||
if not a.manifest_only:
|
||||
if a.density == "high":
|
||||
m.d01_block()
|
||||
|
||||
91
evals/check/regen.py
Normal file
91
evals/check/regen.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""按固定的种子重新生成开发集的马虎投标文件,沿用复核过的结论(同一招标文件里复核过的「单元 + 问题类型」),
|
||||
列出没复核过的新目标(evals/out/check/review/pending-units.json)。
|
||||
|
||||
cd server && ../.venv/bin/python ../evals/check/regen.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
EVALS = pathlib.Path(__file__).resolve().parents[1]
|
||||
OUT = EVALS / "out" / "check"
|
||||
# (规矩的投标文件,马虎的投标文件,种子,密度):开发集的固定组合(GUIDELINES §4)
|
||||
SLOPPY = [
|
||||
("S02-B", "S02-B-sloppy", 11, "high"),
|
||||
("S04-A", "S04-A-sloppy", 7, "high"),
|
||||
("S10-B", "S10-B-sloppy", 13, "high"),
|
||||
("S01-A", "S01-A-sloppy", 17, "high"),
|
||||
("S05-B", "S05-B-sloppy", 23, "high"),
|
||||
("S07-A", "S07-A-sloppy", 19, "high"),
|
||||
("S08-B", "S08-B-sloppy", 29, "high"),
|
||||
("S04-B", "S04-B-low", 31, "low"),
|
||||
("S08-A", "S08-A-low", 37, "low"),
|
||||
("S10-A", "S10-A-low", 41, "low"),
|
||||
]
|
||||
|
||||
|
||||
def reviews(tender: str) -> dict[tuple[str, str], dict]:
|
||||
out = {}
|
||||
for f in OUT.glob(f"{tender}-*/manifest.json"):
|
||||
for uid, e in json.loads(f.read_text())["units"].items():
|
||||
if e.get("review") and e.get("expect") == "defect":
|
||||
out[(uid, e["defect"])] = {k: v for k, v in e["review"].items() if k != "user_spot_check"}
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
prior = {t: reviews(t) for t in {c[:3] for c, *_ in SLOPPY}}
|
||||
pending = []
|
||||
for clean, sloppy, seed, density in SLOPPY:
|
||||
tender = clean[:3]
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(EVALS / "check" / "mutate.py"),
|
||||
clean,
|
||||
sloppy,
|
||||
"--split",
|
||||
"dev",
|
||||
"--seed",
|
||||
str(seed),
|
||||
"--density",
|
||||
density,
|
||||
]
|
||||
print(subprocess.run(cmd, capture_output=True, text=True, check=True).stdout.strip())
|
||||
p = OUT / sloppy / "manifest.json"
|
||||
m = json.loads(p.read_text())
|
||||
gold = {x["id"]: x for x in json.loads((EVALS / "gold" / f"{tender}.json").read_text())["items"]}
|
||||
for uid, e in m["units"].items():
|
||||
if e.get("expect") != "defect":
|
||||
continue
|
||||
if e.get("defect") == "D02":
|
||||
e["mode"] = "evidence"
|
||||
r = prior[tender].get((uid, e["defect"]))
|
||||
if r:
|
||||
e["reviewed"], e["review"] = r["ok"], r
|
||||
else:
|
||||
x = gold[uid]
|
||||
pending.append(
|
||||
{
|
||||
"bid": sloppy,
|
||||
"tender": tender,
|
||||
"unit": uid,
|
||||
"cat": e["cat"],
|
||||
"basis": x.get("basis"),
|
||||
"anchors": x.get("anchors"),
|
||||
"note": x.get("note"),
|
||||
"mode": e["mode"],
|
||||
"defect": e["defect"],
|
||||
"variant": e.get("variant"),
|
||||
}
|
||||
)
|
||||
p.write_text(json.dumps(m, ensure_ascii=False, indent=1))
|
||||
(OUT / "review" / "pending-units.json").write_text(json.dumps(pending, ensure_ascii=False, indent=1))
|
||||
print("待复核的新目标:", len(pending))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -23,7 +23,8 @@ from bid_doctor.ai import prompts
|
||||
from bid_doctor.ai import service as ai_service
|
||||
from bid_doctor.ai.providers.base import near_limit
|
||||
from bid_doctor.ai.service import OutputIncomplete, provider
|
||||
from bid_doctor.check import findings, match
|
||||
from bid_doctor.check import applicability, findings, match
|
||||
from bid_doctor.checklist import response_mode
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.document.pipeline import text_layer
|
||||
@@ -120,6 +121,7 @@ async def main() -> None:
|
||||
ap.add_argument("--name", default="base")
|
||||
ap.add_argument("--baseline", action="store_true", help="只检索、不调模型的基线")
|
||||
ap.add_argument("--v1", action="store_true", help="第 1 步的做法:提示词 v1、不带同一行的单元格(对比基线)")
|
||||
ap.add_argument("--no-modes", action="store_true", help="不按响应方式和「要不要回答」筛选(C12 之前的做法)")
|
||||
args = ap.parse_args()
|
||||
d = OUT / args.bid
|
||||
manifest = json.loads((d / "manifest.json").read_text())
|
||||
@@ -137,9 +139,34 @@ async def main() -> None:
|
||||
return None
|
||||
|
||||
ctx = TenantContext(tenant_id=UUID(int=0), user_id=UUID(int=0))
|
||||
all_items = items
|
||||
decisions: dict[UUID, applicability.Decision] = {}
|
||||
if not args.no_modes:
|
||||
modes = response_mode.detect(gold.sentences(manifest["tender"]))
|
||||
decisions = {i: applicability.decide(p, modes) for i, p in items.items()}
|
||||
items = {i: p for i, p in items.items() if decisions[i].match}
|
||||
results, stats = await match.run(ctx, uuid4(), items, docs, progress)
|
||||
if decisions:
|
||||
match.apply_modes(results, items, decisions)
|
||||
rows = []
|
||||
for item_id, item in items.items():
|
||||
for item_id, item in all_items.items():
|
||||
if item_id not in items:
|
||||
rows.append(
|
||||
{
|
||||
"title": item["title"],
|
||||
"sentences": item["sentences"],
|
||||
"categories": item["categories"],
|
||||
"importance": item["importance"],
|
||||
"verdict": f"skipped:{decisions[item_id].skip}",
|
||||
"layer": None,
|
||||
"evidence_status": "none",
|
||||
"reason_code": None,
|
||||
"evidence": [],
|
||||
"detail": {},
|
||||
"finding": None,
|
||||
}
|
||||
)
|
||||
continue
|
||||
res = results[item_id]
|
||||
spec = findings.spec(res, item)
|
||||
rows.append(
|
||||
|
||||
Reference in New Issue
Block a user