feat(check): 响应方式识别(C12)和「这条要不要投标人回答」;评测集纠错
- checklist/response_mode.py:按类别(技术 / 商务)识别招标文件的响应方式(逐条响应 / 默认接受),规则优先; 两种写法冲突按逐条,认不出按逐条并写进覆盖范围。 - check/applicability.py:无效情形(★ / 斜体下划线标注的条款除外)、日期与递交、没要求附材料的评分标准、 无需提供证明或由采购人查询的资格,不逐条找响应,写进覆盖范围;要求附证明材料按原文判断,中标后才交的不算。 - 默认接受的要求没找到响应:一般要求记 implicit_accepted、不出问题;实质性要求记 implicit_review、提示级 人工核对;要附证明材料的不免(迁移 0006)。 - match.py:修同一行单元格又是别的条目候选句时的 KeyError(每条清单项单独记候选句,句子从总表里查)。 - 评测集:注入器同一要求的所有表格行一起改;散文、附件、格式文件里也响应了的要求不挂漏响应类问题; 错位响应必须是另一个参数;复核判不成立的单元不再挂;regen.py 按固定种子重新生成。 开发集 17 份:规矩投标文件报出的问题 639 → 229,误报 21.3 → 12.4 / 100,召回 68% → 65%,致命级 20 → 21 / 32 (docs/architecture/check-eval.md §4)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
91
evals/check/regen.py
Normal file
91
evals/check/regen.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""按固定的种子重新生成开发集的马虎投标文件,沿用复核过的结论(同一招标文件里复核过的「单元 + 问题类型」),
|
||||
列出没复核过的新目标(evals/out/check/review/pending-units.json)。
|
||||
|
||||
cd server && ../.venv/bin/python ../evals/check/regen.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
EVALS = pathlib.Path(__file__).resolve().parents[1]
|
||||
OUT = EVALS / "out" / "check"
|
||||
# (规矩的投标文件,马虎的投标文件,种子,密度):开发集的固定组合(GUIDELINES §4)
|
||||
SLOPPY = [
|
||||
("S02-B", "S02-B-sloppy", 11, "high"),
|
||||
("S04-A", "S04-A-sloppy", 7, "high"),
|
||||
("S10-B", "S10-B-sloppy", 13, "high"),
|
||||
("S01-A", "S01-A-sloppy", 17, "high"),
|
||||
("S05-B", "S05-B-sloppy", 23, "high"),
|
||||
("S07-A", "S07-A-sloppy", 19, "high"),
|
||||
("S08-B", "S08-B-sloppy", 29, "high"),
|
||||
("S04-B", "S04-B-low", 31, "low"),
|
||||
("S08-A", "S08-A-low", 37, "low"),
|
||||
("S10-A", "S10-A-low", 41, "low"),
|
||||
]
|
||||
|
||||
|
||||
def reviews(tender: str) -> dict[tuple[str, str], dict]:
|
||||
out = {}
|
||||
for f in OUT.glob(f"{tender}-*/manifest.json"):
|
||||
for uid, e in json.loads(f.read_text())["units"].items():
|
||||
if e.get("review") and e.get("expect") == "defect":
|
||||
out[(uid, e["defect"])] = {k: v for k, v in e["review"].items() if k != "user_spot_check"}
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
prior = {t: reviews(t) for t in {c[:3] for c, *_ in SLOPPY}}
|
||||
pending = []
|
||||
for clean, sloppy, seed, density in SLOPPY:
|
||||
tender = clean[:3]
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(EVALS / "check" / "mutate.py"),
|
||||
clean,
|
||||
sloppy,
|
||||
"--split",
|
||||
"dev",
|
||||
"--seed",
|
||||
str(seed),
|
||||
"--density",
|
||||
density,
|
||||
]
|
||||
print(subprocess.run(cmd, capture_output=True, text=True, check=True).stdout.strip())
|
||||
p = OUT / sloppy / "manifest.json"
|
||||
m = json.loads(p.read_text())
|
||||
gold = {x["id"]: x for x in json.loads((EVALS / "gold" / f"{tender}.json").read_text())["items"]}
|
||||
for uid, e in m["units"].items():
|
||||
if e.get("expect") != "defect":
|
||||
continue
|
||||
if e.get("defect") == "D02":
|
||||
e["mode"] = "evidence"
|
||||
r = prior[tender].get((uid, e["defect"]))
|
||||
if r:
|
||||
e["reviewed"], e["review"] = r["ok"], r
|
||||
else:
|
||||
x = gold[uid]
|
||||
pending.append(
|
||||
{
|
||||
"bid": sloppy,
|
||||
"tender": tender,
|
||||
"unit": uid,
|
||||
"cat": e["cat"],
|
||||
"basis": x.get("basis"),
|
||||
"anchors": x.get("anchors"),
|
||||
"note": x.get("note"),
|
||||
"mode": e["mode"],
|
||||
"defect": e["defect"],
|
||||
"variant": e.get("variant"),
|
||||
}
|
||||
)
|
||||
p.write_text(json.dumps(m, ensure_ascii=False, indent=1))
|
||||
(OUT / "review" / "pending-units.json").write_text(json.dumps(pending, ensure_ascii=False, indent=1))
|
||||
print("待复核的新目标:", len(pending))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user