feat(evals): 查标评测集生成器、注入、评分和只检索基线
- build.py：从「招标文件档案」和读标清单生成规矩投标文件稿；技术参数按 op / value / unit 算满足的值， 分项报价按分项上限比例分摊，落款日期取截止日前 3 天，偏离表按格式表头决定列（含设备名称列）， 读标漏抽的技术参数也补上响应；routes 支持「+」组合和 prose 散文响应。 - mutate.py：按词库注入要求项问题（单点 / 连续块，带不带标志词）和规则问题，写响应清单； 符合性条件响应表的问题按表级记一个。 - score.py：按单元去重、类型族比对、读标漏抽单列、端到端召回、无法对照单列。 - run_check.py：加只检索不调模型的基线；render.py：扫描件按最后一次出现定位。 - vocab-dev.json：开发集注入词库（留出集的另行封存，不进 git）。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
183
evals/check/score.py
Normal file
183
evals/check/score.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""查标评分：对照响应清单给一次离线检查打分（evals/check/GUIDELINES.md §7）。
|
||||
|
||||
cd server && ../.venv/bin/python ../evals/check/score.py <投标文件> [--run base] [--include-unreviewed]
|
||||
|
||||
- 清单项 → 标注单元：清单项的招标侧句子和单元的句子有重叠就对上（和读标评分同一口径）；
|
||||
- 按单元去重：同一要求在清单里有几条，只要一条报出就算召回，几条都报不算误报；
|
||||
- 问题类型按类型族比（D01 / D02 / D09 / D11 / D13 → unresponded，D03 / D07 → suspected_unmet，……）；
|
||||
- 只算复核过的问题（决策 E1）；--include-unreviewed 时也算没复核的，报告里标明；
|
||||
- 读标没抽到的问题单列「读标漏抽」：不算查标漏报，但计入端到端召回；
|
||||
- 对不上任何标注单元的问题单列「无法对照」。
|
||||
输出：evals/out/check/<投标文件>/runs/<运行>/score.json、score.md。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Any
|
||||
|
||||
EVALS = pathlib.Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(EVALS))
|
||||
import gold # noqa: E402
|
||||
|
||||
OUT = EVALS / "out" / "check"
|
||||
FAMILY = {
|
||||
"D01": {"unresponded"},
|
||||
"D02": {"unresponded"},
|
||||
"D09": {"unresponded"},
|
||||
"D11": {"unresponded"},
|
||||
"D13": {"unresponded"},
|
||||
"D16": {"unresponded"},
|
||||
"D03": {"suspected_unmet"},
|
||||
"D07": {"suspected_unmet"},
|
||||
"D08": {"suspected_unmet"},
|
||||
"D05": {"negative_deviation", "suspected_unmet"},
|
||||
"D06": {"negative_deviation"},
|
||||
"D10": {"negative_deviation"},
|
||||
"D12": {"qualification"},
|
||||
"D14": {"qualification"},
|
||||
"D15": {"qualification", "unresponded"},
|
||||
}
|
||||
FATAL_CATS = {"substantive", "qualification"}
|
||||
|
||||
|
||||
def pct(a: int, b: int) -> str:
|
||||
return f"{a}/{b}（{a / b:.0%}）" if b else "—"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("bid")
|
||||
ap.add_argument("--run", default="base")
|
||||
ap.add_argument("--include-unreviewed", action="store_true")
|
||||
a = ap.parse_args()
|
||||
d = OUT / a.bid
|
||||
manifest = json.loads((d / "manifest.json").read_text())
|
||||
res = json.loads((d / "runs" / a.run / "results.json").read_text())
|
||||
units, _ = gold.units(manifest["tender"])
|
||||
sid_units: dict[str, set[str]] = defaultdict(set)
|
||||
for u in units:
|
||||
for s in u.sids:
|
||||
sid_units[s].add(u.id)
|
||||
|
||||
per_unit: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
unmapped: list[dict[str, Any]] = []
|
||||
for it in res["items"]:
|
||||
hit = {u for s in it["sentences"] for u in sid_units.get(s, ())}
|
||||
for u in hit:
|
||||
per_unit[u].append(it)
|
||||
if not hit and it["finding"]:
|
||||
unmapped.append(it)
|
||||
|
||||
rec: Counter[str] = Counter()
|
||||
rec_total: Counter[str] = Counter()
|
||||
sev_ok = sev_n = 0
|
||||
misses: list[str] = []
|
||||
extraction_miss: list[str] = []
|
||||
fp: list[str] = []
|
||||
fp_by_trap: Counter[str] = Counter()
|
||||
ok_units = trap_units = 0
|
||||
other_findings = 0
|
||||
for uid, e in manifest["units"].items():
|
||||
items = per_unit.get(uid, [])
|
||||
found = [it for it in items if it["finding"]]
|
||||
if e["expect"] == "defect":
|
||||
if not e.get("reviewed") and not a.include_unreviewed:
|
||||
continue
|
||||
keys = [e["defect"], f"粒度:{e['granularity']}", "带标志词" if e.get("worded") else "不带标志词"]
|
||||
fatal = e["cat"] in FATAL_CATS
|
||||
if fatal:
|
||||
keys.append("致命级")
|
||||
for k in keys:
|
||||
rec_total[k] += 1
|
||||
rec_total["端到端"] += 1
|
||||
if not items:
|
||||
extraction_miss.append(f"{uid}（{e['defect']}，{e['cat']}）")
|
||||
continue
|
||||
rec_total["查标"] += 1
|
||||
good = [it for it in found if it["finding"]["type"] in FAMILY.get(e["defect"], set())]
|
||||
if good:
|
||||
for k in [*keys, "端到端", "查标"]:
|
||||
rec[k] += 1
|
||||
if fatal:
|
||||
sev_n += 1
|
||||
sev_ok += any(it["finding"]["severity"] == "fatal" for it in good)
|
||||
else:
|
||||
got = ", ".join(sorted({it["verdict"] for it in items}))
|
||||
misses.append(f"{uid}（{e['defect']} {e.get('variant', '')}）→ 查标结论：{got}")
|
||||
elif e["expect"] == "ok":
|
||||
ok_units += 1
|
||||
if e.get("trap"):
|
||||
trap_units += 1
|
||||
if found:
|
||||
fp.append(f"{uid}（{e['cat']}，{e['mode']}）：{found[0]['finding']['title'][:40]}")
|
||||
if e.get("trap"):
|
||||
fp_by_trap[e["trap"]] += 1
|
||||
elif found:
|
||||
other_findings += 1
|
||||
|
||||
findings = [it for it in res["items"] if it["finding"]]
|
||||
lines = [
|
||||
f"# 查标评分：{a.bid} · 运行 {a.run}",
|
||||
"",
|
||||
f"- 投标文件：{manifest['kind']}，风格 {manifest['style']}，{manifest['split']}；清单 {len(res['items'])} 条，报出问题 {len(findings)} 个",
|
||||
f"- 金标准：{'含未复核的问题（试跑口径）' if a.include_unreviewed else '只算复核过的问题'}；复核方式见决策 E1",
|
||||
f"- 模型：{res['summary']['usage']}",
|
||||
"",
|
||||
"## 召回",
|
||||
"",
|
||||
f"- 查标召回（读标抽到的要求上）：{pct(rec['查标'], rec_total['查标'])}",
|
||||
f"- 端到端召回（含读标漏抽）：{pct(rec['端到端'], rec_total['端到端'])}",
|
||||
f"- 致命级：{pct(rec['致命级'], rec_total['致命级'])}；其中级别也判成致命的 {pct(sev_ok, sev_n)}",
|
||||
"",
|
||||
"| 分类 | 召回 |",
|
||||
"|---|---|",
|
||||
*[f"| {k} | {pct(rec[k], rec_total[k])} |" for k in sorted(rec_total) if k not in ("端到端", "查标", "致命级")],
|
||||
"",
|
||||
"## 误报",
|
||||
"",
|
||||
f"- 规矩响应的要求 {ok_units} 条，报了问题的 {len(fp)} 条（每 100 条 {100 * len(fp) / ok_units:.1f}）"
|
||||
if ok_units
|
||||
else "- 没有规矩响应的要求",
|
||||
f"- 干扰项 {trap_units} 条，误报 {dict(fp_by_trap)}",
|
||||
f"- 要求本来不需要响应或无法判断（na / unknown）却报了问题：{other_findings}",
|
||||
f"- 无法对照（对不上任何标注单元）的问题：{len(unmapped)}",
|
||||
"",
|
||||
f"## 漏报（{len(misses)}）",
|
||||
"",
|
||||
*[f"- {x}" for x in misses],
|
||||
"",
|
||||
f"## 读标漏抽（{len(extraction_miss)}）",
|
||||
"",
|
||||
*[f"- {x}" for x in extraction_miss],
|
||||
"",
|
||||
f"## 误报明细（{len(fp)}）",
|
||||
"",
|
||||
*[f"- {x}" for x in fp[:80]],
|
||||
*(["- ……"] if len(fp) > 80 else []),
|
||||
"",
|
||||
f"## 规则问题（第 2 步才查，这里只列出）：{len(manifest['rule_defects'])} 个",
|
||||
"",
|
||||
]
|
||||
out = d / "runs" / a.run
|
||||
(out / "score.md").write_text("\n".join(lines))
|
||||
summary = {
|
||||
"recall": {k: [rec[k], rec_total[k]] for k in rec_total},
|
||||
"fp": len(fp),
|
||||
"ok_units": ok_units,
|
||||
"fp_by_trap": dict(fp_by_trap),
|
||||
"unmapped": len(unmapped),
|
||||
"other_findings": other_findings,
|
||||
"extraction_miss": len(extraction_miss),
|
||||
"include_unreviewed": a.include_unreviewed,
|
||||
}
|
||||
(out / "score.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
print("\n".join(lines[:20]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user