db1b1f6f9f
- check/rules.py（规则集 2026.11-r1，不调模型）：大小写金额、分项合计、超最高限价、 多个报价、别的项目名称、没填的地方、落款日期、投标有效期、附加条件、响应表里的未响应； 每条挂投标原文，有依据的挂招标侧出处；缺限价、截止时间、有效期要求时写进覆盖范围 - 文字层的句子带上单元格框，表格按单元格上沿分行 - 按 Fable 审读意见收紧致命级：认不准的降为重要或提示 - 评测：score.py 评规则问题；P07 注入记录改为空白处前后原文，剔除注入到人员表的 D10 - 上传页的检查范围文案同步 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
"""汇总一批投标文件的查标评分（evals/check/GUIDELINES.md §7）。先对每份跑过 score.py。
|
||
|
||
cd server && ../.venv/bin/python ../evals/check/report.py --run base <投标文件> [...]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import pathlib
|
||
from collections import Counter
|
||
|
||
OUT = pathlib.Path(__file__).resolve().parents[1] / "out" / "check"
|
||
|
||
|
||
def pct(a: int, b: int) -> str:
|
||
return f"{a}/{b}（{a / b:.0%}）" if b else "—"
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("bids", nargs="+")
|
||
ap.add_argument("--run", default="base")
|
||
a = ap.parse_args()
|
||
rec: Counter[str] = Counter()
|
||
tot: Counter[str] = Counter()
|
||
by_style: Counter[str] = Counter()
|
||
by_style_tot: Counter[str] = Counter()
|
||
fp = ok = unmapped = other = 0
|
||
traps: Counter[str] = Counter()
|
||
tokens = [0, 0]
|
||
findings_clean = items_clean = 0
|
||
r_found: Counter[str] = Counter()
|
||
r_total: Counter[str] = Counter()
|
||
r_fp: Counter[str] = Counter()
|
||
r_fp_clean = r_clean = 0
|
||
for bid in a.bids:
|
||
d = OUT / bid
|
||
m = json.loads((d / "manifest.json").read_text())
|
||
s = json.loads((d / "runs" / a.run / "score.json").read_text())
|
||
r = json.loads((d / "runs" / a.run / "results.json").read_text())
|
||
tokens[0] += r["summary"]["usage"]["tokens_in"]
|
||
tokens[1] += r["summary"]["usage"]["tokens_out"]
|
||
for k, (hit, n) in s["recall"].items():
|
||
rec[k] += hit
|
||
tot[k] += n
|
||
if "查标" in s["recall"]:
|
||
by_style[m["style"]] += s["recall"]["查标"][0]
|
||
by_style_tot[m["style"]] += s["recall"]["查标"][1]
|
||
rules = s.get("rules") or {}
|
||
r_found.update(rules.get("found") or {})
|
||
r_total.update(rules.get("total") or {})
|
||
r_fp.update(rules.get("fp_by_code") or {})
|
||
if m["kind"] == "clean":
|
||
r_fp_clean += rules.get("fp", 0)
|
||
r_clean += 1
|
||
if m["kind"] == "clean":
|
||
fp += s["fp"]
|
||
ok += s["ok_units"]
|
||
other += s["other_findings"]
|
||
unmapped += s["unmapped"]
|
||
traps.update(s["fp_by_trap"])
|
||
findings_clean += sum(1 for it in r["items"] if it["finding"])
|
||
items_clean += len(r["items"])
|
||
lines = [
|
||
f"# 汇总：{len(a.bids)} 份投标文件 · 运行 {a.run}",
|
||
"",
|
||
f"- 查标召回：{pct(rec['查标'], tot['查标'])}；端到端：{pct(rec['端到端'], tot['端到端'])}；致命级：{pct(rec['致命级'], tot['致命级'])}",
|
||
"- 按风格：" + "；".join(f"{k} {pct(by_style[k], by_style_tot[k])}" for k in sorted(by_style_tot)),
|
||
"",
|
||
"| 分类 | 召回 |",
|
||
"|---|---|",
|
||
*[f"| {k} | {pct(rec[k], tot[k])} |" for k in sorted(tot) if k not in ("查标", "端到端", "致命级")],
|
||
f"| 召回里只是「扫描件待人工核对」的 | {rec['其中只是扫描件待人工核对']} |",
|
||
"",
|
||
f"- 规矩的投标文件：清单 {items_clean} 条，报出问题 {findings_clean} 个；其中规矩响应的要求上 {fp}/{ok}（每 100 条 {100 * fp / ok:.1f}）"
|
||
if ok
|
||
else "",
|
||
f"- 规矩的投标文件里，不需要响应（na / unknown）的要求上报了问题 {other} 个，对不上标注单元的问题 {unmapped} 个",
|
||
f"- 干扰项误报：{dict(traps)}",
|
||
f"- token：输入 {tokens[0]:,}，输出 {tokens[1]:,}（{len(a.bids)} 次检查）",
|
||
"",
|
||
f"## 规则问题：查出 {pct(sum(r_found.values()), sum(r_total.values()))}；误报 {sum(r_fp.values())} 个"
|
||
f"（其中规矩的投标文件 {r_clean} 份上 {r_fp_clean} 个）",
|
||
"",
|
||
"| 问题 | 查出 |",
|
||
"|---|---|",
|
||
*[f"| {k} | {pct(r_found[k], r_total[k])} |" for k in sorted(r_total)],
|
||
"",
|
||
f"- 误报按规则：{dict(r_fp)}",
|
||
]
|
||
print("\n".join(lines))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|