76 lines
3.0 KiB
Python
76 lines
3.0 KiB
Python
"""汇总一批投标文件的查标评分（evals/check/GUIDELINES.md §7）。先对每份跑过 score.py。
|
||
|
||
cd server && ../.venv/bin/python ../evals/check/report.py --run base <投标文件> [...]
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import pathlib
|
||
from collections import Counter
|
||
|
||
OUT = pathlib.Path(__file__).resolve().parents[1] / "out" / "check"
|
||
|
||
|
||
def pct(a: int, b: int) -> str:
|
||
return f"{a}/{b}（{a / b:.0%}）" if b else "—"
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("bids", nargs="+")
|
||
ap.add_argument("--run", default="base")
|
||
a = ap.parse_args()
|
||
rec: Counter[str] = Counter()
|
||
tot: Counter[str] = Counter()
|
||
by_style: Counter[str] = Counter()
|
||
by_style_tot: Counter[str] = Counter()
|
||
fp = ok = unmapped = other = 0
|
||
traps: Counter[str] = Counter()
|
||
tokens = [0, 0]
|
||
findings_clean = items_clean = 0
|
||
for bid in a.bids:
|
||
d = OUT / bid
|
||
m = json.loads((d / "manifest.json").read_text())
|
||
s = json.loads((d / "runs" / a.run / "score.json").read_text())
|
||
r = json.loads((d / "runs" / a.run / "results.json").read_text())
|
||
tokens[0] += r["summary"]["usage"]["tokens_in"]
|
||
tokens[1] += r["summary"]["usage"]["tokens_out"]
|
||
for k, (hit, n) in s["recall"].items():
|
||
rec[k] += hit
|
||
tot[k] += n
|
||
if "查标" in s["recall"]:
|
||
by_style[m["style"]] += s["recall"]["查标"][0]
|
||
by_style_tot[m["style"]] += s["recall"]["查标"][1]
|
||
if m["kind"] == "clean":
|
||
fp += s["fp"]
|
||
ok += s["ok_units"]
|
||
other += s["other_findings"]
|
||
unmapped += s["unmapped"]
|
||
traps.update(s["fp_by_trap"])
|
||
findings_clean += sum(1 for it in r["items"] if it["finding"])
|
||
items_clean += len(r["items"])
|
||
lines = [
|
||
f"# 汇总：{len(a.bids)} 份投标文件 · 运行 {a.run}",
|
||
"",
|
||
f"- 查标召回：{pct(rec['查标'], tot['查标'])}；端到端：{pct(rec['端到端'], tot['端到端'])}；致命级：{pct(rec['致命级'], tot['致命级'])}",
|
||
"- 按风格：" + "；".join(f"{k} {pct(by_style[k], by_style_tot[k])}" for k in sorted(by_style_tot)),
|
||
"",
|
||
"| 分类 | 召回 |",
|
||
"|---|---|",
|
||
*[f"| {k} | {pct(rec[k], tot[k])} |" for k in sorted(tot) if k not in ("查标", "端到端", "致命级")],
|
||
"",
|
||
f"- 规矩的投标文件：清单 {items_clean} 条，报出问题 {findings_clean} 个；其中规矩响应的要求上 {fp}/{ok}（每 100 条 {100 * fp / ok:.1f}）"
|
||
if ok
|
||
else "",
|
||
f"- 规矩的投标文件里，不需要响应（na / unknown）的要求上报了问题 {other} 个，对不上标注单元的问题 {unmapped} 个",
|
||
f"- 干扰项误报：{dict(traps)}",
|
||
f"- token：输入 {tokens[0]:,}，输出 {tokens[1]:,}（{len(a.bids)} 次检查）",
|
||
]
|
||
print("\n".join(lines))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|