docs(evals): 查标评测集规范 v0.2、留出集划分；渲染和离线查标脚本；新增待定事项 C12

- evals/check/GUIDELINES.md：请 Fable 审查 v0.1 后修改。响应方式按「文件 × 类别」记（逐条 / 默认接受 / 证明材料（投标时、中标后）/ 文件要在 / 平台内应答），10 份招标文件逐份核对；规矩投标文件靠构造保证（表格脚本生成，LLM 只写散文和改写），对全部要求响应，问题只挂人工复核过的标注单元；注入分单点 / 连续块 / 全局三种粒度、带不带标志词、两档密度；留出集注入词库和 C 风格说明开工时封存；评分补只检索基线、按单元去重、端到端召回。 - evals/check/split.json：开发集 S01 S02 S04 S05 S07 S08 S10，留出集 S03 S06 S09 + 真实国动投标文件。 - evals/check/render.py、run_check.py。 - m3-plan.md：新增待定事项 C12（每条要求怎么才算「响应了」）。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 12:02:26 +08:00
parent e499de7486
commit 4ed945c1de
6 changed files with 588 additions and 1 deletions
--- a/evals/check/run_check.py
+++ b/evals/check/run_check.py
@@ -0,0 +1,143 @@
+"""离线查标评测：拿读标评测的清单（evals/out/runs/<运行>/<招标文件>/items.json）对一份虚构投标文件跑响应匹配。
+
+用法（在 server/ 下，模型配置读根目录 .env）：
+  ../.venv/bin/python ../evals/check/run_check.py <投标文件编号> [--checklist fix3] [--name <运行名>]
+输入：evals/out/check/<投标文件编号>/v*.pdf（render.py 渲染的）、manifest.json 里的 tender。
+输出：evals/out/check/<投标文件编号>/runs/<运行名>/results.json（每条清单项的结论、投标侧证据、生成的问题）。
+同一模型 + 同一提示词版本 + 同一输入只调用一次模型（和读标评测共用 evals/out/cache/），只改规则时重放不花钱。
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hashlib
+import json
+import pathlib
+import sys
+from typing import Any
+from uuid import UUID, uuid4
+
+from bid_doctor.ai import prompts
+from bid_doctor.ai import service as ai_service
+from bid_doctor.ai.providers.base import near_limit
+from bid_doctor.ai.service import OutputIncomplete, provider
+from bid_doctor.check import findings, match
+from bid_doctor.core.config import get_settings
+from bid_doctor.core.db import TenantContext
+from bid_doctor.document.pipeline import text_layer
+
+EVALS = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(EVALS))
+import gold  # noqa: E402
+
+OUT = EVALS / "out" / "check"
+CACHE = EVALS / "out" / "cache"
+usage = {"calls": 0, "cached": 0, "tokens_in": 0, "tokens_out": 0}
+
+
+async def cached_run_json(
+    ctx: Any, *, job_type: str, job_id: Any, task: str, prompt_id: str, version: int, user: str
+) -> dict[str, Any]:
+    del ctx, job_type, job_id
+    p = prompts.get(prompt_id, version)
+    s = get_settings()
+    model = s.ai_models.get(p.capability) or s.ai_model
+    key = hashlib.sha256(f"{model}\n{prompt_id}\n{version}\n{p.body}\n{user}".encode()).hexdigest()[:32]
+    f = CACHE / f"{key}.json"
+    if f.exists():
+        data = json.loads(f.read_text())
+        usage["cached"] += 1
+    else:
+        r = await provider().complete_json(p.body, user, p.capability, task)
+        data = {
+            "data": r.data,
+            "model": r.model,
+            "tokens_in": r.tokens_in,
+            "tokens_out": r.tokens_out,
+            "incomplete": r.incomplete,
+        }
+        f.write_text(json.dumps(data, ensure_ascii=False))
+    usage["calls"] += 1
+    usage["tokens_in"] += data.get("tokens_in") or 0
+    usage["tokens_out"] += data.get("tokens_out") or 0
+    if data.get("incomplete") or near_limit(data.get("tokens_out")):
+        raise OutputIncomplete(task, data["data"])
+    return data["data"]
+
+
+def checklist(tender: str, run: str) -> dict[UUID, dict[str, Any]]:
+    """读标评测输出的正式清单 → 查标的清单项（和检查快照的 payload 同样的字段）。"""
+    rows = json.loads((EVALS / "out" / "runs" / run / tender / "items.json").read_text())
+    by_sid = {gold.sid(s): s for s in gold.sentences(tender)}
+    out: dict[UUID, dict[str, Any]] = {}
+    for i, r in enumerate(rows):
+        if r.get("zone") != "formal":
+            continue
+        text = "".join(by_sid[k].text for k in r.get("sentences") or [] if k in by_sid)
+        raw = (r.get("raw") or {}).get("tech_param") or {}
+        spec = {"tech_param": {k: raw[k] for k in ("param", "op", "value", "unit") if raw.get(k)}} if raw else {}
+        out[UUID(int=i + 1)] = {
+            "title": r.get("title"),
+            "requirement_text": text or None,
+            "categories": r.get("categories") or [],
+            "importance": r.get("importance"),
+            "importance_basis": r.get("importance_basis"),
+            "spec": spec,
+            "sentences": r.get("sentences") or [],
+        }
+    return out
+
+
+async def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("bid")
+    ap.add_argument("--checklist", default="fix3")
+    ap.add_argument("--name", default="base")
+    args = ap.parse_args()
+    d = OUT / args.bid
+    manifest = json.loads((d / "manifest.json").read_text())
+    items = checklist(manifest["tender"], args.checklist)
+    docs = []
+    for n, pdf in enumerate(sorted(d.glob("v*.pdf")), start=1):
+        pages = text_layer.extract(f"{args.bid}-{pdf.stem}", pdf.read_bytes())
+        docs.append(match.BidDoc(uuid4(), n, pdf.stem, text_layer.body_sentences(pages)))
+    ai_service.run_json = cached_run_json  # type: ignore[assignment]
+
+    async def progress(_: float) -> None:
+        return None
+
+    ctx = TenantContext(tenant_id=UUID(int=0), user_id=UUID(int=0))
+    results, stats = await match.run(ctx, uuid4(), items, docs, progress)
+    rows = []
+    for item_id, item in items.items():
+        res = results[item_id]
+        spec = findings.spec(res, item)
+        rows.append(
+            {
+                "title": item["title"],
+                "sentences": item["sentences"],
+                "categories": item["categories"],
+                "importance": item["importance"],
+                "verdict": res.verdict,
+                "layer": res.layer,
+                "evidence_status": res.evidence_status,
+                "reason_code": res.reason_code,
+                "evidence": [{"volume": doc.label, "p": s.page_no, "text": s.text} for doc, s in res.evidence],
+                "detail": res.detail,
+                "finding": {"type": spec.type, "severity": spec.severity, "title": spec.title} if spec else None,
+            }
+        )
+    out = d / "runs" / args.name
+    out.mkdir(parents=True, exist_ok=True)
+    summary = {"items": len(rows), "usage": usage, "verify_failures": stats.verify_failures, "splits": stats.splits}
+    (out / "results.json").write_text(json.dumps({"summary": summary, "items": rows}, ensure_ascii=False, indent=1))
+    findings_n = sum(r["finding"] is not None for r in rows)
+    print(
+        f"{args.bid}：清单 {len(rows)} 条，问题 {findings_n} 个，模型调用 {usage['calls']} 次（缓存 {usage['cached']}），"
+        f"token {usage['tokens_in']} / {usage['tokens_out']}"
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())