df3ebbb6a1
- deviation.py：模型判已响应的条目，证据所在表格行的偏离格写着「偏离 / 部分满足」等 就改判负偏离；照抄要求格、响应格写的是别的内容的改判「响应可能对不上」 - reuse.py：按「提示词 + 要求 + 候选句原文」算缓存键，存模型原始结论；复查时内容没变、 证据找得回的条目直接沿用，不调模型 - 每次检查送给模型的字数有上限，超出的条目列为请人工核对，写进覆盖范围 - 开发集 17 份：召回 61% → 73%，负偏离未说明 64% → 90%，规矩投标文件问题数不变 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
320 lines
13 KiB
Python
320 lines
13 KiB
Python
"""离线查标评测：拿读标评测的清单（evals/out/runs/<运行>/<招标文件>/items.json）对一份虚构投标文件跑响应匹配。
|
||
|
||
用法（在 server/ 下，模型配置读根目录 .env）：
|
||
../.venv/bin/python ../evals/check/run_check.py <投标文件编号> [--checklist fix3] [--name <运行名>]
|
||
输入：evals/out/check/<投标文件编号>/v*.pdf（render.py 渲染的）、manifest.json 里的 tender。
|
||
输出：evals/out/check/<投标文件编号>/runs/<运行名>/results.json（每条清单项的结论、投标侧证据、生成的问题）。
|
||
同一模型 + 同一提示词版本 + 同一输入只调用一次模型（和读标评测共用 evals/out/cache/），只改规则时重放不花钱。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import datetime as dt
|
||
import hashlib
|
||
import json
|
||
import pathlib
|
||
import re
|
||
import sys
|
||
from decimal import Decimal
|
||
from typing import Any
|
||
from uuid import UUID, uuid4
|
||
|
||
from bid_doctor.ai import prompts
|
||
from bid_doctor.ai import service as ai_service
|
||
from bid_doctor.ai.providers.base import near_limit
|
||
from bid_doctor.ai.service import OutputIncomplete, provider
|
||
from bid_doctor.check import applicability, deviation, findings, match, rules
|
||
from bid_doctor.checklist import response_mode
|
||
from bid_doctor.checklist import price
|
||
from bid_doctor.checklist.extract import amount_yuan
|
||
from bid_doctor.core.config import get_settings
|
||
from bid_doctor.core.db import TenantContext
|
||
from bid_doctor.document.pipeline import text_layer
|
||
|
||
EVALS = pathlib.Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(EVALS))
|
||
import gold # noqa: E402
|
||
|
||
OUT = EVALS / "out" / "check"
|
||
CACHE = EVALS / "out" / "cache"
|
||
usage = {"calls": 0, "cached": 0, "tokens_in": 0, "tokens_out": 0}
|
||
|
||
|
||
async def cached_run_json(
|
||
ctx: Any, *, job_type: str, job_id: Any, task: str, prompt_id: str, version: int, user: str
|
||
) -> dict[str, Any]:
|
||
del ctx, job_type, job_id
|
||
p = prompts.get(prompt_id, version)
|
||
s = get_settings()
|
||
model = s.ai_models.get(p.capability) or s.ai_model
|
||
key = hashlib.sha256(f"{model}\n{prompt_id}\n{version}\n{p.body}\n{user}".encode()).hexdigest()[:32]
|
||
f = CACHE / f"{key}.json"
|
||
if f.exists():
|
||
data = json.loads(f.read_text())
|
||
usage["cached"] += 1
|
||
else:
|
||
r = await provider().complete_json(p.body, user, p.capability, task)
|
||
data = {
|
||
"data": r.data,
|
||
"model": r.model,
|
||
"tokens_in": r.tokens_in,
|
||
"tokens_out": r.tokens_out,
|
||
"incomplete": r.incomplete,
|
||
}
|
||
f.write_text(json.dumps(data, ensure_ascii=False))
|
||
usage["calls"] += 1
|
||
usage["tokens_in"] += data.get("tokens_in") or 0
|
||
usage["tokens_out"] += data.get("tokens_out") or 0
|
||
if data.get("incomplete") or near_limit(data.get("tokens_out")):
|
||
raise OutputIncomplete(task, data["data"])
|
||
return data["data"]
|
||
|
||
|
||
ITEM = re.compile(r"^\[(I\d+)\] ")
|
||
CANDS = re.compile(r"^候选句：(.*)$")
|
||
FRAG = re.compile(r"^\[(d\d+p\d+s\d+)\]（[^）]*）(.*)$")
|
||
|
||
|
||
async def baseline_run_json(
|
||
ctx: Any, *, job_type: str, job_id: Any, task: str, prompt_id: str, version: int, user: str
|
||
) -> dict[str, Any]:
|
||
"""只检索基线（GUIDELINES §7）：不调模型，检索第一名就当作已响应。"""
|
||
del ctx, job_type, job_id, task, prompt_id, version
|
||
frags = {m.group(1): m.group(2) for line in user.splitlines() if (m := FRAG.match(line.strip()))}
|
||
results, key = [], None
|
||
for line in user.splitlines():
|
||
if m := ITEM.match(line.strip()):
|
||
key = m.group(1)
|
||
elif (m := CANDS.match(line.strip())) and key:
|
||
first = m.group(1).split("、")[0]
|
||
results.append(
|
||
{"item": key, "verdict": "responded", "evidence": [{"sid": first, "quote": frags.get(first, "")}]}
|
||
)
|
||
return {"results": results}
|
||
|
||
|
||
def checklist(tender: str, run: str) -> dict[UUID, dict[str, Any]]:
|
||
"""读标评测输出的正式清单 → 查标的清单项（和检查快照的 payload 同样的字段）。"""
|
||
rows = json.loads((EVALS / "out" / "runs" / run / tender / "items.json").read_text())
|
||
by_sid = {gold.sid(s): s for s in gold.sentences(tender)}
|
||
out: dict[UUID, dict[str, Any]] = {}
|
||
for i, r in enumerate(rows):
|
||
if r.get("zone") != "formal":
|
||
continue
|
||
text = "".join(by_sid[k].text for k in r.get("sentences") or [] if k in by_sid)
|
||
raw = (r.get("raw") or {}).get("tech_param") or {}
|
||
spec = {"tech_param": {k: raw[k] for k in ("param", "op", "value", "unit") if raw.get(k)}} if raw else {}
|
||
sched = (r.get("raw") or {}).get("schedule") or {}
|
||
if sched:
|
||
spec["schedule"] = {k: sched[k] for k in ("date_kind", "at") if sched.get(k)}
|
||
out[UUID(int=i + 1)] = {
|
||
"title": r.get("title"),
|
||
"requirement_text": text or None,
|
||
"categories": r.get("categories") or [],
|
||
"importance": r.get("importance"),
|
||
"importance_basis": r.get("importance_basis"),
|
||
"spec": spec,
|
||
"sentences": r.get("sentences") or [],
|
||
}
|
||
return out
|
||
|
||
|
||
def facts(tender: str, run: str, items: dict[UUID, dict[str, Any]]) -> rules.Facts:
|
||
"""项目侧的值：读标评测抽出的项目基本信息（和产品里一样来自抽取）；所投的包按招标档案（相当于用户选包）。"""
|
||
base = EVALS / "out" / "runs" / run / tender
|
||
info = json.loads((base / "info.json").read_text())
|
||
f = {k: v.get("value") for k, v in info["fields"].items()}
|
||
profile = json.loads((OUT / "tenders" / f"{tender}.json").read_text())["project"]
|
||
deadline = next(
|
||
(
|
||
dt.date.fromisoformat(s["at"][:10])
|
||
for r in json.loads((base / "items.json").read_text())
|
||
if r.get("zone") == "formal"
|
||
and (s := (r.get("raw") or {}).get("schedule") or {}).get("date_kind") == "bid_deadline"
|
||
and s.get("at")
|
||
),
|
||
None,
|
||
)
|
||
lots = info.get("lots") or []
|
||
ceiling, scope, note = None, "total", None
|
||
if lots:
|
||
lot = next(
|
||
(x for x in lots if str(x["lot_no"]).lstrip("0") == str(profile.get("bidding_lot", "")).lstrip("0")), None
|
||
)
|
||
if lot is None and len(lots) == 1:
|
||
lot = lots[0]
|
||
if lot is not None:
|
||
c = amount_yuan((lot.get("ceiling") or {}).get("value"), (lot.get("ceiling") or {}).get("unit"))
|
||
ceiling, scope = (Decimal(c) if c else None), "lot" if len(lots) > 1 else "total"
|
||
else:
|
||
note = "所投采购包不明确，报价没有和限价比对"
|
||
else:
|
||
c = amount_yuan(f.get("ceiling_total"), info["fields"].get("ceiling_total", {}).get("unit"))
|
||
ceiling = Decimal(c) if c else None
|
||
days, item = rules.required_validity(items)
|
||
lot_no = str(profile.get("bidding_lot") or "").lstrip("0")
|
||
caps = [
|
||
rules.ItemCap(c.name, c.cap, c.per)
|
||
for c in price.caps(gold.sentences(tender))
|
||
if not c.lot_no or not lot_no or c.lot_no.lstrip("0") == lot_no
|
||
]
|
||
return rules.Facts(
|
||
name=f.get("name"),
|
||
code=f.get("code"),
|
||
purchaser=f.get("purchaser"),
|
||
agency=f.get("agency"),
|
||
deadline=deadline,
|
||
ceiling=ceiling,
|
||
ceiling_scope=scope,
|
||
ceiling_note=note,
|
||
validity_days=days,
|
||
validity_item=item,
|
||
caps=caps,
|
||
)
|
||
|
||
|
||
async def main() -> None:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("bid")
|
||
ap.add_argument("--checklist", default="fix3")
|
||
ap.add_argument("--name", default="base")
|
||
ap.add_argument("--baseline", action="store_true", help="只检索、不调模型的基线")
|
||
ap.add_argument("--v1", action="store_true", help="第 1 步的做法：提示词 v1、不带同一行的单元格（对比基线）")
|
||
ap.add_argument("--no-modes", action="store_true", help="不按响应方式和「要不要回答」筛选（C12 之前的做法）")
|
||
ap.add_argument("--rules-only", action="store_true", help="只重跑规则检查，写回已有运行的 results.json")
|
||
ap.add_argument("--no-direct", action="store_true", help="不做偏离栏直读（对比用）")
|
||
ap.add_argument("--no-mismatch", action="store_true", help="不做响应错位判断（对比用）")
|
||
args = ap.parse_args()
|
||
d = OUT / args.bid
|
||
manifest = json.loads((d / "manifest.json").read_text())
|
||
items = checklist(manifest["tender"], args.checklist)
|
||
docs = []
|
||
for n, pdf in enumerate(sorted(d.glob("v*.pdf")), start=1):
|
||
pages = text_layer.extract(f"{args.bid}-{pdf.stem}", pdf.read_bytes())
|
||
docs.append(
|
||
match.BidDoc(
|
||
uuid4(),
|
||
n,
|
||
pdf.stem,
|
||
text_layer.body_sentences(pages),
|
||
[p.page_no for p in pages if p.text_source == "empty"],
|
||
[s for p in pages for s in p.sentences if s.location == "header_footer"],
|
||
)
|
||
)
|
||
hits, rules_cov = rules.run(
|
||
facts(manifest["tender"], args.checklist, items), docs, gold.sentences(manifest["tender"])
|
||
)
|
||
|
||
def rule_rows(covered: set[tuple[str, int, str]]) -> list[dict[str, Any]]:
|
||
return [
|
||
{
|
||
"code": h.code,
|
||
"type": h.type,
|
||
"severity": h.severity,
|
||
"title": h.title,
|
||
"explanation": h.explanation,
|
||
"evidence": [{"volume": doc.label, "p": s.page_no, "text": s.text} for doc, s in h.evidence],
|
||
}
|
||
for h in rules.drop_covered(hits, covered)
|
||
]
|
||
|
||
if args.rules_only:
|
||
prev = d / "runs" / args.name / "results.json"
|
||
data = json.loads(prev.read_text())
|
||
covered = {
|
||
(e["volume"], e["p"], "".join(e["text"].split()))
|
||
for it in data["items"]
|
||
if it["verdict"] == "negative_deviation"
|
||
for e in it["evidence"]
|
||
}
|
||
data["rules"], data["rules_coverage"] = rule_rows(covered), rules_cov
|
||
prev.write_text(json.dumps(data, ensure_ascii=False, indent=1))
|
||
print(f"{args.bid}：规则问题 {len(data['rules'])} 个")
|
||
return
|
||
ai_service.run_json = baseline_run_json if args.baseline else cached_run_json # type: ignore[assignment]
|
||
if args.v1:
|
||
match.PROMPT = ("check.match", 1)
|
||
match.ROW_MATES = 0
|
||
|
||
async def progress(_: float) -> None:
|
||
return None
|
||
|
||
ctx = TenantContext(tenant_id=UUID(int=0), user_id=UUID(int=0))
|
||
all_items = items
|
||
decisions: dict[UUID, applicability.Decision] = {}
|
||
if not args.no_modes:
|
||
modes = response_mode.detect(gold.sentences(manifest["tender"]))
|
||
decisions = {i: applicability.decide(p, modes) for i, p in items.items()}
|
||
items = {i: p for i, p in items.items() if decisions[i].match}
|
||
results, stats = await match.run(ctx, uuid4(), items, docs, progress)
|
||
if not args.no_direct:
|
||
deviation.direct_read(results, docs)
|
||
if not args.no_mismatch:
|
||
deviation.mismatch_read(results, items, docs)
|
||
if decisions:
|
||
match.apply_modes(results, items, decisions)
|
||
match.scanned_fallback(results, items, decisions, docs)
|
||
rows = []
|
||
for item_id, item in all_items.items():
|
||
if item_id not in items:
|
||
rows.append(
|
||
{
|
||
"title": item["title"],
|
||
"sentences": item["sentences"],
|
||
"categories": item["categories"],
|
||
"importance": item["importance"],
|
||
"verdict": f"skipped:{decisions[item_id].skip}",
|
||
"layer": None,
|
||
"evidence_status": "none",
|
||
"reason_code": None,
|
||
"evidence": [],
|
||
"detail": {},
|
||
"finding": None,
|
||
}
|
||
)
|
||
continue
|
||
res = results[item_id]
|
||
spec = findings.spec(res, item)
|
||
rows.append(
|
||
{
|
||
"title": item["title"],
|
||
"sentences": item["sentences"],
|
||
"categories": item["categories"],
|
||
"importance": item["importance"],
|
||
"verdict": res.verdict,
|
||
"layer": res.layer,
|
||
"evidence_status": res.evidence_status,
|
||
"reason_code": res.reason_code,
|
||
"evidence": [{"volume": doc.label, "p": s.page_no, "text": s.text} for doc, s in res.evidence],
|
||
"detail": res.detail,
|
||
"finding": {"type": spec.type, "severity": spec.severity, "title": spec.title} if spec else None,
|
||
}
|
||
)
|
||
out = d / "runs" / args.name
|
||
out.mkdir(parents=True, exist_ok=True)
|
||
summary = {"items": len(rows), "usage": usage, "verify_failures": stats.verify_failures, "splits": stats.splits}
|
||
covered = {
|
||
(doc.label, s.page_no, "".join(s.text.split()))
|
||
for res in results.values()
|
||
if res.verdict == "negative_deviation"
|
||
for doc, s in res.evidence
|
||
}
|
||
(out / "results.json").write_text(
|
||
json.dumps(
|
||
{"summary": summary, "items": rows, "rules": rule_rows(covered), "rules_coverage": rules_cov},
|
||
ensure_ascii=False,
|
||
indent=1,
|
||
)
|
||
)
|
||
findings_n = sum(r["finding"] is not None for r in rows)
|
||
print(
|
||
f"{args.bid}：清单 {len(rows)} 条，问题 {findings_n} 个，模型调用 {usage['calls']} 次（缓存 {usage['cached']}），"
|
||
f"token {usage['tokens_in']} / {usage['tokens_out']}"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|