feat(evals): 查标评测集生成器、注入、评分和只检索基线
- build.py:从「招标文件档案」和读标清单生成规矩投标文件稿;技术参数按 op / value / unit 算满足的值, 分项报价按分项上限比例分摊,落款日期取截止日前 3 天,偏离表按格式表头决定列(含设备名称列), 读标漏抽的技术参数也补上响应;routes 支持「+」组合和 prose 散文响应。 - mutate.py:按词库注入要求项问题(单点 / 连续块,带不带标志词)和规则问题,写响应清单; 符合性条件响应表的问题按表级记一个。 - score.py:按单元去重、类型族比对、读标漏抽单列、端到端召回、无法对照单列。 - run_check.py:加只检索不调模型的基线;render.py:扫描件按最后一次出现定位。 - vocab-dev.json:开发集注入词库(留出集的另行封存,不进 git)。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,7 @@ import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
@@ -66,6 +67,29 @@ async def cached_run_json(
|
||||
return data["data"]
|
||||
|
||||
|
||||
ITEM = re.compile(r"^\[(I\d+)\] ")
|
||||
CANDS = re.compile(r"^候选句:(.*)$")
|
||||
FRAG = re.compile(r"^\[(d\d+p\d+s\d+)\]([^)]*)(.*)$")
|
||||
|
||||
|
||||
async def baseline_run_json(
|
||||
ctx: Any, *, job_type: str, job_id: Any, task: str, prompt_id: str, version: int, user: str
|
||||
) -> dict[str, Any]:
|
||||
"""只检索基线(GUIDELINES §7):不调模型,检索第一名就当作已响应。"""
|
||||
del ctx, job_type, job_id, task, prompt_id, version
|
||||
frags = {m.group(1): m.group(2) for line in user.splitlines() if (m := FRAG.match(line.strip()))}
|
||||
results, key = [], None
|
||||
for line in user.splitlines():
|
||||
if m := ITEM.match(line.strip()):
|
||||
key = m.group(1)
|
||||
elif (m := CANDS.match(line.strip())) and key:
|
||||
first = m.group(1).split("、")[0]
|
||||
results.append(
|
||||
{"item": key, "verdict": "responded", "evidence": [{"sid": first, "quote": frags.get(first, "")}]}
|
||||
)
|
||||
return {"results": results}
|
||||
|
||||
|
||||
def checklist(tender: str, run: str) -> dict[UUID, dict[str, Any]]:
|
||||
"""读标评测输出的正式清单 → 查标的清单项(和检查快照的 payload 同样的字段)。"""
|
||||
rows = json.loads((EVALS / "out" / "runs" / run / tender / "items.json").read_text())
|
||||
@@ -94,6 +118,7 @@ async def main() -> None:
|
||||
ap.add_argument("bid")
|
||||
ap.add_argument("--checklist", default="fix3")
|
||||
ap.add_argument("--name", default="base")
|
||||
ap.add_argument("--baseline", action="store_true", help="只检索、不调模型的基线")
|
||||
args = ap.parse_args()
|
||||
d = OUT / args.bid
|
||||
manifest = json.loads((d / "manifest.json").read_text())
|
||||
@@ -102,7 +127,7 @@ async def main() -> None:
|
||||
for n, pdf in enumerate(sorted(d.glob("v*.pdf")), start=1):
|
||||
pages = text_layer.extract(f"{args.bid}-{pdf.stem}", pdf.read_bytes())
|
||||
docs.append(match.BidDoc(uuid4(), n, pdf.stem, text_layer.body_sentences(pages)))
|
||||
ai_service.run_json = cached_run_json # type: ignore[assignment]
|
||||
ai_service.run_json = baseline_run_json if args.baseline else cached_run_json # type: ignore[assignment]
|
||||
|
||||
async def progress(_: float) -> None:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user