docs(evals): S01–S10 标注集 v1、评测工具和评测报告

- evals/gold:1,887 个标注单元(AI 标注、逐条裁定,未经人工复核),规范见 GUIDELINES.md
- prepare / run_all / score / gold:导出文字层、批量抽取、打分、标注定位
- docs/architecture/extract-eval.md:修复前后的结果(三项目标全部达标)和剩余问题
- decisions.md 记 M1(只用 Flash)和 Q1–Q4;架构 §13 SK13;AGENTS.md 进度

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
2026-11-03 23:19:30 +08:00
parent f98fd51128
commit 602703c47c
21 changed files with 29427 additions and 18 deletions

58
evals/prepare.py Normal file
View File

@@ -0,0 +1,58 @@
"""准备标注集评测的输入:S01–S10 转成规范化 PDF,导出带句子编号的全文(给人工标注用)。
用法(在 server/ 下):
../.venv/bin/python ../evals/prepare.py [S01 S02 ...]
输出 evals/out/docs/<ID>/:normalized.pdf、sentences.txt(每行 `[p12s3] 句子`,带格式标记和〔上级:…〕)。
规范化 PDF 生成一次后复用,抽取评测(run_extract.py)也用它作输入,标注和评测看到的是同一份文字层。
"""
from __future__ import annotations
import asyncio
import pathlib
import sys
from bid_doctor.checklist import extract
from bid_doctor.document import formats
from bid_doctor.document.pipeline import convert, structure, text_layer
ROOT = pathlib.Path(__file__).resolve().parent
SAMPLES = ROOT.parent / "samples" / "tenders"
OUT = ROOT / "out" / "docs"
async def prepare(doc_id: str) -> None:
out = OUT / doc_id
out.mkdir(parents=True, exist_ok=True)
pdf_path = out / "normalized.pdf"
if not pdf_path.exists():
src = next(p for p in (SAMPLES / doc_id).iterdir() if p.suffix in (".doc", ".docx", ".pdf"))
raw = src.read_bytes()
fmt = formats.detect(raw, src.name).format
pdf_path.write_bytes(raw if fmt == "pdf" else await convert.to_pdf(raw, fmt))
pages = text_layer.extract("D", pdf_path.read_bytes())
body = text_layer.body_sentences(pages)
parents = structure.parents(body)
notes = extract.marking_notes(body)
lines = [f"# {doc_id}:{len(pages)} 页,正文句子 {len(body)}", "## 标注规则说明(规则识别)"]
lines += [f"- [p{n.sentence.page_no}s{n.sentence.seq}] {n.sentence.text}" for n in notes] or ["(无)"]
page = 0
for s in body:
if s.page_no != page:
page = s.page_no
lines.append(f"\n=== 第 {page} 页")
line = f"[p{s.page_no}s{s.seq}] {s.marked}"
if p := parents.get(s.key):
line += f"〔上级:{p.text[:40]}〕"
lines.append(line)
(out / "sentences.txt").write_text("\n".join(lines) + "\n")
print(doc_id, len(pages), "页", len(body), "句")
async def main() -> None:
ids = sys.argv[1:] or [f"S{i:02d}" for i in range(1, 11)]
for doc_id in ids:
await prepare(doc_id)
asyncio.run(main())