docs(evals): S01–S10 标注集 v1、评测工具和评测报告
- evals/gold:1,887 个标注单元(AI 标注、逐条裁定,未经人工复核),规范见 GUIDELINES.md - prepare / run_all / score / gold:导出文字层、批量抽取、打分、标注定位 - docs/architecture/extract-eval.md:修复前后的结果(三项目标全部达标)和剩余问题 - decisions.md 记 M1(只用 Flash)和 Q1–Q4;架构 §13 SK13;AGENTS.md 进度 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
58
evals/prepare.py
Normal file
58
evals/prepare.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""准备标注集评测的输入:S01–S10 转成规范化 PDF,导出带句子编号的全文(给人工标注用)。
|
||||
|
||||
用法(在 server/ 下):
|
||||
../.venv/bin/python ../evals/prepare.py [S01 S02 ...]
|
||||
输出 evals/out/docs/<ID>/:normalized.pdf、sentences.txt(每行 `[p12s3] 句子`,带格式标记和〔上级:…〕)。
|
||||
规范化 PDF 生成一次后复用,抽取评测(run_extract.py)也用它作输入,标注和评测看到的是同一份文字层。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from bid_doctor.checklist import extract
|
||||
from bid_doctor.document import formats
|
||||
from bid_doctor.document.pipeline import convert, structure, text_layer
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parent
|
||||
SAMPLES = ROOT.parent / "samples" / "tenders"
|
||||
OUT = ROOT / "out" / "docs"
|
||||
|
||||
|
||||
async def prepare(doc_id: str) -> None:
|
||||
out = OUT / doc_id
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
pdf_path = out / "normalized.pdf"
|
||||
if not pdf_path.exists():
|
||||
src = next(p for p in (SAMPLES / doc_id).iterdir() if p.suffix in (".doc", ".docx", ".pdf"))
|
||||
raw = src.read_bytes()
|
||||
fmt = formats.detect(raw, src.name).format
|
||||
pdf_path.write_bytes(raw if fmt == "pdf" else await convert.to_pdf(raw, fmt))
|
||||
pages = text_layer.extract("D", pdf_path.read_bytes())
|
||||
body = text_layer.body_sentences(pages)
|
||||
parents = structure.parents(body)
|
||||
notes = extract.marking_notes(body)
|
||||
lines = [f"# {doc_id}:{len(pages)} 页,正文句子 {len(body)}", "## 标注规则说明(规则识别)"]
|
||||
lines += [f"- [p{n.sentence.page_no}s{n.sentence.seq}] {n.sentence.text}" for n in notes] or ["(无)"]
|
||||
page = 0
|
||||
for s in body:
|
||||
if s.page_no != page:
|
||||
page = s.page_no
|
||||
lines.append(f"\n=== 第 {page} 页")
|
||||
line = f"[p{s.page_no}s{s.seq}] {s.marked}"
|
||||
if p := parents.get(s.key):
|
||||
line += f"〔上级:{p.text[:40]}〕"
|
||||
lines.append(line)
|
||||
(out / "sentences.txt").write_text("\n".join(lines) + "\n")
|
||||
print(doc_id, len(pages), "页", len(body), "句")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
ids = sys.argv[1:] or [f"S{i:02d}" for i in range(1, 11)]
|
||||
for doc_id in ids:
|
||||
await prepare(doc_id)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user