"""准备标注集评测的输入:S01–S10 转成规范化 PDF,导出带句子编号的全文(给人工标注用)。 用法(在 server/ 下): ../.venv/bin/python ../evals/prepare.py [S01 S02 ...] 输出 evals/out/docs//:normalized.pdf、sentences.txt(每行 `[p12s3] 句子`,带格式标记和〔上级:…〕)。 规范化 PDF 生成一次后复用,抽取评测(run_extract.py)也用它作输入,标注和评测看到的是同一份文字层。 """ from __future__ import annotations import asyncio import pathlib import sys from bid_doctor.checklist import extract from bid_doctor.document import formats from bid_doctor.document.pipeline import convert, structure, text_layer ROOT = pathlib.Path(__file__).resolve().parent SAMPLES = ROOT.parent / "samples" / "tenders" OUT = ROOT / "out" / "docs" async def prepare(doc_id: str) -> None: out = OUT / doc_id out.mkdir(parents=True, exist_ok=True) pdf_path = out / "normalized.pdf" if not pdf_path.exists(): src = next(p for p in (SAMPLES / doc_id).iterdir() if p.suffix in (".doc", ".docx", ".pdf")) raw = src.read_bytes() fmt = formats.detect(raw, src.name).format pdf_path.write_bytes(raw if fmt == "pdf" else await convert.to_pdf(raw, fmt)) pages = text_layer.extract("D", pdf_path.read_bytes()) body = text_layer.body_sentences(pages) parents = structure.parents(body) notes = extract.marking_notes(body) lines = [f"# {doc_id}:{len(pages)} 页,正文句子 {len(body)}", "## 标注规则说明(规则识别)"] lines += [f"- [p{n.sentence.page_no}s{n.sentence.seq}] {n.sentence.text}" for n in notes] or ["(无)"] page = 0 for s in body: if s.page_no != page: page = s.page_no lines.append(f"\n=== 第 {page} 页") line = f"[p{s.page_no}s{s.seq}] {s.marked}" if p := parents.get(s.key): line += f"〔上级:{p.text[:40]}〕" lines.append(line) (out / "sentences.txt").write_text("\n".join(lines) + "\n") print(doc_id, len(pages), "页", len(body), "句") async def main() -> None: ids = sys.argv[1:] or [f"S{i:02d}" for i in range(1, 11)] for doc_id in ids: await prepare(doc_id) asyncio.run(main())