docs(architecture): 完成技术验证 S1–S4,确定 FR-M2-05/06 目标值
- spikes/:Word 转 PDF(Gotenberg)、格式识别、原句坐标、DeepSeek 抽取四项验证代码与两版提示词;样本派生产物和 .env 不进 git - tech-validation.md:完整结论。四项都可行:WPS 字体字符集要修正并改用行高调整过的字体;格式识别不需要 Docling;表格内切句待改进;抽取需第三道依据校验与规则兜底 - architecture.md:§3.5 加预处理与字体包,§3.6 证据校验改为三道,§10 回填结论 - 03 功能需求与 decisions.md(TV1):召回 ≥ 99%、采纳准确率 ≥ 95%、页码 100%、高亮正文 ≥ 98% / 表格 ≥ 95% Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
50
spikes/s1-convert/toc_truth.py
Normal file
50
spikes/s1-convert/toc_truth.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""S1 替代真值:DOCX 目录里缓存的页码(生成文档的软件排版时写入)。
|
||||
比较「各标题之间相隔的页数」,消除页码起始值的影响。"""
|
||||
import json, re, sys, zipfile, pathlib
|
||||
import pdfplumber
|
||||
|
||||
OUT = pathlib.Path(__file__).parent / "out"
|
||||
norm = lambda s: re.sub(r"[\s ·.…\-—_]", "", s)
|
||||
|
||||
def toc_entries(docx):
|
||||
x = zipfile.ZipFile(docx).read("word/document.xml").decode()
|
||||
ents = []
|
||||
for p in re.findall(r"<w:p[ >].*?</w:p>", x, re.S):
|
||||
if "PAGEREF" not in p:
|
||||
continue
|
||||
# 段落里 PAGEREF 字段之前的文字是标题,字段结果是缓存页码
|
||||
before, after = p.split("PAGEREF", 1)
|
||||
title = norm("".join(re.findall(r"<w:t[^>]*>([^<]*)</w:t>", before)))
|
||||
num = re.findall(r"<w:t[^>]*>(\d+)</w:t>", after)
|
||||
if title and num:
|
||||
ents.append((title, int(num[0])))
|
||||
return ents
|
||||
|
||||
def heading_pages(pdf_path, titles):
|
||||
with pdfplumber.open(pdf_path) as pdf:
|
||||
texts = [norm(pg.extract_text() or "") for pg in pdf.pages]
|
||||
found = {}
|
||||
for t in titles:
|
||||
key = t[:18]
|
||||
hits = [i for i, tx in enumerate(texts) if key in tx]
|
||||
# 跳过目录页本身:取最后一个出现位置之前、且不在前 6 页的第一个命中
|
||||
body = [i for i in hits if i >= 4]
|
||||
found[t] = body[0] + 1 if body else None
|
||||
return found
|
||||
|
||||
rows = {}
|
||||
for sid in ["S03", "S04", "S06"]:
|
||||
ents = toc_entries(OUT / "fixed-docx" / f"{sid}.docx")
|
||||
rows[sid] = {"entries": len(ents)}
|
||||
for cfg in ["A-fixed", "B-fixed", "C10-fixed", "C13-fixed"]:
|
||||
hp = heading_pages(OUT / cfg / f"{sid}.pdf", [t for t, _ in ents])
|
||||
pairs = [(n, hp[t]) for t, n in ents if hp[t]]
|
||||
if len(pairs) < 2:
|
||||
rows[sid][cfg] = None; continue
|
||||
(c0, p0), (c1, p1) = pairs[0], pairs[-1]
|
||||
span_t, span_p = c1 - c0, p1 - p0
|
||||
dev = [abs((p - p0) - (c - c0)) for c, p in pairs]
|
||||
rows[sid][cfg] = {"matched": len(pairs), "span_truth": span_t, "span_pdf": span_p,
|
||||
"span_err_pct": round(100 * (span_p - span_t) / span_t, 1), "max_dev_pages": max(dev)}
|
||||
print(sid, json.dumps(rows[sid], ensure_ascii=False))
|
||||
(OUT / "toc_truth.json").write_text(json.dumps(rows, ensure_ascii=False, indent=1))
|
||||
Reference in New Issue
Block a user