604e889ca4
- spikes/：Word 转 PDF（Gotenberg）、格式识别、原句坐标、DeepSeek 抽取四项验证代码与两版提示词；样本派生产物和 .env 不进 git - tech-validation.md：完整结论。四项都可行：WPS 字体字符集要修正并改用行高调整过的字体；格式识别不需要 Docling；表格内切句待改进；抽取需第三道依据校验与规则兜底 - architecture.md：§3.5 加预处理与字体包，§3.6 证据校验改为三道，§10 回填结论 - 03 功能需求与 decisions.md（TV1）：召回 ≥ 99%、采纳准确率 ≥ 95%、页码 100%、高亮正文 ≥ 98% / 表格 ≥ 95% Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
51 lines
2.2 KiB
Python
51 lines
2.2 KiB
Python
"""S1 替代真值：DOCX 目录里缓存的页码（生成文档的软件排版时写入）。
|
||
比较「各标题之间相隔的页数」，消除页码起始值的影响。"""
|
||
import json, re, sys, zipfile, pathlib
|
||
import pdfplumber
|
||
|
||
OUT = pathlib.Path(__file__).parent / "out"
|
||
norm = lambda s: re.sub(r"[\s ·.…\-—_]", "", s)
|
||
|
||
def toc_entries(docx):
|
||
x = zipfile.ZipFile(docx).read("word/document.xml").decode()
|
||
ents = []
|
||
for p in re.findall(r"<w:p[ >].*?</w:p>", x, re.S):
|
||
if "PAGEREF" not in p:
|
||
continue
|
||
# 段落里 PAGEREF 字段之前的文字是标题，字段结果是缓存页码
|
||
before, after = p.split("PAGEREF", 1)
|
||
title = norm("".join(re.findall(r"<w:t[^>]*>([^<]*)</w:t>", before)))
|
||
num = re.findall(r"<w:t[^>]*>(\d+)</w:t>", after)
|
||
if title and num:
|
||
ents.append((title, int(num[0])))
|
||
return ents
|
||
|
||
def heading_pages(pdf_path, titles):
|
||
with pdfplumber.open(pdf_path) as pdf:
|
||
texts = [norm(pg.extract_text() or "") for pg in pdf.pages]
|
||
found = {}
|
||
for t in titles:
|
||
key = t[:18]
|
||
hits = [i for i, tx in enumerate(texts) if key in tx]
|
||
# 跳过目录页本身：取最后一个出现位置之前、且不在前 6 页的第一个命中
|
||
body = [i for i in hits if i >= 4]
|
||
found[t] = body[0] + 1 if body else None
|
||
return found
|
||
|
||
rows = {}
|
||
for sid in ["S03", "S04", "S06"]:
|
||
ents = toc_entries(OUT / "fixed-docx" / f"{sid}.docx")
|
||
rows[sid] = {"entries": len(ents)}
|
||
for cfg in ["A-fixed", "B-fixed", "C10-fixed", "C13-fixed"]:
|
||
hp = heading_pages(OUT / cfg / f"{sid}.pdf", [t for t, _ in ents])
|
||
pairs = [(n, hp[t]) for t, n in ents if hp[t]]
|
||
if len(pairs) < 2:
|
||
rows[sid][cfg] = None; continue
|
||
(c0, p0), (c1, p1) = pairs[0], pairs[-1]
|
||
span_t, span_p = c1 - c0, p1 - p0
|
||
dev = [abs((p - p0) - (c - c0)) for c, p in pairs]
|
||
rows[sid][cfg] = {"matched": len(pairs), "span_truth": span_t, "span_pdf": span_p,
|
||
"span_err_pct": round(100 * (span_p - span_t) / span_t, 1), "max_dev_pages": max(dev)}
|
||
print(sid, json.dumps(rows[sid], ensure_ascii=False))
|
||
(OUT / "toc_truth.json").write_text(json.dumps(rows, ensure_ascii=False, indent=1))
|