docs(architecture): 完成技术验证 S1–S4，确定 FR-M2-05/06 目标值
- spikes/：Word 转 PDF（Gotenberg）、格式识别、原句坐标、DeepSeek 抽取四项验证代码与两版提示词；样本派生产物和 .env 不进 git - tech-validation.md：完整结论。四项都可行：WPS 字体字符集要修正并改用行高调整过的字体；格式识别不需要 Docling；表格内切句待改进；抽取需第三道依据校验与规则兜底 - architecture.md：§3.5 加预处理与字体包，§3.6 证据校验改为三道，§10 回填结论 - 03 功能需求与 decisions.md（TV1）：召回 ≥ 99%、采纳准确率 ≥ 95%、页码 100%、高亮正文 ≥ 98% / 表格 ≥ 95% Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
37
spikes/s2-format/docling_check.py
Normal file
37
spikes/s2-format/docling_check.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""S2 对比：Docling 是否保留斜体 / 下划线。分别喂 DOCX 原文件和规范化 PDF。
|
||||
用法：python -I docling_check.py <docling 地址> <docx> <pdf> <银标准 JSON>"""
|
||||
import json, re, sys, time, pathlib, collections
|
||||
import httpx
|
||||
|
||||
base, docx, pdf, silver = sys.argv[1], pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]), json.loads(pathlib.Path(sys.argv[4]).read_text())
|
||||
OUT = pathlib.Path(__file__).parent / "out"
|
||||
WS = re.compile(r"[\s ]")
|
||||
silver_chars = sum(map(len, silver))
|
||||
|
||||
def run(path, name, extra):
|
||||
t = time.monotonic()
|
||||
data = {"to_formats": ["json"], "do_ocr": "false", **extra}
|
||||
r = httpx.post(f"{base}/v1/convert/file", files={"files": (name, path.read_bytes())}, data=data, timeout=1800)
|
||||
dt = round(time.monotonic() - t, 1)
|
||||
r.raise_for_status()
|
||||
doc = r.json()["document"]["json_content"]
|
||||
(OUT / f"docling-{name}.json").write_text(json.dumps(doc, ensure_ascii=False))
|
||||
fmt = collections.Counter(); iu = []
|
||||
has_prov_bbox = 0
|
||||
for item in doc.get("texts", []):
|
||||
f = item.get("formatting") or {}
|
||||
txt = WS.sub("", item.get("text", ""))
|
||||
for k in ("bold", "italic", "underline"):
|
||||
if f.get(k):
|
||||
fmt[k] += len(txt)
|
||||
if f.get("italic") and f.get("underline"):
|
||||
iu.append(txt)
|
||||
has_prov_bbox += bool(item.get("prov"))
|
||||
got = Counter_overlap = sum((collections.Counter("".join(iu)) & collections.Counter("".join(silver))).values())
|
||||
return {"sec": dt, "texts": len(doc.get("texts", [])), "tables": len(doc.get("tables", [])), "fmt_chars": dict(fmt),
|
||||
"iu_chars": sum(map(len, iu)), "silver_iu_chars": silver_chars, "iu_char_overlap": round(got / silver_chars, 3),
|
||||
"texts_with_page_bbox": has_prov_bbox}
|
||||
|
||||
res = {"docx": run(docx, "S06.docx", {}), "pdf": run(pdf, "S06.pdf", {"pdf_backend": "dlparse_v4"})}
|
||||
print(json.dumps(res, ensure_ascii=False, indent=1))
|
||||
(OUT / "docling-result.json").write_text(json.dumps(res, ensure_ascii=False, indent=1))
|
||||
164
spikes/s2-format/format_check.py
Normal file
164
spikes/s2-format/format_check.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""S2：在规范化 PDF 里识别斜体 / 下划线 / 加粗，并和 DOCX 原文件的格式交叉校验。
|
||||
|
||||
银标准：DOCX 里「斜体且下划线」的文字（含样式继承：文档默认 → 段落样式链 → 字符样式链 → 直接格式）。
|
||||
PDF 侧：
|
||||
斜体 = 字体名含 Italic/Oblique，或字形矩阵有倾斜（中文没有斜体字形，LibreOffice 用倾斜合成）
|
||||
下划线 = 字符底边上方 0.25 个字号到下方 4pt 之间，有一条覆盖该字水平中心的横线（排除很长的表格边框需要另外判断，这里先不排除，看精确率）
|
||||
加粗 = 字体名含 Bold
|
||||
比较方式：把两侧「斜体且下划线」的文字分别按段拼接、去空白，算字符级召回和精确率（以片段是否能在对方文本里找到为准）。
|
||||
|
||||
用法：python -I format_check.py <规范化 PDF 目录> <DOCX 目录> S06 S08 S09
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections import Counter
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import zipfile
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import pdfplumber
|
||||
|
||||
W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
||||
OUT = pathlib.Path(__file__).parent / "out"
|
||||
WS = re.compile(r"[\s ]")
|
||||
|
||||
|
||||
# ---------- DOCX 侧 ----------
|
||||
|
||||
def _flags(rpr) -> dict:
|
||||
"""从 rPr 读出 i / u / b，返回只包含显式设置项的字典。"""
|
||||
out = {}
|
||||
if rpr is None:
|
||||
return out
|
||||
for tag, key in ((f"{W}i", "i"), (f"{W}b", "b")):
|
||||
el = rpr.find(tag)
|
||||
if el is not None:
|
||||
out[key] = el.get(f"{W}val", "true") not in ("0", "false", "none")
|
||||
el = rpr.find(f"{W}u")
|
||||
if el is not None:
|
||||
out["u"] = el.get(f"{W}val", "single") not in ("none", "0", "false")
|
||||
return out
|
||||
|
||||
|
||||
def load_styles(z: zipfile.ZipFile):
|
||||
root = ET.fromstring(z.read("word/styles.xml"))
|
||||
styles, defaults = {}, {}
|
||||
dd = root.find(f"{W}docDefaults/{W}rPrDefault/{W}rPr")
|
||||
defaults.update(_flags(dd))
|
||||
for s in root.findall(f"{W}style"):
|
||||
sid = s.get(f"{W}styleId")
|
||||
based = s.find(f"{W}basedOn")
|
||||
styles[sid] = {"based": based.get(f"{W}val") if based is not None else None,
|
||||
"flags": _flags(s.find(f"{W}rPr"))}
|
||||
return styles, defaults
|
||||
|
||||
|
||||
def style_flags(styles, sid, depth=0) -> dict:
|
||||
if not sid or sid not in styles or depth > 20:
|
||||
return {}
|
||||
base = style_flags(styles, styles[sid]["based"], depth + 1)
|
||||
return {**base, **styles[sid]["flags"]}
|
||||
|
||||
|
||||
def docx_iu_segments(path: pathlib.Path) -> tuple[list[str], int]:
|
||||
"""返回「斜体且下划线」的连续文字片段，以及全文字数。"""
|
||||
z = zipfile.ZipFile(path)
|
||||
styles, defaults = load_styles(z)
|
||||
root = ET.fromstring(z.read("word/document.xml"))
|
||||
segs, cur, total = [], [], 0
|
||||
for p in root.iter(f"{W}p"):
|
||||
ppr = p.find(f"{W}pPr")
|
||||
pstyle = ppr.find(f"{W}pStyle").get(f"{W}val") if ppr is not None and ppr.find(f"{W}pStyle") is not None else None
|
||||
pflags = {**defaults, **style_flags(styles, pstyle)}
|
||||
for r in p.iter(f"{W}r"):
|
||||
rpr = r.find(f"{W}rPr")
|
||||
rstyle = rpr.find(f"{W}rStyle").get(f"{W}val") if rpr is not None and rpr.find(f"{W}rStyle") is not None else None
|
||||
f = {**pflags, **style_flags(styles, rstyle), **_flags(rpr)}
|
||||
text = WS.sub("", "".join(t.text or "" for t in r.iter(f"{W}t")))
|
||||
total += len(text)
|
||||
if not text:
|
||||
continue
|
||||
if f.get("i") and f.get("u"):
|
||||
cur.append(text)
|
||||
elif cur:
|
||||
segs.append("".join(cur)); cur = []
|
||||
if cur:
|
||||
segs.append("".join(cur)); cur = []
|
||||
return [s for s in segs if s], total
|
||||
|
||||
|
||||
# ---------- PDF 侧 ----------
|
||||
|
||||
def pdf_chars_with_flags(path: pathlib.Path):
|
||||
"""逐页返回字符和 i/u/b 标记（按 pdfplumber 的阅读顺序）。"""
|
||||
with pdfplumber.open(path) as pdf:
|
||||
for page in pdf.pages:
|
||||
hlines = [(l["x0"], l["x1"], l["top"]) for l in page.lines if abs(l["top"] - l["bottom"]) < 0.5]
|
||||
hlines += [(r["x0"], r["x1"], r["top"]) for r in page.rects if r["height"] < 1.5 and r["width"] > 2]
|
||||
for c in page.chars:
|
||||
if not c["text"].strip():
|
||||
continue
|
||||
name = c["fontname"].split("+")[-1]
|
||||
italic = "Italic" in name or "Oblique" in name or abs(c["matrix"][2]) > 0.1
|
||||
mid = (c["x0"] + c["x1"]) / 2
|
||||
# 下划线一般在基线下 1～2pt；字体下行部分小时会落在字符框内，所以窗口向上放宽 0.25 个字号
|
||||
under = any(x0 - 0.5 <= mid <= x1 + 0.5 and -0.25 * c["size"] <= y - c["bottom"] <= 4 for x0, x1, y in hlines)
|
||||
yield page.page_number, c["text"], italic, under, "Bold" in name
|
||||
|
||||
|
||||
def pdf_iu_segments(path: pathlib.Path):
|
||||
segs, cur, stats = [], [], {"chars": 0, "italic": 0, "under": 0, "bold": 0}
|
||||
for _, ch, it, un, bd in pdf_chars_with_flags(path):
|
||||
stats["chars"] += 1; stats["italic"] += it; stats["under"] += un; stats["bold"] += bd
|
||||
if it and un:
|
||||
cur.append(ch)
|
||||
elif cur:
|
||||
segs.append("".join(cur)); cur = []
|
||||
if cur:
|
||||
segs.append("".join(cur))
|
||||
return [WS.sub("", s) for s in segs if WS.sub("", s)], stats
|
||||
|
||||
|
||||
def coverage(segs: list[str], haystack: str, min_len: int = 2) -> tuple[int, int]:
|
||||
"""片段按字符数加权：能在对方文本里找到的片段占多少字。"""
|
||||
hit = tot = 0
|
||||
for s in segs:
|
||||
if len(s) < min_len:
|
||||
continue
|
||||
tot += len(s)
|
||||
if s in haystack:
|
||||
hit += len(s)
|
||||
else: # 片段可能被对方切成了几段：退一步按 6 字窗口找
|
||||
win = [s[i:i + 6] for i in range(0, max(1, len(s) - 5), 6)]
|
||||
hit += round(len(s) * sum(w in haystack for w in win) / len(win))
|
||||
return hit, tot
|
||||
|
||||
|
||||
def main():
|
||||
pdf_dir, docx_dir, sids = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3:]
|
||||
report = {}
|
||||
for sid in sids:
|
||||
d_segs, d_total = docx_iu_segments(docx_dir / f"{sid}.docx")
|
||||
p_segs, p_stats = pdf_iu_segments(pdf_dir / f"{sid}.pdf")
|
||||
r_hit, r_tot = coverage(d_segs, "\u0001".join(p_segs)) # 召回：DOCX 的片段在 PDF 侧找到多少
|
||||
p_hit, p_tot = coverage(p_segs, "\u0001".join(d_segs)) # 精确率：PDF 侧的片段有多少是真的
|
||||
report[sid] = {
|
||||
"docx_iu_segments": len(d_segs), "docx_iu_chars": sum(map(len, d_segs)), "docx_chars": d_total,
|
||||
"pdf_iu_segments": len(p_segs), "pdf_iu_chars": sum(map(len, p_segs)), "pdf_stats": p_stats,
|
||||
"recall": round(r_hit / r_tot, 3) if r_tot else None,
|
||||
# 按字符多重集合比较：不受 PDF 阅读顺序（表格单元格拼接）影响
|
||||
"char_overlap": round(sum((Counter("".join(d_segs)) & Counter("".join(p_segs))).values()) / max(1, sum(map(len, d_segs))), 3),
|
||||
"precision": round(p_hit / p_tot, 3) if p_tot else None,
|
||||
"missed_examples": [s[:40] for s in d_segs if len(s) >= 6 and s[:6] not in "\u0001".join(p_segs)][:5],
|
||||
"false_examples": [s[:40] for s in p_segs if len(s) >= 6 and s[:6] not in "\u0001".join(d_segs)][:5],
|
||||
}
|
||||
print(sid, json.dumps(report[sid], ensure_ascii=False))
|
||||
(OUT / f"{sid}-docx-iu.json").write_text(json.dumps(d_segs, ensure_ascii=False, indent=0))
|
||||
(OUT / "result.json").write_text(json.dumps(report, ensure_ascii=False, indent=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user