bid-doctor/spikes/s2-format/format_check.py

"""S2：在规范化 PDF 里识别斜体 / 下划线 / 加粗，并和 DOCX 原文件的格式交叉校验。

银标准：DOCX 里「斜体且下划线」的文字（含样式继承：文档默认 → 段落样式链 → 字符样式链 → 直接格式）。
PDF 侧：
  斜体  = 字体名含 Italic/Oblique，或字形矩阵有倾斜（中文没有斜体字形，LibreOffice 用倾斜合成）
  下划线 = 字符底边上方 0.25 个字号到下方 4pt 之间，有一条覆盖该字水平中心的横线（排除很长的表格边框需要另外判断，这里先不排除，看精确率）
  加粗  = 字体名含 Bold
比较方式：把两侧「斜体且下划线」的文字分别按段拼接、去空白，算字符级召回和精确率（以片段是否能在对方文本里找到为准）。

用法：python -I format_check.py <规范化 PDF 目录> <DOCX 目录> S06 S08 S09
"""
from __future__ import annotations

import json
from collections import Counter
import pathlib
import re
import sys
import zipfile
from xml.etree import ElementTree as ET

import pdfplumber

W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
OUT = pathlib.Path(__file__).parent / "out"
WS = re.compile(r"[\s　]")


# ---------- DOCX 侧 ----------

def _flags(rpr) -> dict:
    """从 rPr 读出 i / u / b，返回只包含显式设置项的字典。"""
    out = {}
    if rpr is None:
        return out
    for tag, key in ((f"{W}i", "i"), (f"{W}b", "b")):
        el = rpr.find(tag)
        if el is not None:
            out[key] = el.get(f"{W}val", "true") not in ("0", "false", "none")
    el = rpr.find(f"{W}u")
    if el is not None:
        out["u"] = el.get(f"{W}val", "single") not in ("none", "0", "false")
    return out


def load_styles(z: zipfile.ZipFile):
    root = ET.fromstring(z.read("word/styles.xml"))
    styles, defaults = {}, {}
    dd = root.find(f"{W}docDefaults/{W}rPrDefault/{W}rPr")
    defaults.update(_flags(dd))
    for s in root.findall(f"{W}style"):
        sid = s.get(f"{W}styleId")
        based = s.find(f"{W}basedOn")
        styles[sid] = {"based": based.get(f"{W}val") if based is not None else None,
                       "flags": _flags(s.find(f"{W}rPr"))}
    return styles, defaults


def style_flags(styles, sid, depth=0) -> dict:
    if not sid or sid not in styles or depth > 20:
        return {}
    base = style_flags(styles, styles[sid]["based"], depth + 1)
    return {**base, **styles[sid]["flags"]}


def docx_iu_segments(path: pathlib.Path) -> tuple[list[str], int]:
    """返回「斜体且下划线」的连续文字片段，以及全文字数。"""
    z = zipfile.ZipFile(path)
    styles, defaults = load_styles(z)
    root = ET.fromstring(z.read("word/document.xml"))
    segs, cur, total = [], [], 0
    for p in root.iter(f"{W}p"):
        ppr = p.find(f"{W}pPr")
        pstyle = ppr.find(f"{W}pStyle").get(f"{W}val") if ppr is not None and ppr.find(f"{W}pStyle") is not None else None
        pflags = {**defaults, **style_flags(styles, pstyle)}
        for r in p.iter(f"{W}r"):
            rpr = r.find(f"{W}rPr")
            rstyle = rpr.find(f"{W}rStyle").get(f"{W}val") if rpr is not None and rpr.find(f"{W}rStyle") is not None else None
            f = {**pflags, **style_flags(styles, rstyle), **_flags(rpr)}
            text = WS.sub("", "".join(t.text or "" for t in r.iter(f"{W}t")))
            total += len(text)
            if not text:
                continue
            if f.get("i") and f.get("u"):
                cur.append(text)
            elif cur:
                segs.append("".join(cur)); cur = []
        if cur:
            segs.append("".join(cur)); cur = []
    return [s for s in segs if s], total


# ---------- PDF 侧 ----------

def pdf_chars_with_flags(path: pathlib.Path):
    """逐页返回字符和 i/u/b 标记（按 pdfplumber 的阅读顺序）。"""
    with pdfplumber.open(path) as pdf:
        for page in pdf.pages:
            hlines = [(l["x0"], l["x1"], l["top"]) for l in page.lines if abs(l["top"] - l["bottom"]) < 0.5]
            hlines += [(r["x0"], r["x1"], r["top"]) for r in page.rects if r["height"] < 1.5 and r["width"] > 2]
            for c in page.chars:
                if not c["text"].strip():
                    continue
                name = c["fontname"].split("+")[-1]
                italic = "Italic" in name or "Oblique" in name or abs(c["matrix"][2]) > 0.1
                mid = (c["x0"] + c["x1"]) / 2
                # 下划线一般在基线下 1～2pt；字体下行部分小时会落在字符框内，所以窗口向上放宽 0.25 个字号
                under = any(x0 - 0.5 <= mid <= x1 + 0.5 and -0.25 * c["size"] <= y - c["bottom"] <= 4 for x0, x1, y in hlines)
                yield page.page_number, c["text"], italic, under, "Bold" in name


def pdf_iu_segments(path: pathlib.Path):
    segs, cur, stats = [], [], {"chars": 0, "italic": 0, "under": 0, "bold": 0}
    for _, ch, it, un, bd in pdf_chars_with_flags(path):
        stats["chars"] += 1; stats["italic"] += it; stats["under"] += un; stats["bold"] += bd
        if it and un:
            cur.append(ch)
        elif cur:
            segs.append("".join(cur)); cur = []
    if cur:
        segs.append("".join(cur))
    return [WS.sub("", s) for s in segs if WS.sub("", s)], stats


def coverage(segs: list[str], haystack: str, min_len: int = 2) -> tuple[int, int]:
    """片段按字符数加权：能在对方文本里找到的片段占多少字。"""
    hit = tot = 0
    for s in segs:
        if len(s) < min_len:
            continue
        tot += len(s)
        if s in haystack:
            hit += len(s)
        else:  # 片段可能被对方切成了几段：退一步按 6 字窗口找
            win = [s[i:i + 6] for i in range(0, max(1, len(s) - 5), 6)]
            hit += round(len(s) * sum(w in haystack for w in win) / len(win))
    return hit, tot


def main():
    pdf_dir, docx_dir, sids = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3:]
    report = {}
    for sid in sids:
        d_segs, d_total = docx_iu_segments(docx_dir / f"{sid}.docx")
        p_segs, p_stats = pdf_iu_segments(pdf_dir / f"{sid}.pdf")
        r_hit, r_tot = coverage(d_segs, "\u0001".join(p_segs))   # 召回：DOCX 的片段在 PDF 侧找到多少
        p_hit, p_tot = coverage(p_segs, "\u0001".join(d_segs))   # 精确率：PDF 侧的片段有多少是真的
        report[sid] = {
            "docx_iu_segments": len(d_segs), "docx_iu_chars": sum(map(len, d_segs)), "docx_chars": d_total,
            "pdf_iu_segments": len(p_segs), "pdf_iu_chars": sum(map(len, p_segs)), "pdf_stats": p_stats,
            "recall": round(r_hit / r_tot, 3) if r_tot else None,
            # 按字符多重集合比较：不受 PDF 阅读顺序（表格单元格拼接）影响
            "char_overlap": round(sum((Counter("".join(d_segs)) & Counter("".join(p_segs))).values()) / max(1, sum(map(len, d_segs))), 3),
            "precision": round(p_hit / p_tot, 3) if p_tot else None,
            "missed_examples": [s[:40] for s in d_segs if len(s) >= 6 and s[:6] not in "\u0001".join(p_segs)][:5],
            "false_examples": [s[:40] for s in p_segs if len(s) >= 6 and s[:6] not in "\u0001".join(d_segs)][:5],
        }
        print(sid, json.dumps(report[sid], ensure_ascii=False))
        (OUT / f"{sid}-docx-iu.json").write_text(json.dumps(d_segs, ensure_ascii=False, indent=0))
    (OUT / "result.json").write_text(json.dumps(report, ensure_ascii=False, indent=1))


if __name__ == "__main__":
    main()