604e889ca4
- spikes/：Word 转 PDF（Gotenberg）、格式识别、原句坐标、DeepSeek 抽取四项验证代码与两版提示词；样本派生产物和 .env 不进 git - tech-validation.md：完整结论。四项都可行：WPS 字体字符集要修正并改用行高调整过的字体；格式识别不需要 Docling；表格内切句待改进；抽取需第三道依据校验与规则兜底 - architecture.md：§3.5 加预处理与字体包，§3.6 证据校验改为三道，§10 回填结论 - 03 功能需求与 decisions.md（TV1）：召回 ≥ 99%、采纳准确率 ≥ 95%、页码 100%、高亮正文 ≥ 98% / 表格 ≥ 95% Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
29 lines
1.7 KiB
Python
29 lines
1.7 KiB
Python
"""汇总 S4 一次运行：按银标准类型（★ / 斜体下划线）算总召回，列出漏掉的句子。"""
|
|
import json, sys, pathlib
|
|
OUT = pathlib.Path(__file__).parent / "out"
|
|
tag = sys.argv[1]
|
|
summary = json.loads((OUT / f"run-{tag}" / "summary.json").read_text())
|
|
agg = {"star": [0, 0, 0], "iu": [0, 0, 0]} # 严格命中、相邻命中、总数
|
|
missed = []
|
|
for sid in summary["docs"]:
|
|
sents = json.loads((OUT / f"{sid}-input.json").read_text())
|
|
order = {s["id"]: i for i, s in enumerate(sents)}
|
|
res = json.loads((OUT / f"run-{tag}" / f"{sid}.json").read_text())
|
|
cited = {e["sid"] for r in res for it in r["accepted"] for e in it["evidence"]}
|
|
near = {sents[j]["id"] for c in cited for j in (order[c] - 1, order[c], order[c] + 1) if 0 <= j < len(sents)}
|
|
for s in sents:
|
|
if s["silver"]:
|
|
a = agg[s["silver"]]; a[2] += 1; a[0] += s["id"] in cited; a[1] += s["id"] in near
|
|
if s["id"] not in near:
|
|
missed.append((sid, s["silver"], s["id"], s["text"][:60]))
|
|
for k, (st, nb, n) in agg.items():
|
|
print(f"{k}: strict {st}/{n} = {st/n:.1%}, neighbor {nb}/{n} = {nb/n:.1%}")
|
|
acc = sum(d["eval"]["accepted_items"] for d in summary["docs"].values())
|
|
ok = sum(round(d["eval"]["precision_incl_explicit_invalid"] * d["eval"]["accepted_items"]) for d in summary["docs"].values())
|
|
pend = sum(d["eval"]["pending_items"] for d in summary["docs"].values())
|
|
evt = sum(d["eval"]["evidence_total"] for d in summary["docs"].values()); evp = sum(d["eval"]["evidence_pass"] for d in summary["docs"].values())
|
|
print(f"accepted {acc}, on-silver-or-explicit-invalid {ok} = {ok/acc:.1%}, pending {pend}, evidence pass {evp}/{evt} = {evp/evt:.1%}")
|
|
print("total usage", summary["total_usage"])
|
|
for m in missed:
|
|
print(" MISSED", m)
|