fix(evals): 漏响应在所有表里都删;评分按句子重叠合并重复的标注单元

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
2026-11-04 12:26:21 +08:00
parent a07663cbdb
commit 54534be0c1
2 changed files with 28 additions and 2 deletions

View File

@@ -82,6 +82,7 @@ class Mutator:
for r in chunk:
self.mark(r.get("responds") or [], "D01", "删连续一段", "block", worded=False)
del rows[start : start + n]
self.drop_everywhere([u for r in chunk for u in r.get("responds") or []])
return
def row_ops(self, n: int) -> None:
@@ -101,6 +102,11 @@ class Mutator:
done += 1
break
def drop_everywhere(self, units: list[str]) -> None:
"""漏响应要在所有表里都漏:S04 的技术参数同时列在全量偏离表和 ★ 表里,只删一张表等于没漏。"""
for t in self.tables():
t["rows"] = [r for r in t["rows"] if not set(r.get("responds") or []) & set(units)]
@staticmethod
def col(t: dict[str, Any], field: str) -> int | None:
cols = t.get("columns") or ["no", "requirement", "answer", "deviation", "note"]
@@ -114,6 +120,7 @@ class Mutator:
units = r.get("responds") or []
if op == "D01":
t["rows"].remove(r)
self.drop_everywhere(units)
self.mark(units, "D01", "删一行", worded=False)
elif op == "D03":
return self._unmet(r, tp, ans, dev)

View File

@@ -64,6 +64,20 @@ def main() -> None:
for s in u.sids:
sid_units[s].add(u.id)
# 标注集里同一条要求有时记了两个单元(两个类别各一条,或表格一行拆成几条):句子有重叠的单元合成一组,按组计
parent = {u.id: u.id for u in units}
def find(x: str) -> str:
while parent[x] != x:
x = parent[x]
return x
for ids in sid_units.values():
first, *rest = sorted(ids)
for o in rest:
parent[find(o)] = find(first)
group = {u.id: find(u.id) for u in units}
per_unit: dict[str, list[dict[str, Any]]] = defaultdict(list)
unmapped: list[dict[str, Any]] = []
for it in res["items"]:
@@ -82,8 +96,13 @@ def main() -> None:
fp_by_trap: Counter[str] = Counter()
ok_units = trap_units = 0
other_findings = 0
for uid, e in manifest["units"].items():
items = per_unit.get(uid, [])
seen_groups: set[str] = set()
for uid, e in sorted(manifest["units"].items(), key=lambda kv: kv[1].get("expect") != "defect"):
g = group.get(uid, uid)
if g in seen_groups:
continue # 同组的单元只算一次(挂了问题的那个优先)
seen_groups.add(g)
items = [it for x in manifest["units"] if group.get(x, x) == g for it in per_unit.get(x, [])]
found = [it for it in items if it["finding"]]
if e["expect"] == "defect":
if not e.get("reviewed") and not a.include_unreviewed: