fix(evals): 漏响应在所有表里都删;评分按句子重叠合并重复的标注单元
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,7 @@ class Mutator:
|
||||
for r in chunk:
|
||||
self.mark(r.get("responds") or [], "D01", "删连续一段", "block", worded=False)
|
||||
del rows[start : start + n]
|
||||
self.drop_everywhere([u for r in chunk for u in r.get("responds") or []])
|
||||
return
|
||||
|
||||
def row_ops(self, n: int) -> None:
|
||||
@@ -101,6 +102,11 @@ class Mutator:
|
||||
done += 1
|
||||
break
|
||||
|
||||
def drop_everywhere(self, units: list[str]) -> None:
|
||||
"""漏响应要在所有表里都漏:S04 的技术参数同时列在全量偏离表和 ★ 表里,只删一张表等于没漏。"""
|
||||
for t in self.tables():
|
||||
t["rows"] = [r for r in t["rows"] if not set(r.get("responds") or []) & set(units)]
|
||||
|
||||
@staticmethod
|
||||
def col(t: dict[str, Any], field: str) -> int | None:
|
||||
cols = t.get("columns") or ["no", "requirement", "answer", "deviation", "note"]
|
||||
@@ -114,6 +120,7 @@ class Mutator:
|
||||
units = r.get("responds") or []
|
||||
if op == "D01":
|
||||
t["rows"].remove(r)
|
||||
self.drop_everywhere(units)
|
||||
self.mark(units, "D01", "删一行", worded=False)
|
||||
elif op == "D03":
|
||||
return self._unmet(r, tp, ans, dev)
|
||||
|
||||
@@ -64,6 +64,20 @@ def main() -> None:
|
||||
for s in u.sids:
|
||||
sid_units[s].add(u.id)
|
||||
|
||||
# 标注集里同一条要求有时记了两个单元(两个类别各一条,或表格一行拆成几条):句子有重叠的单元合成一组,按组计
|
||||
parent = {u.id: u.id for u in units}
|
||||
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
for ids in sid_units.values():
|
||||
first, *rest = sorted(ids)
|
||||
for o in rest:
|
||||
parent[find(o)] = find(first)
|
||||
group = {u.id: find(u.id) for u in units}
|
||||
|
||||
per_unit: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
unmapped: list[dict[str, Any]] = []
|
||||
for it in res["items"]:
|
||||
@@ -82,8 +96,13 @@ def main() -> None:
|
||||
fp_by_trap: Counter[str] = Counter()
|
||||
ok_units = trap_units = 0
|
||||
other_findings = 0
|
||||
for uid, e in manifest["units"].items():
|
||||
items = per_unit.get(uid, [])
|
||||
seen_groups: set[str] = set()
|
||||
for uid, e in sorted(manifest["units"].items(), key=lambda kv: kv[1].get("expect") != "defect"):
|
||||
g = group.get(uid, uid)
|
||||
if g in seen_groups:
|
||||
continue # 同组的单元只算一次(挂了问题的那个优先)
|
||||
seen_groups.add(g)
|
||||
items = [it for x in manifest["units"] if group.get(x, x) == g for it in per_unit.get(x, [])]
|
||||
found = [it for it in items if it["finding"]]
|
||||
if e["expect"] == "defect":
|
||||
if not e.get("reviewed") and not a.include_unreviewed:
|
||||
|
||||
Reference in New Issue
Block a user