diff --git a/evals/check/build.py b/evals/check/build.py index 26c0f7f..9037c51 100644 --- a/evals/check/build.py +++ b/evals/check/build.py @@ -366,8 +366,18 @@ def build(tender: str, bid: str, style: str, run: str) -> None: if f.get("only_deviations"): devs = [r for r in src if r["deviation"] != "无偏离"] rows = [_row(k, r, columns) for k, r in enumerate(devs, 1)] - empty = {"no": "1", "requirement": "—", "answer": "—", "deviation": "无偏离"} # 苏采云范本:没有偏离时注明「无偏离」 - rows = rows or [{"cells": [empty.get(c, "") for c in columns], "anchor": columns.index("deviation") if "deviation" in columns else 0}] + empty = { + "no": "1", + "requirement": "—", + "answer": "—", + "deviation": "无偏离", + } # 苏采云范本:没有偏离时注明「无偏离」 + rows = rows or [ + { + "cells": [empty.get(c, "") for c in columns], + "anchor": columns.index("deviation") if "deviation" in columns else 0, + } + ] pool = [ _row(0, r, columns) for r in src if r["deviation"] == "无偏离" ] # 没列出的(默认接受):注入时可以挑来写成偏离 diff --git a/evals/check/mutate.py b/evals/check/mutate.py index a1fb8f7..b15e720 100644 --- a/evals/check/mutate.py +++ b/evals/check/mutate.py @@ -29,6 +29,11 @@ from build import rmb_upper # noqa: E402 OUT = EVALS / "out" / "check" VOCAB = {"dev": EVALS / "check" / "vocab-dev.json", "holdout": OUT / "_sealed" / "vocab-holdout.json"} +# 「暂不支持」只对功能类要求说得通;证书、清单、服务义务、文档交付、改造范围这类不是功能(复核两批共拒掉 12 条) +FUNCTIONAL = re.compile(r"支持|功能|具备|实现|可对|能够|可以|兼容|提供.{0,6}接口") +NON_FUNCTIONAL = re.compile(r"证书|认证|清单|服务|培训|文档|资料|方案|范围|记录|报告|期限|质保|交付|人员|工期|成果") +# 条件性要求:只对特定投标人适用(「如依法免税则须提供……」),规矩的投标人不适用,不能当作必须提供的材料 +CONDITIONAL = re.compile(r"(如|若|对于)[^,。;]{0,12}(免税|不需|无需|依法|属于|是)") DOC_REQ = re.compile(r"提供.{0,12}(方案|证明|报告|文件|承诺|截图|复印件|证书)") UNIT_DOWN = {"TB": ("GB", 1024), "GB": ("MB", 1024), "Gbps": ("Mbps", 1000), "s": ("ms", 1000), "秒": ("毫秒", 1000)} @@ -38,6 +43,7 @@ class Mutator: self.spec, self.vocab, self.rng = spec, vocab, rng self.unit_defects: dict[str, dict[str, Any]] = {} self.rule_defects: list[dict[str, Any]] = [] + self.umbrella: set[str] = set() self.vols = spec["volumes"] @property @@ -140,8 +146,8 @@ class Mutator: if DOC_REQ.search(req_text): # 「提供……方案 / 证明 / 报告」是文件要在(presence),不注入参数类问题 return False numeric = tp.get("op") in (">=", "<=", "=") and tp.get("value") - if op == "D05" and numeric: # 「暂不支持」只套在功能类要求上(尺寸、费用这类数值参数不会这么写) - return False + if op == "D05" and (numeric or not FUNCTIONAL.search(req_text) or NON_FUNCTIONAL.search(req_text)): + return False # 「暂不支持」只套在功能类要求上(数值参数、证书、清单、服务义务不会这么写) if op == "D01": t["rows"].remove(r) self.drop_everywhere(units) @@ -230,7 +236,19 @@ class Mutator: row["cells"][dev] = "负偏离" self.mark(row["responds"], "D06", "数值不满足并写负偏离") else: - param = tp.get("param") or (row["cells"][req][:12] if req is not None else "该功能") + req_text = row["cells"][req] if req is not None else "" + if not FUNCTIONAL.search(req_text) or NON_FUNCTIONAL.search(req_text): + row["cells"][dev] = self.phrase("D06", "deviations") # 非功能要求:写负偏离而不是「不支持」 + if note is not None: + row["cells"][note] = "" + self.mark(row["responds"], "D06", row["cells"][dev]) + if t["rows"] and "—" in t["rows"][0]["cells"]: + t["rows"].clear() + if self.col(t, "no") is not None: + row["cells"][self.col(t, "no")] = str(len(t["rows"]) + 1) + t["rows"].append(row) + continue + param = tp.get("param") or (req_text[:12] or "该功能") row["cells"][ans] = self.phrase("D05", "phrases").format(param=param) row["cells"][dev] = "负偏离" self.mark(row["responds"], "D05", row["cells"][ans]) @@ -240,12 +258,26 @@ class Mutator: row["cells"][self.col(t, "no")] = str(len(t["rows"]) + 1) t["rows"].append(row) - def attachments(self, n: int, cats: dict[str, str]) -> None: - """删掉附件:资格条件 → D11,评分项 → D13,其他 → D02。""" - cands = [b for b in self.blocks if b.get("attach") and self.free(b.get("responds") or [])] + def attachments(self, n: int, cats: dict[str, str], texts: dict[str, str]) -> None: + """删掉附件:资格条件 → D11,评分项 → D13,其他 → D02。 + 伞状单元(「第二十二条规定的条件」这类总述,挂在 3 份以上附件上)不挂问题:查标报的是具体条目,和总述对不上; + 条件性要求(「如依法免税则须提供」)规矩的投标人不适用,它的附件不删。""" + uses: dict[str, int] = {} + for b in self.blocks: + if b.get("attach"): + for u in b.get("responds") or []: + uses[u] = uses.get(u, 0) + 1 + self.umbrella = {u for u, k in uses.items() if k >= 3} + cands = [ + b + for b in self.blocks + if b.get("attach") + and self.free([u for u in b.get("responds") or [] if u not in self.umbrella]) + and not any(CONDITIONAL.search(texts.get(u, "")) for u in b.get("responds") or []) + ] self.rng.shuffle(cands) for b in cands[:n]: - units = b.get("responds") or [] + units = [u for u in b.get("responds") or [] if u not in self.umbrella] kinds = {cats.get(u) for u in units} defect = "D11" if "qualification" in kinds else "D13" if "scoring" in kinds else "D02" self.mark(units, defect, "附件缺失", worded=False) @@ -382,6 +414,10 @@ def main() -> None: tender = spec["tender"] units, _ = gold.units(tender) cats = {u.id: u.cat for u in units} + texts = { + u.id: "".join(str(x.get("t", "")) for x in u.raw.get("anchors") or []) + str(u.raw.get("note") or "") + for u in units + } optional = {u.id for u in units if u.optional} modes = unit_modes(spec) # 注入前算:被删掉的行也要知道它本来该怎么响应 m = Mutator(spec, json.loads(VOCAB[a.split].read_text()), random.Random(a.seed)) # noqa: S311 评测数据,不用于安全 @@ -390,13 +426,13 @@ def main() -> None: m.d01_block() m.row_ops(18) m.implicit_deviation(4) - m.attachments(4, cats) + m.attachments(4, cats, texts) m.compliance() for op in (m.p04, m.p05, m.p06, m.p07, m.p08, m.p09, m.p10, m.p12): op() else: m.row_ops(4) - m.attachments(1, cats) + m.attachments(1, cats, texts) m.p04() m.p07() bid = a.clean if a.manifest_only else a.out @@ -406,6 +442,10 @@ def main() -> None: for u in units: mode = modes.get(u.id) or ("na" if u.id in na else "unknown") entry: dict[str, Any] = {"cat": u.cat, "mode": mode, "optional": u.id in optional} + if u.id in m.umbrella and not a.manifest_only: + entry["expect"] = "unknown" # 伞状单元:附件删了一部分,算不算问题看具体条目,评分不算 + manifest_units[u.id] = entry + continue if u.id in m.unit_defects: entry.update({"expect": "defect", "reviewed": False, **m.unit_defects[u.id]}) elif mode == "implicit": diff --git a/evals/check/report.py b/evals/check/report.py index d41233f..a19b5aa 100644 --- a/evals/check/report.py +++ b/evals/check/report.py @@ -1,6 +1,6 @@ """汇总一批投标文件的查标评分(evals/check/GUIDELINES.md §7)。先对每份跑过 score.py。 - cd server && ../.venv/bin/python ../evals/check/report.py --run base <投标文件> [...] +cd server && ../.venv/bin/python ../evals/check/report.py --run base <投标文件> [...] """ from __future__ import annotations diff --git a/evals/check/run_check.py b/evals/check/run_check.py index 77c8770..c32b5db 100644 --- a/evals/check/run_check.py +++ b/evals/check/run_check.py @@ -119,6 +119,7 @@ async def main() -> None: ap.add_argument("--checklist", default="fix3") ap.add_argument("--name", default="base") ap.add_argument("--baseline", action="store_true", help="只检索、不调模型的基线") + ap.add_argument("--v1", action="store_true", help="第 1 步的做法:提示词 v1、不带同一行的单元格(对比基线)") args = ap.parse_args() d = OUT / args.bid manifest = json.loads((d / "manifest.json").read_text()) @@ -128,6 +129,9 @@ async def main() -> None: pages = text_layer.extract(f"{args.bid}-{pdf.stem}", pdf.read_bytes()) docs.append(match.BidDoc(uuid4(), n, pdf.stem, text_layer.body_sentences(pages))) ai_service.run_json = baseline_run_json if args.baseline else cached_run_json # type: ignore[assignment] + if args.v1: + match.PROMPT = ("check.match", 1) + match.ROW_MATES = 0 async def progress(_: float) -> None: return None