"""结构识别(最小版):给编号条目找到它的上级条款(架构 §3.5「结构识别」)。 用处:「25.3 未实质性响应磋商文件的情形」下面列出的 25.3.1~25.3.11,每一条本身都不写「无效」, 要靠上级标题才知道是无效情形。抽取时把上级标题作为上下文给模型,依据校验也认上级标题。 认这几种写法: 1. 十进制分级编号:25.3 → 25.3.1,上级是编号去掉最后一段的那一句; 2. 冒号引出的列表:「有下列情形之一的,投标无效:」后面跟(1)(2)……或 1. 2. ……或 ①②…… 3. 中文编号的小节:「三、符合条件(必须提供,否则投标无效)」下面的 1. 2. 3.……(S02 第 20 页); 4. 以冒号结尾的编号条目:「1.报价符合性审查:」下面不带编号的句子,到下一个编号为止。 上级还可以有上级(第 4 种的上级常常是第 3 种),用 ancestors() 取整条链。 另外三种结构(S01–S10 评测,docs/architecture/extract-eval.md §3): - chapters():每句属于哪一章(公告、须知、合同、采购需求、评审、格式……),抽取任务据此跳过不相干的章; - trailing_notes():表格、列表下方往回指的说明(「上述资格证明文件 1~6 项必须全部提供,否则投标无效」); - star_sections():★ 标在节标题上(「三、★商务要求」),整节都是实质性要求(口径 Q1)。 TODO(结构识别):表格结构,分包识别。 """ from __future__ import annotations import re from bid_doctor.document.pipeline.text_layer import Sentence DOTTED = re.compile(r"^(\d{1,3}(?:\.\d{1,3})+)(?![.\d])") LIST_ITEM = re.compile(r"^((\d{1,2})|\(\d{1,2}\)|\d{1,2}[.、.](?!\d)|[①-⑳]|[a-z][.、))])") CHAPTER = re.compile(r"^第[一二三四五六七八九十百\d]+[章节]") CN_SECTION = re.compile(r"^(?:[一二三四五六七八九十]{1,3}、|([一二三四五六七八九十]{1,3}))") LEAD_MAX = 30 # 冒号引出的列表,最多往后认这么多句 CN_NUM = "零一二三四五六七八九十" def _list_no(text: str) -> int | None: """列表条目的序号:(1)(1) 1. 1、 ① a. 都换成整数。""" m = LIST_ITEM.match(text) if not m: return None token = m.group(1) if token[0] in "①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳": return ord(token[0]) - ord("①") + 1 if token[0].isascii() and token[0].isalpha(): return ord(token[0]) - ord("a") + 1 return int(re.sub(r"\D", "", token)) def parents(sentences: list[Sentence]) -> dict[str, Sentence]: """返回 {句子编号: 上级条款那一句}。只给能确定的条目,拿不准的不给。""" out: dict[str, Sentence] = {} by_number: dict[str, Sentence] = {} lead: Sentence | None = None lead_left = 0 expect = 1 # 冒号引出的列表:序号要从 1 开始连续,接不上就是列表结束了(例如下一节的「26.评审」) section: Sentence | None = None # 中文编号的小节标题(第 3 种) section_expect = 1 sub: Sentence | None = None # 以冒号结尾的编号条目(第 4 种) for s in sentences: text = s.text.strip() m = DOTTED.match(text) no = _list_no(text) if CHAPTER.match(text) or m: section, sub = None, None if CHAPTER.match(text): lead, by_number = None, {} if CN_SECTION.match(text): section, section_expect, sub = s, 1, None elif lead is None and section is not None and not m and no is not None: if no == section_expect: out[s.key] = section section_expect += 1 sub = s if text.endswith((":", ":")) else None else: section, sub = None, None # 序号接不上:小节里的列表结束了 elif sub is not None and no is None and not m: out[s.key] = sub if m: number = m.group(1) parent_no = number.rsplit(".", 1)[0] if "." in number and parent_no in by_number: out[s.key] = by_number[parent_no] by_number[number] = s lead = None elif lead is not None and lead_left > 0 and no is not None: if no == expect: out[s.key] = lead expect += 1 lead_left -= 1 else: lead = None elif lead is not None and lead_left > 0 and expect > 1: lead_left -= 1 # 列表条目被句号切成了几句:接在条目后面的句子不单独算 if text.endswith((":", ":")) and no is None: lead, lead_left, expect = s, LEAD_MAX, 1 return out def ancestors(key: str, parents: dict[str, Sentence], depth: int = 3) -> list[Sentence]: """上级、上级的上级……(最多 depth 层)。""" out: list[Sentence] = [] while key in parents and len(out) < depth: p = parents[key] out.append(p) key = p.key return out # ---------------------------------------------------------------- 章 # 章标题:「第四章采购需求」,也可能前面粘着页眉(S03「常州市政府采购项目公开招标文件第三章资格审查」) CHAPTER_TITLE = re.compile(r"第([一二三四五六七八九十]{1,3})章\s*(\S{2,16})") # 章的类型按标题关键词判断,先判断的优先(「合同授予」不是合同,「主要条款及格式」算合同) CHAPTER_KINDS = ( ("award", ("授予",)), ("complaint", ("质疑", "投诉")), ("notice", ("公告", "邀请")), ("instructions", ("须知", "资料表")), ("qualification", ("资格审查",)), ("evaluation", ("评标", "评审", "评分")), ("contract", ("合同", "条款")), ("requirements", ("需求", "技术规格", "技术要求")), ("formats", ("格式", "组成")), ) def cn_int(s: str) -> int: """「三」→ 3,「十二」→ 12(只处理章节编号用到的范围)。""" if "十" not in s: return CN_NUM.index(s) if len(s) == 1 else 0 head, _, tail = s.partition("十") return (CN_NUM.index(head) if head else 1) * 10 + (CN_NUM.index(tail) if tail else 0) def chapters(sentences: list[Sentence]) -> dict[str, str]: """{句子编号: 章的类型}。认不出章的句子不在里面(调用方按「不跳过」处理)。 只认正文里的章标题:句子短(不超过 40 字)、不是目录(目录行带省略号,或者前后紧挨着另一个章标题,中间只隔页码)、 不是引用(「见第一章《投标邀请》」「(8)第三章评标标准中……」)、章号比上一章大。 认出来的章不可靠时(不到 3 章,或最后一章占了全文一半以上)返回空:按章跳过的任务就看全文,宁可多看不漏看。 """ parents_of = parents(sentences) texts = [s.text.strip() for s in sentences] cands: list[tuple[int, int, str]] = [] # (句子位置, 章号, 类型) for i, s in enumerate(sentences): text = texts[i] m = CHAPTER_TITLE.search(text) if not m or len(text) > 40 or "…" in text or "...." in text or "见" in text[: m.start()] or "《" in text: continue if LIST_ITEM.match(text) or DOTTED.match(text) or s.key in parents_of or re.match(r"[第\d]", m.group(2)): continue # 编号条目里提到某一章、引用「第二章第26.1.2条」,不是章标题 before = [t for t in texts[max(0, i - 3) : i] if not t.isdigit()][-1:] after = [t for t in texts[i + 1 : i + 4] if not t.isdigit()][:1] if any(CHAPTER_TITLE.match(t) and len(t) <= 40 for t in before + after): continue # 目录:几个章标题挨着(中间可能隔着页码) kind = next((k for k, words in CHAPTER_KINDS if any(w in m.group(2) for w in words)), "other") cands.append((i, cn_int(m.group(1)), kind)) # 章号要严格递增:取最长的递增序列,正文里偶尔提到「第八章」的句子就被排除了(S04 p14)。 # 一样长时取靠后的(目录在前、正文在后) best: list[list[tuple[int, int, str]]] = [] for c in cands: prev = max((b for b in best if b[-1][1] < c[1]), key=lambda b: (len(b), b[0][0]), default=[]) best.append([*prev, c]) heads = [(i, kind) for i, _, kind in max(best, key=lambda b: (len(b), b[0][0]), default=[])] spans = list(zip(heads, [*heads[1:], (len(sentences), "")], strict=False)) # 目录那一串胜出时,正文会全被算成最后一章:最后一章超过全文一半就不可靠(采购需求章本身可以很长,S06) if len(heads) < 3 or len(sentences) - heads[-1][0] > 0.5 * len(sentences): return {} out: dict[str, str] = {} for (start, kind), (end, _) in spans: for s in sentences[start:end]: out[s.key] = kind return out # ---------------------------------------------------------------- 表后往回指的说明 TRAILING_NOTE = re.compile( r"^(?:(\d{1,2})|\(\d{1,2}\)|\d{1,2}[.、.]|总?备注[::]|注[::]|说明[::])?\s*(?:上述|以上)" ) CONSEQUENCE = re.compile(r"无效|不实质性响应|未实质性响应|不合格|否决|不予受理|拒绝") TRAILING_MAX = 40 # 往回最多认这么多句 CN_TOP = re.compile(r"^[一二三四五六七八九十]{1,3}、") TRAILING_ROW_MAX = 60 # 说明的是标注方式,或者说的是「不导致无效」:不是往回指的无效说明(顾问审读第 5 条) NOT_CONSEQUENCE = re.compile(r"★|*|\*|斜体|下划线|非实质|不导致|不属于|不作为|不视为") def trailing_notes(sentences: list[Sentence]) -> dict[str, Sentence]: """表格、列表下方往回指的说明:{被说明的句子编号: 说明那一句}。 「(1)上述资格证明文件 1~6 项必须全部提供……否则投标文件无效」写在资格证明表的下面(S05 p54), 表里的每一行本身不写「无效」,上级条款也没有;只往上找上级的话认不出来(评测 §3.2,33 条漏抽)。 范围:从说明往回,到章、「一、」这样的中文编号小节、十进制编号条款为止(「附件十一、……」这样的附件标题 算在里面),最多 TRAILING_MAX 句、不超过前两页。 """ out: dict[str, Sentence] = {} for i, s in enumerate(sentences): text = s.text.strip() if not (TRAILING_NOTE.match(text) and CONSEQUENCE.search(text)) or NOT_CONSEQUENCE.search(text): continue for x in reversed(sentences[max(0, i - TRAILING_MAX) : i]): xt = x.text.strip() if x.page_no < s.page_no - 2 or CHAPTER.match(xt) or CN_TOP.match(xt) or DOTTED.match(xt): break if TRAILING_NOTE.match(xt) and CONSEQUENCE.search(xt): continue # 同一组说明里的另一条 if xt.endswith((":", ":")) and len(xt) <= 8: continue # 「说明:」这类引导句 if not (LIST_ITEM.match(xt) or DOTTED.match(xt) or len(xt) <= TRAILING_ROW_MAX): continue # 只覆盖编号条目和表格行,不覆盖夹在中间的解释段落(S05 p54「专门面向中小企业,是指……」) out.setdefault(x.key, s) if xt.startswith("附件"): break return out # ---------------------------------------------------------------- ★ 节 # 编号 + ★ 的节标题:「三、★商务要求」「★(一)技术要求」「8★售后服务(质保期)」 STAR_HEADING = re.compile( r"^(?P★)?\s*(?:(?P[一二三四五六七八九十]{1,3})、|((?P[一二三四五六七八九十]{1,3}))|(?P\d{1,2})(?![\d..])[、]?)" r"\s*(?P★)?" ) # 节标题去掉编号和 ★ 后的文字:短、不带标点和数字(「★5、采用2*15W功放芯片,声音清晰;」是参数行,不是节标题) STAR_TITLE = re.compile(r"^[^,,;;::。\d]{2,12}(?:([^)]{1,8}))?$") STAR_SCOPE_MAX = 800 def _heading(text: str) -> tuple[str, int] | None: """节编号的写法和序号:("cn", 3)("paren", 1)("num", 8)。""" m = STAR_HEADING.match(text) if not m or not any(m.group(k) for k in ("cn", "paren", "num")): return None for family in ("cn", "paren", "num"): if m.group(family): v = m.group(family) return family, int(v) if family == "num" else cn_int(v) return None def star_sections(sentences: list[Sentence]) -> dict[str, Sentence]: """★ 标在节标题上时,节里的每一句:{句子编号: 节标题}(标题本身不在里面,它自己带 ★)。 节标题:带编号、带 ★、不超过 20 字、不以句号结尾。节到下一个同级编号(三 → 四、(一)→(二)、8 → 9)、 更高一级的标题或下一章为止。 """ out: dict[str, Sentence] = {} for i, s in enumerate(sentences): text = s.text.strip() h = _heading(text) m = STAR_HEADING.match(text) if h is None or m is None or not (m.group("s1") or m.group("s2")) or not STAR_TITLE.match(text[m.end() :]): continue family, no = h higher = {"cn": (), "paren": ("cn",), "num": ("cn", "paren")}[family] for x in sentences[i + 1 : i + 1 + STAR_SCOPE_MAX]: xt = x.text.strip() if CHAPTER.match(xt) or (CHAPTER_TITLE.search(xt[:40]) and len(xt) <= 40): break xh = _heading(xt) if xh and (xh[0] in higher or (xh[0] == family and xh[1] == no + 1)): break out.setdefault(x.key, s) return out