feat: 读标补齐五类清单与项目基本信息,修复国动样本暴露的 4 个问题
- 文字层 v2:页眉页脚带单独成句;写到页尾的句子和下一页首句接续(出处两页各一条) - 结构识别(最小版):十进制编号、冒号引出的列表找上级条款;上级条款写明无效的,条目照样认定 - 抽取:资格、评分、日期与递交、技术参数、项目基本信息各一份提示词;实质性条款出 v2 (认上级条款、补「做无效响应处理」等写法);各类取值核对;按首句合并; 拆开被合成一条的列举条款;描述标注方式的句子和表头碎片不当条款;* 号标注 - 项目:解析出的名称、编号、采购人、预算、截止时间、分包写回项目,用户改过的不覆盖 - 前端:清单项显示分值、参数要求、日期、证明材料 - 测试 40 个,新增两页虚构样例 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
68
server/src/bid_doctor/document/pipeline/structure.py
Normal file
68
server/src/bid_doctor/document/pipeline/structure.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""结构识别(最小版):给编号条目找到它的上级条款(架构 §3.5「结构识别」)。
|
||||
|
||||
用处:「25.3 未实质性响应磋商文件的情形」下面列出的 25.3.1~25.3.11,每一条本身都不写「无效」,
|
||||
要靠上级标题才知道是无效情形。抽取时把上级标题作为上下文给模型,依据校验也认上级标题。
|
||||
|
||||
只认两种最常见的写法:
|
||||
1. 十进制分级编号:25.3 → 25.3.1,上级是编号去掉最后一段的那一句;
|
||||
2. 冒号引出的列表:「有下列情形之一的,投标无效:」后面跟(1)(2)……或 1. 2. ……或 ①②……
|
||||
TODO(结构识别):章、节层级,表格结构,分包识别。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from bid_doctor.document.pipeline.text_layer import Sentence
|
||||
|
||||
DOTTED = re.compile(r"^(\d{1,3}(?:\.\d{1,3})+)(?![.\d])")
|
||||
LIST_ITEM = re.compile(r"^((\d{1,2})|\(\d{1,2}\)|\d{1,2}[.、.](?!\d)|[①-⑳]|[a-z][.、))])")
|
||||
CHAPTER = re.compile(r"^第[一二三四五六七八九十百\d]+[章节]")
|
||||
LEAD_MAX = 30 # 冒号引出的列表,最多往后认这么多句
|
||||
|
||||
|
||||
def _list_no(text: str) -> int | None:
|
||||
"""列表条目的序号:(1)(1) 1. 1、 ① a. 都换成整数。"""
|
||||
m = LIST_ITEM.match(text)
|
||||
if not m:
|
||||
return None
|
||||
token = m.group(1)
|
||||
if token[0] in "①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳":
|
||||
return ord(token[0]) - ord("①") + 1
|
||||
if token[0].isascii() and token[0].isalpha():
|
||||
return ord(token[0]) - ord("a") + 1
|
||||
return int(re.sub(r"\D", "", token))
|
||||
|
||||
|
||||
def parents(sentences: list[Sentence]) -> dict[str, Sentence]:
|
||||
"""返回 {句子编号: 上级条款那一句}。只给能确定的条目,拿不准的不给。"""
|
||||
out: dict[str, Sentence] = {}
|
||||
by_number: dict[str, Sentence] = {}
|
||||
lead: Sentence | None = None
|
||||
lead_left = 0
|
||||
expect = 1 # 冒号引出的列表:序号要从 1 开始连续,接不上就是列表结束了(例如下一节的「26.评审」)
|
||||
for s in sentences:
|
||||
text = s.text.strip()
|
||||
m = DOTTED.match(text)
|
||||
if CHAPTER.match(text):
|
||||
lead, by_number = None, {}
|
||||
no = _list_no(text)
|
||||
if m:
|
||||
number = m.group(1)
|
||||
parent_no = number.rsplit(".", 1)[0]
|
||||
if "." in number and parent_no in by_number:
|
||||
out[s.key] = by_number[parent_no]
|
||||
by_number[number] = s
|
||||
lead = None
|
||||
elif lead is not None and lead_left > 0 and no is not None:
|
||||
if no == expect:
|
||||
out[s.key] = lead
|
||||
expect += 1
|
||||
lead_left -= 1
|
||||
else:
|
||||
lead = None
|
||||
elif lead is not None and lead_left > 0 and expect > 1:
|
||||
lead_left -= 1 # 列表条目被句号切成了几句:接在条目后面的句子不单独算
|
||||
if text.endswith((":", ":")) and no is None:
|
||||
lead, lead_left, expect = s, LEAD_MAX, 1
|
||||
return out
|
||||
Reference in New Issue
Block a user