feat: 读标补齐五类清单与项目基本信息，修复国动样本暴露的 4 个问题
- 文字层 v2：页眉页脚带单独成句；写到页尾的句子和下一页首句接续（出处两页各一条） - 结构识别（最小版）：十进制编号、冒号引出的列表找上级条款；上级条款写明无效的，条目照样认定 - 抽取：资格、评分、日期与递交、技术参数、项目基本信息各一份提示词；实质性条款出 v2 （认上级条款、补「做无效响应处理」等写法）；各类取值核对；按首句合并； 拆开被合成一条的列举条款；描述标注方式的句子和表头碎片不当条款；* 号标注 - 项目：解析出的名称、编号、采购人、预算、截止时间、分包写回项目，用户改过的不覆盖 - 前端：清单项显示分值、参数要求、日期、证明材料 - 测试 40 个，新增两页虚构样例 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
@@ -28,14 +28,25 @@ from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.pipeline import convert, structure, text_layer
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import extracted as project_extracted
|
||||
from bid_doctor.project import service as projects
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
PROGRESS = {"detect": 5, "convert": 15, "text_layer": 40, "ocr": 45, "extract": 55, "verify": 90, "merge": 95}
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v1", "sentence_split": "s3-v1"}
|
||||
PROGRESS = {
|
||||
"detect": 5,
|
||||
"convert": 15,
|
||||
"text_layer": 30,
|
||||
"ocr": 35,
|
||||
"structure": 40,
|
||||
"extract": 45,
|
||||
"verify": 92,
|
||||
"merge": 95,
|
||||
}
|
||||
# text_layer v2：页眉页脚带单独成句，句子编号和 v1 不同（同一份规范化 PDF 在 v1、v2 下的编号不能混用）
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v2", "sentence_split": "s3-v2"}
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
@@ -67,6 +78,20 @@ async def run(tenant_id: UUID, job_id: UUID) -> None:
|
||||
await _finish(ctx, job_id, "succeeded", None, None, result)
|
||||
|
||||
|
||||
async def _progress(ctx: TenantContext, job_id: UUID, stage: str, pct: int) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
if job.cancel_requested_at:
|
||||
raise Cancelled
|
||||
job.progress = pct
|
||||
await events.publish(
|
||||
ctx.tenant_id,
|
||||
"job.progress",
|
||||
project_id=str(job.project_id),
|
||||
job={"kind": "parse_job", "id": str(job_id), "status": "running", "stage": stage, "progress": pct},
|
||||
)
|
||||
|
||||
|
||||
async def _stage(ctx: TenantContext, job_id: UUID, stage: str) -> dict[str, Any]:
|
||||
"""进入一个阶段：写进度、检查取消标记，返回已有的检查点。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
@@ -141,16 +166,38 @@ async def _stages(ctx: TenantContext, job_id: UUID) -> dict[str, int]:
|
||||
if pages and empty / len(pages) > 0.5:
|
||||
raise AppError("ocr_required")
|
||||
|
||||
await _stage(ctx, job_id, "structure")
|
||||
body = text_layer.body_sentences(pages)
|
||||
notes = extract.marking_notes(body)
|
||||
doc_info = extract.Doc(
|
||||
sentences=body,
|
||||
parents=structure.parents(body),
|
||||
continuations=text_layer.continuations(body),
|
||||
notes=notes,
|
||||
declared={m for n in notes for m in n.markers},
|
||||
)
|
||||
|
||||
await _stage(ctx, job_id, "extract")
|
||||
sentences = [x for p in pages for x in p.sentences]
|
||||
extracted, notes = await extract.run(ctx, job_id, sentences)
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
job.prompt_versions = extract.prompt_versions()
|
||||
last = [PROGRESS["extract"]]
|
||||
|
||||
async def progress(ratio: float) -> None:
|
||||
pct = PROGRESS["extract"] + int((PROGRESS["verify"] - PROGRESS["extract"]) * ratio)
|
||||
if pct > last[0]: # 只在百分比变了时写库、推送
|
||||
last[0] = pct
|
||||
await _progress(ctx, job_id, "extract", pct)
|
||||
|
||||
items, info = await extract.run(ctx, job_id, doc_info, progress)
|
||||
|
||||
await _stage(ctx, job_id, "verify") # 三道校验在抽取时逐条做完；这里只是进度节点
|
||||
await _stage(ctx, job_id, "merge")
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
stats = await checklist.merge(s, ctx, d, job_id, extracted, notes)
|
||||
return {**stats, "lots_found": 0}
|
||||
lots = await project_extracted.apply(s, ctx, d, info, extract.legal_regime_by_rule(body), items)
|
||||
stats = await checklist.merge(s, ctx, d, job_id, items, notes, lots)
|
||||
return {**stats, "lots_found": len([k for k in lots if k != "默认"])}
|
||||
|
||||
|
||||
async def _finish(
|
||||
|
||||
68
server/src/bid_doctor/document/pipeline/structure.py
Normal file
68
server/src/bid_doctor/document/pipeline/structure.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""结构识别（最小版）：给编号条目找到它的上级条款（架构 §3.5「结构识别」）。
|
||||
|
||||
用处：「25.3 未实质性响应磋商文件的情形」下面列出的 25.3.1～25.3.11，每一条本身都不写「无效」，
|
||||
要靠上级标题才知道是无效情形。抽取时把上级标题作为上下文给模型，依据校验也认上级标题。
|
||||
|
||||
只认两种最常见的写法：
|
||||
1. 十进制分级编号：25.3 → 25.3.1，上级是编号去掉最后一段的那一句；
|
||||
2. 冒号引出的列表：「有下列情形之一的，投标无效：」后面跟（1）（2）……或 1. 2. ……或 ①②……
|
||||
TODO(结构识别)：章、节层级，表格结构，分包识别。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from bid_doctor.document.pipeline.text_layer import Sentence
|
||||
|
||||
DOTTED = re.compile(r"^(\d{1,3}(?:\.\d{1,3})+)(?![.\d])")
|
||||
LIST_ITEM = re.compile(r"^(（\d{1,2}）|\(\d{1,2}\)|\d{1,2}[.、．](?!\d)|[①-⑳]|[a-z][.、)）])")
|
||||
CHAPTER = re.compile(r"^第[一二三四五六七八九十百\d]+[章节]")
|
||||
LEAD_MAX = 30 # 冒号引出的列表，最多往后认这么多句
|
||||
|
||||
|
||||
def _list_no(text: str) -> int | None:
|
||||
"""列表条目的序号：（1）(1) 1. 1、 ① a. 都换成整数。"""
|
||||
m = LIST_ITEM.match(text)
|
||||
if not m:
|
||||
return None
|
||||
token = m.group(1)
|
||||
if token[0] in "①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳":
|
||||
return ord(token[0]) - ord("①") + 1
|
||||
if token[0].isascii() and token[0].isalpha():
|
||||
return ord(token[0]) - ord("a") + 1
|
||||
return int(re.sub(r"\D", "", token))
|
||||
|
||||
|
||||
def parents(sentences: list[Sentence]) -> dict[str, Sentence]:
|
||||
"""返回 {句子编号: 上级条款那一句}。只给能确定的条目，拿不准的不给。"""
|
||||
out: dict[str, Sentence] = {}
|
||||
by_number: dict[str, Sentence] = {}
|
||||
lead: Sentence | None = None
|
||||
lead_left = 0
|
||||
expect = 1 # 冒号引出的列表：序号要从 1 开始连续，接不上就是列表结束了（例如下一节的「26.评审」）
|
||||
for s in sentences:
|
||||
text = s.text.strip()
|
||||
m = DOTTED.match(text)
|
||||
if CHAPTER.match(text):
|
||||
lead, by_number = None, {}
|
||||
no = _list_no(text)
|
||||
if m:
|
||||
number = m.group(1)
|
||||
parent_no = number.rsplit(".", 1)[0]
|
||||
if "." in number and parent_no in by_number:
|
||||
out[s.key] = by_number[parent_no]
|
||||
by_number[number] = s
|
||||
lead = None
|
||||
elif lead is not None and lead_left > 0 and no is not None:
|
||||
if no == expect:
|
||||
out[s.key] = lead
|
||||
expect += 1
|
||||
lead_left -= 1
|
||||
else:
|
||||
lead = None
|
||||
elif lead is not None and lead_left > 0 and expect > 1:
|
||||
lead_left -= 1 # 列表条目被句号切成了几句：接在条目后面的句子不单独算
|
||||
if text.endswith(("：", ":")) and no is None:
|
||||
lead, lead_left, expect = s, LEAD_MAX, 1
|
||||
return out
|
||||
@@ -55,6 +55,7 @@ class Sentence:
|
||||
quads: list[list[float]]
|
||||
fmt_runs: list[tuple[int, int, str]] = field(default_factory=list) # [起, 止, 格式]，格式取 b / i / u 的组合
|
||||
location: str = "body"
|
||||
open_end: bool = False # 写到页面末尾还没结束（没有句末标点、末行写满），可能接续到下一页
|
||||
|
||||
@property
|
||||
def marked(self) -> str:
|
||||
@@ -78,6 +79,7 @@ class Sentence:
|
||||
"quads": self.quads,
|
||||
"format_runs": [list(r) for r in self.fmt_runs],
|
||||
"location": self.location,
|
||||
"open_end": self.open_end,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -90,6 +92,7 @@ class Sentence:
|
||||
quads=d["quads"],
|
||||
fmt_runs=[(a, b, f) for a, b, f in d["format_runs"]],
|
||||
location=d["location"],
|
||||
open_end=d.get("open_end", False),
|
||||
)
|
||||
|
||||
|
||||
@@ -164,26 +167,64 @@ def _fragments(chars: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
|
||||
return frags
|
||||
|
||||
|
||||
def _page(doc_id: str, page: Any) -> Page:
|
||||
BAND = 0.07 # 页眉页脚带：页面上、下各 7% 的高度
|
||||
PAGE_NO = re.compile(r"^[—\-–－]*\s*(第\s*)?\d+\s*(页|/\s*\d+|页\s*共\s*\d+\s*页)?\s*[—\-–－]*$")
|
||||
DIGITS = re.compile(r"\d+")
|
||||
|
||||
|
||||
def _frag_text(frag: list[dict[str, Any]]) -> str:
|
||||
return norm("".join(_char(str(c["text"])) for c in frag))
|
||||
|
||||
|
||||
def _in_band(frag: list[dict[str, Any]], height: float) -> bool:
|
||||
return all(c["bottom"] <= height * BAND for c in frag) or all(c["top"] >= height * (1 - BAND) for c in frag)
|
||||
|
||||
|
||||
def _band_repeats(pages: list[tuple[Any, list[list[dict[str, Any]]]]]) -> set[str]:
|
||||
"""在多页的页眉页脚带里重复出现的文字（数字换成 #，页码不同也算同一行），判为页眉页脚。"""
|
||||
seen: dict[str, set[int]] = {}
|
||||
for page, frags in pages:
|
||||
for f in frags:
|
||||
if _in_band(f, float(page.height)):
|
||||
seen.setdefault(DIGITS.sub("#", _frag_text(f)), set()).add(page.page_number)
|
||||
need = max(3, int(len(pages) * 0.3))
|
||||
return {t for t, ps in seen.items() if t and len(ps) >= need}
|
||||
|
||||
|
||||
def _page(doc_id: str, page: Any, frags: list[list[dict[str, Any]]], repeats: set[str]) -> Page:
|
||||
"""页眉页脚带里的页码和重复行单独成句（location = header_footer），不并进正文句子（S3 的改进点 2）。"""
|
||||
hlines = [(ln["x0"], ln["x1"], ln["top"]) for ln in page.lines if abs(ln["top"] - ln["bottom"]) < 0.5]
|
||||
hlines += [(r["x0"], r["x1"], r["top"]) for r in page.rects if r["height"] < 1.5 and r["width"] > 2]
|
||||
chars = [c for c in page.chars if str(c["text"]).strip()]
|
||||
right = max((c["x1"] for c in chars), default=page.width)
|
||||
groups: list[list[tuple[int, dict[str, Any]]]] = []
|
||||
height = float(page.height)
|
||||
|
||||
def is_band(f: list[dict[str, Any]]) -> bool:
|
||||
t = _frag_text(f)
|
||||
return _in_band(f, height) and (bool(PAGE_NO.match(t)) or DIGITS.sub("#", t) in repeats)
|
||||
|
||||
body = [f for f in frags if not is_band(f)]
|
||||
right = max((c["x1"] for f in body for c in f), default=page.width)
|
||||
groups: list[tuple[str, list[tuple[int, dict[str, Any]]]]] = []
|
||||
open_tail = False
|
||||
cur: list[tuple[int, dict[str, Any]]] = []
|
||||
for fi, frag in enumerate(_fragments(chars)):
|
||||
for fi, frag in enumerate(frags):
|
||||
if is_band(frag):
|
||||
groups.append(("header_footer", [(fi, c) for c in frag]))
|
||||
continue
|
||||
for c in frag:
|
||||
cur.append((fi, c))
|
||||
if c["text"] in END:
|
||||
groups.append(cur)
|
||||
groups.append(("body", cur))
|
||||
cur = []
|
||||
if cur and frag[-1]["x1"] < right - 2 * frag[-1]["size"]:
|
||||
groups.append(cur)
|
||||
groups.append(("body", cur))
|
||||
cur = []
|
||||
if cur:
|
||||
groups.append(cur)
|
||||
groups.append(("body", cur))
|
||||
open_tail = True # 页面写完了句子还没结束
|
||||
tail_seq = max((i for i, (loc, _) in enumerate(groups, 1) if loc == "body"), default=0) if open_tail else 0
|
||||
sentences: list[Sentence] = []
|
||||
for seq, g in enumerate(groups, 1):
|
||||
for seq, (location, g) in enumerate(groups, 1):
|
||||
quads: dict[int, list[float]] = {}
|
||||
runs: list[tuple[int, int, str]] = []
|
||||
text_parts: list[str] = []
|
||||
@@ -204,6 +245,8 @@ def _page(doc_id: str, page: Any) -> Page:
|
||||
text="".join(text_parts),
|
||||
quads=[[round(v, 2) for v in q] for q in quads.values()],
|
||||
fmt_runs=runs,
|
||||
location=location,
|
||||
open_end=seq == tail_seq,
|
||||
)
|
||||
)
|
||||
return Page(
|
||||
@@ -218,7 +261,38 @@ def _page(doc_id: str, page: Any) -> Page:
|
||||
|
||||
def extract(doc_id: str, pdf: bytes) -> list[Page]:
|
||||
with pdfplumber.open(io.BytesIO(pdf)) as doc:
|
||||
return [_page(doc_id, p) for p in doc.pages]
|
||||
pages = [(p, _fragments([c for c in p.chars if str(c["text"]).strip()])) for p in doc.pages]
|
||||
repeats = _band_repeats(pages)
|
||||
return [_page(doc_id, p, frags, repeats) for p, frags in pages]
|
||||
|
||||
|
||||
CLOSERS = set("”’）)》」』】")
|
||||
|
||||
|
||||
def body_sentences(pages: list[Page]) -> list[Sentence]:
|
||||
"""正文句子（去掉页眉页脚），按阅读顺序。"""
|
||||
return [s for p in pages for s in p.sentences if s.location != "header_footer"]
|
||||
|
||||
|
||||
HEADING = re.compile(r"^(第[一二三四五六七八九十百\d]+[章节条]|[一二三四五六七八九十]+[、.．]|\d+(\.\d+)*[、.．]?\D)")
|
||||
|
||||
|
||||
def continuations(sentences: list[Sentence]) -> dict[str, Sentence]:
|
||||
"""跨页接续：一页的最后一句写到页尾还没结束，下一页第一句又不是新的标题或编号，就是同一句话。
|
||||
返回 {上一页末句编号: 下一页首句}。句子本身不跨页（出处按页定位，S3），引用时两页各出一条出处。
|
||||
"""
|
||||
out: dict[str, Sentence] = {}
|
||||
for a, b in itertools.pairwise(sentences):
|
||||
tail = a.text.rstrip().rstrip("".join(CLOSERS))
|
||||
if (
|
||||
a.open_end
|
||||
and b.page_no == a.page_no + 1
|
||||
and tail[-1:] not in ("：", ":") # 「全称（公章）：」这类填空行
|
||||
and not HEADING.match(b.text)
|
||||
and not b.text.startswith("序号") # 下一页是新表格的表头
|
||||
):
|
||||
out[a.key] = b
|
||||
return out
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user