feat(server): 工程骨架后端——上传 → 解析 → 清单最小链路
- core：配置（prod 拒绝默认密钥 / dev 短信 / 假模型）、租户事务 + 行级安全策略、 problem+json 错误码、信封加密与签名下载、对象存储、事务内入队、Valkey stream 事件流、权限点 - identity：手机号 HMAC 存储、注册、登录（挤下线需确认，API3）、服务端会话 - billing：额度流水只追加，预扣 / 结算 / 退回，一笔预扣只有一个结局 - document：分片上传、按文件头识别格式、Gotenberg 转换、文字层与稳定句子编号 - checklist：标注规则识别、模型抽取、三道校验、标注类条款规则兜底、重跑对齐 - ai：提示词注册表（extract.substantive v1 来自 S4 v2）、假模型、OpenAI 兼容供应商、ai_call 流水 - 迁移：data-model 基线、套餐、Procrastinate、会话状态函数 - 测试 30 个：租户隔离、额度、失败退回、对象加密、契约一致性 - Compose（含 migrate）、镜像、CI Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
YANG JIANKUAN
0
server/src/bid_doctor/document/pipeline/__init__.py
Normal file
0
server/src/bid_doctor/document/pipeline/__init__.py
Normal file
60
server/src/bid_doctor/document/pipeline/convert.py
Normal file
60
server/src/bid_doctor/document/pipeline/convert.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""转换：Word → 规范化 PDF（架构 §3.5，技术验证 S1）。PDF 原样作为规范化 PDF。
|
||||
|
||||
TODO(S1)：
|
||||
- 转换镜像要带按宋体行高调整过的字体包（BD Song / BD Hei）和 fontconfig 替换规则，现在用的是 Gotenberg 默认字体，
|
||||
分页偏差会偏大（S1：默认配置 +17%～+37%）；
|
||||
- WPS 生成的旧版 DOC 要先转 DOCX、修正字体字符集再转 PDF，否则正文可能是空白（S1 §1.3）。现在直接转。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
import httpx
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.errors import AppError
|
||||
|
||||
CJK_LATIN = {"SimSun", "NSimSun", "SimHei", "FangSong", "KaiTi", "Microsoft YaHei", "DengXian"}
|
||||
TIMEOUT = httpx.Timeout(300, connect=10) # 服务端 240s 先超时，错误由转换服务明确返回
|
||||
|
||||
|
||||
def fix_wps_charset(docx: bytes) -> bytes:
|
||||
"""WPS 把中文字体标成 SYMBOL(02) 字符集时改成 GB2312(86)，否则 LibreOffice 会用符号字体排版（S1）。"""
|
||||
src = zipfile.ZipFile(io.BytesIO(docx))
|
||||
if "word/fontTable.xml" not in src.namelist():
|
||||
return docx
|
||||
|
||||
def repl(m: re.Match[str]) -> str:
|
||||
name, body = m.group(1), m.group(2)
|
||||
if (re.search(r"[一-鿿]", name) or name in CJK_LATIN) and 'w:charset w:val="02"' in body:
|
||||
body = body.replace('w:charset w:val="02"', 'w:charset w:val="86"')
|
||||
return f'<w:font w:name="{name}">{body}</w:font>'
|
||||
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst:
|
||||
for item in src.infolist():
|
||||
data = src.read(item.filename)
|
||||
if item.filename == "word/fontTable.xml":
|
||||
data = re.sub(
|
||||
r'<w:font w:name="([^"]+)">(.*?)</w:font>', repl, data.decode("utf-8"), flags=re.S
|
||||
).encode()
|
||||
dst.writestr(item, data)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
async def to_pdf(data: bytes, fmt: str) -> bytes:
|
||||
if fmt == "pdf":
|
||||
return data
|
||||
if fmt == "docx":
|
||||
data = fix_wps_charset(data)
|
||||
async with httpx.AsyncClient(base_url=get_settings().gotenberg_url, timeout=TIMEOUT) as c:
|
||||
r = await c.post("/forms/libreoffice/convert", files={"files": (f"input.{fmt}", data)})
|
||||
if r.status_code == 200:
|
||||
return r.content
|
||||
if r.status_code in (400, 422): # 文件本身转不了（加密、损坏）：输入问题，不扣点
|
||||
raise AppError("file_corrupted", "这份 Word 文件无法转换，可能已加密或损坏")
|
||||
r.raise_for_status()
|
||||
raise AssertionError("unreachable")
|
||||
194
server/src/bid_doctor/document/pipeline/runner.py
Normal file
194
server/src/bid_doctor/document/pipeline/runner.py
Normal file
@@ -0,0 +1,194 @@
|
||||
"""解析流水线（架构 §3.4、§3.5）：格式识别 → 转换 → 文字层 → OCR 补全 → 抽取 → 证据校验 → 清单合并。
|
||||
|
||||
- 每个阶段结束写检查点；重跑时已完成的阶段直接跳过（规范化 PDF 只生成一次，本来也不能重写）。
|
||||
- 阶段之间检查取消标记。
|
||||
- 结局由错误类别决定扣不扣点：基础设施错误、输入问题、取消 → 退回；成功 → 结算（架构 §3.6 的错误分类表）。
|
||||
TODO(结构识别)：章节、表格、分包识别还没做，现在只抽实质性条款。TODO(S5)：OCR 还没接，没有文字层的页直接报「需要 OCR」。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from bid_doctor.ai.providers.base import TransientError
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.checklist import extract
|
||||
from bid_doctor.checklist import service as checklist
|
||||
from bid_doctor.core import events, keys
|
||||
from bid_doctor.core.crypto import open_object, seal_object, sha256
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import service as projects
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
PROGRESS = {"detect": 5, "convert": 15, "text_layer": 40, "ocr": 45, "extract": 55, "verify": 90, "merge": 95}
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v1", "sentence_split": "s3-v1"}
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def run(tenant_id: UUID, job_id: UUID) -> None:
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
if job.status not in ("queued", "running"):
|
||||
return # 重复派发：已经有结局了
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=job.created_by, actor_name="系统")
|
||||
job.status, job.started_at = "running", job.started_at or datetime.now(UTC)
|
||||
try:
|
||||
result = await _stages(ctx, job_id)
|
||||
except Cancelled:
|
||||
await _finish(ctx, job_id, "cancelled", "cancelled", None)
|
||||
except AppError as e:
|
||||
await _finish(ctx, job_id, "failed", "input", e.code)
|
||||
except (TransientError, httpx.HTTPError) as e:
|
||||
log.warning("parse job %s infra error: %s", job_id, e)
|
||||
code = "model_unavailable" if isinstance(e, TransientError) else "converter_unavailable"
|
||||
await _finish(ctx, job_id, "failed", "infra", code)
|
||||
except Exception:
|
||||
log.exception("parse job %s crashed", job_id)
|
||||
await _finish(ctx, job_id, "failed", "infra", "internal_error")
|
||||
else:
|
||||
await _finish(ctx, job_id, "succeeded", None, None, result)
|
||||
|
||||
|
||||
async def _stage(ctx: TenantContext, job_id: UUID, stage: str) -> dict[str, Any]:
|
||||
"""进入一个阶段：写进度、检查取消标记，返回已有的检查点。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
if job.cancel_requested_at:
|
||||
raise Cancelled
|
||||
job.stage, job.progress = stage, PROGRESS[stage]
|
||||
checkpoints = dict(job.checkpoints or {})
|
||||
await events.publish(
|
||||
ctx.tenant_id,
|
||||
"job.progress",
|
||||
project_id=str(job.project_id),
|
||||
job={"kind": "parse_job", "id": str(job_id), "status": "running", "stage": stage, "progress": PROGRESS[stage]},
|
||||
)
|
||||
return checkpoints
|
||||
|
||||
|
||||
async def _stages(ctx: TenantContext, job_id: UUID) -> dict[str, int]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == job.document_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
|
||||
await _stage(ctx, job_id, "detect") # 上传完成时已识别过格式；这里只核对原件没被替换
|
||||
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
|
||||
if sha256(original) != doc.original_sha256:
|
||||
raise AppError("file_corrupted")
|
||||
|
||||
await _stage(ctx, job_id, "convert")
|
||||
if doc.normalized_key is None:
|
||||
pdf = await convert.to_pdf(original, doc.detected_format)
|
||||
key = f"{doc_prefix(doc)}/normalized.pdf"
|
||||
storage().put(key, seal_object(dk, pdf, key))
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
|
||||
d.pipeline_profile = PROFILE
|
||||
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
|
||||
else:
|
||||
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
|
||||
|
||||
await _stage(ctx, job_id, "text_layer")
|
||||
pages = text_layer.extract(str(doc.id), pdf)
|
||||
text_key = f"{doc_prefix(doc)}/text.json"
|
||||
storage().put(
|
||||
text_key, seal_object(dk, json.dumps([p.to_json() for p in pages], ensure_ascii=False).encode(), text_key)
|
||||
)
|
||||
async with tenant_tx(ctx) as s:
|
||||
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等：先清残留
|
||||
for p in pages:
|
||||
s.add(
|
||||
DocumentPage(
|
||||
tenant_id=ctx.tenant_id,
|
||||
document_id=doc.id,
|
||||
page_no=p.page_no,
|
||||
width_pt=p.width,
|
||||
height_pt=p.height,
|
||||
rotation=p.rotation,
|
||||
text_source=p.text_source,
|
||||
char_count=p.char_count,
|
||||
sentence_count=len(p.sentences),
|
||||
)
|
||||
)
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.text_layer_key, d.page_count, d.status = text_key, len(pages), "ready"
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(pages) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
|
||||
await _stage(ctx, job_id, "ocr")
|
||||
empty = sum(p.text_source == "empty" for p in pages)
|
||||
if pages and empty / len(pages) > 0.5:
|
||||
raise AppError("ocr_required")
|
||||
|
||||
await _stage(ctx, job_id, "extract")
|
||||
sentences = [x for p in pages for x in p.sentences]
|
||||
extracted, notes = await extract.run(ctx, job_id, sentences)
|
||||
|
||||
await _stage(ctx, job_id, "verify") # 三道校验在抽取时逐条做完；这里只是进度节点
|
||||
await _stage(ctx, job_id, "merge")
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
stats = await checklist.merge(s, ctx, d, job_id, extracted, notes)
|
||||
return {**stats, "lots_found": 0}
|
||||
|
||||
|
||||
async def _finish(
|
||||
ctx: TenantContext,
|
||||
job_id: UUID,
|
||||
status: str,
|
||||
error_class: str | None,
|
||||
code: str | None,
|
||||
result: dict[str, int] | None = None,
|
||||
) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
job.status, job.finished_at, job.error_class, job.error_code = status, datetime.now(UTC), error_class, code
|
||||
job.progress = 100 if status == "succeeded" else job.progress
|
||||
if result is not None:
|
||||
job.checkpoints = {**(job.checkpoints or {}), "result": result}
|
||||
if job.quota_hold_id:
|
||||
if status == "succeeded":
|
||||
await billing.settle(s, ctx, job.quota_hold_id)
|
||||
else:
|
||||
await billing.release(s, ctx, job.quota_hold_id) # 失败、超时、取消都不扣点（红线 9）
|
||||
if status == "succeeded":
|
||||
await projects.mark_drafting(s, await projects.get(s, ctx, job.project_id))
|
||||
if status != "cancelled":
|
||||
s.add(
|
||||
Notification(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
user_id=job.created_by,
|
||||
kind="parse_succeeded" if status == "succeeded" else "parse_failed",
|
||||
project_id=job.project_id,
|
||||
payload={"job_id": str(job.id)},
|
||||
)
|
||||
)
|
||||
project_id = job.project_id
|
||||
await events.publish(
|
||||
ctx.tenant_id,
|
||||
"job.finished",
|
||||
project_id=str(project_id),
|
||||
job={"kind": "parse_job", "id": str(job_id), "status": status, "progress": 100},
|
||||
)
|
||||
225
server/src/bid_doctor/document/pipeline/text_layer.py
Normal file
225
server/src/bid_doctor/document/pipeline/text_layer.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""文字层：从规范化 PDF 取逐字坐标和格式 → 切句 → 每句生成 quads 和稳定编号（架构 §3.5，技术验证 S2、S3）。
|
||||
|
||||
切句规则：
|
||||
1. 同一行里字间空隙大于 1.5 个字号时断开（表格单元格、制表位），得到「行片段」；
|
||||
2. 句子在 。！？； 处结束；行片段没写到右边距附近（段落末行）时也结束；
|
||||
3. 句子不跨页。编号 <文档>:p<页>:s<页内序号>；规范化 PDF 不再重新生成，所以编号稳定。
|
||||
格式：斜体看字体名或字形倾斜；下划线看字下方的横线；加粗看字体名（S2：斜体且下划线按字符 100% 一致）。
|
||||
TODO(S3 改进)：表格按单元格切句；页眉页脚带单独识别（现在都算正文）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pdfplumber
|
||||
|
||||
END = set("。！？；")
|
||||
WS = re.compile(r"[\s ]")
|
||||
# pdfminer 对缺 FontBBox 的字体逐字报警告，没有信息量
|
||||
logging.getLogger("pdfminer").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
# 部首补充区没有 NFKC 映射，简化字形的部首要单独查表
|
||||
RADICAL_SUPPLEMENT = dict(
|
||||
zip(
|
||||
"⺟⺠⻅⻆⻉⻋⻓⻔⻘⻚⻛⻜⻝⻢⻣⻤⻥⻦⻧⻨⻩⻬⻮⻰⻳",
|
||||
"母民见角贝车长门青页风飞食马骨鬼鱼鸟卤麦黄齐齿龙龟",
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _char(t: str) -> str:
|
||||
"""有的 PDF（如 Chromium 生成的）把汉字映射成康熙部首或部首补充区的码位（「目」成了 U+2F6C）。
|
||||
只把这两个区换回常用汉字；全角标点等其他字符保持原样，否则引文和原文对不上。"""
|
||||
if len(t) != 1:
|
||||
return t
|
||||
if "⼀" <= t <= "":
|
||||
return unicodedata.normalize("NFKC", t)
|
||||
return RADICAL_SUPPLEMENT.get(t, t)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Sentence:
|
||||
key: str
|
||||
page_no: int
|
||||
seq: int
|
||||
text: str
|
||||
quads: list[list[float]]
|
||||
fmt_runs: list[tuple[int, int, str]] = field(default_factory=list) # [起, 止, 格式]，格式取 b / i / u 的组合
|
||||
location: str = "body"
|
||||
|
||||
@property
|
||||
def marked(self) -> str:
|
||||
"""给模型看的文本：斜体且下划线 → <iu>，只下划线 → <u>，只斜体 → <i>（S4 的输入格式）。"""
|
||||
out, pos = [], 0
|
||||
for start, end, f in self.fmt_runs:
|
||||
tag = "iu" if "i" in f and "u" in f else "u" if "u" in f else "i" if "i" in f else ""
|
||||
if not tag:
|
||||
continue
|
||||
out.append(self.text[pos:start])
|
||||
out.append(f"<{tag}>{self.text[start:end]}</{tag}>")
|
||||
pos = end
|
||||
out.append(self.text[pos:])
|
||||
return "".join(out)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"key": self.key,
|
||||
"seq": self.seq,
|
||||
"text": self.text,
|
||||
"quads": self.quads,
|
||||
"format_runs": [list(r) for r in self.fmt_runs],
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, page_no: int, d: dict[str, Any]) -> Sentence:
|
||||
return cls(
|
||||
key=d["key"],
|
||||
page_no=page_no,
|
||||
seq=d["seq"],
|
||||
text=d["text"],
|
||||
quads=d["quads"],
|
||||
fmt_runs=[(a, b, f) for a, b, f in d["format_runs"]],
|
||||
location=d["location"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Page:
|
||||
page_no: int
|
||||
width: float
|
||||
height: float
|
||||
rotation: int
|
||||
char_count: int
|
||||
sentences: list[Sentence]
|
||||
|
||||
@property
|
||||
def text_source(self) -> str:
|
||||
return "text" if self.char_count >= 20 else "empty"
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"page_no": self.page_no,
|
||||
"width_pt": self.width,
|
||||
"height_pt": self.height,
|
||||
"rotation": self.rotation,
|
||||
"char_count": self.char_count,
|
||||
"sentences": [s.to_json() for s in self.sentences],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, d: dict[str, Any]) -> Page:
|
||||
return cls(
|
||||
page_no=d["page_no"],
|
||||
width=d["width_pt"],
|
||||
height=d["height_pt"],
|
||||
rotation=d["rotation"],
|
||||
char_count=d["char_count"],
|
||||
sentences=[Sentence.from_json(d["page_no"], s) for s in d["sentences"]],
|
||||
)
|
||||
|
||||
|
||||
def _flags(c: dict[str, Any], hlines: list[tuple[float, float, float]]) -> str:
|
||||
name = str(c["fontname"]).split("+")[-1]
|
||||
italic = "Italic" in name or "Oblique" in name or abs(c["matrix"][2]) > 0.1
|
||||
mid = (c["x0"] + c["x1"]) / 2
|
||||
# 下划线一般在基线下 1～2pt；字体下行部分小时会落在字符框内，所以窗口向上放宽 0.25 个字号
|
||||
under = any(x0 - 0.5 <= mid <= x1 + 0.5 and -0.25 * c["size"] <= y - c["bottom"] <= 4 for x0, x1, y in hlines)
|
||||
return ("b" if "Bold" in name else "") + ("i" if italic else "") + ("u" if under else "")
|
||||
|
||||
|
||||
def _fragments(chars: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
|
||||
chars = sorted(chars, key=lambda c: (round(c["bottom"]), c["x0"]))
|
||||
lines: list[list[dict[str, Any]]] = []
|
||||
cur: list[dict[str, Any]] = []
|
||||
base: float | None = None
|
||||
for c in chars:
|
||||
if base is not None and abs(c["bottom"] - base) > 0.45 * c["size"]:
|
||||
lines.append(cur)
|
||||
cur = []
|
||||
if not cur:
|
||||
base = c["bottom"]
|
||||
cur.append(c)
|
||||
if cur:
|
||||
lines.append(cur)
|
||||
frags: list[list[dict[str, Any]]] = []
|
||||
for ln in lines:
|
||||
ln.sort(key=lambda c: c["x0"])
|
||||
f = [ln[0]]
|
||||
for a, b in itertools.pairwise(ln):
|
||||
if b["x0"] - a["x1"] > 1.5 * a["size"]:
|
||||
frags.append(f)
|
||||
f = []
|
||||
f.append(b)
|
||||
frags.append(f)
|
||||
return frags
|
||||
|
||||
|
||||
def _page(doc_id: str, page: Any) -> Page:
|
||||
hlines = [(ln["x0"], ln["x1"], ln["top"]) for ln in page.lines if abs(ln["top"] - ln["bottom"]) < 0.5]
|
||||
hlines += [(r["x0"], r["x1"], r["top"]) for r in page.rects if r["height"] < 1.5 and r["width"] > 2]
|
||||
chars = [c for c in page.chars if str(c["text"]).strip()]
|
||||
right = max((c["x1"] for c in chars), default=page.width)
|
||||
groups: list[list[tuple[int, dict[str, Any]]]] = []
|
||||
cur: list[tuple[int, dict[str, Any]]] = []
|
||||
for fi, frag in enumerate(_fragments(chars)):
|
||||
for c in frag:
|
||||
cur.append((fi, c))
|
||||
if c["text"] in END:
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
if cur and frag[-1]["x1"] < right - 2 * frag[-1]["size"]:
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
if cur:
|
||||
groups.append(cur)
|
||||
sentences: list[Sentence] = []
|
||||
for seq, g in enumerate(groups, 1):
|
||||
quads: dict[int, list[float]] = {}
|
||||
runs: list[tuple[int, int, str]] = []
|
||||
text_parts: list[str] = []
|
||||
for pos, (fi, c) in enumerate(g):
|
||||
q = quads.setdefault(fi, [c["x0"], c["top"], c["x1"], c["bottom"]])
|
||||
q[0], q[1], q[2], q[3] = min(q[0], c["x0"]), min(q[1], c["top"]), max(q[2], c["x1"]), max(q[3], c["bottom"])
|
||||
text_parts.append(_char(str(c["text"])))
|
||||
f = _flags(c, hlines)
|
||||
if f and runs and runs[-1][1] == pos and runs[-1][2] == f:
|
||||
runs[-1] = (runs[-1][0], pos + 1, f)
|
||||
elif f:
|
||||
runs.append((pos, pos + 1, f))
|
||||
sentences.append(
|
||||
Sentence(
|
||||
key=f"{doc_id}:p{page.page_number}:s{seq}",
|
||||
page_no=page.page_number,
|
||||
seq=seq,
|
||||
text="".join(text_parts),
|
||||
quads=[[round(v, 2) for v in q] for q in quads.values()],
|
||||
fmt_runs=runs,
|
||||
)
|
||||
)
|
||||
return Page(
|
||||
page_no=page.page_number,
|
||||
width=float(page.width),
|
||||
height=float(page.height),
|
||||
rotation=int(page.rotation or 0) % 360,
|
||||
char_count=len(chars),
|
||||
sentences=sentences,
|
||||
)
|
||||
|
||||
|
||||
def extract(doc_id: str, pdf: bytes) -> list[Page]:
|
||||
with pdfplumber.open(io.BytesIO(pdf)) as doc:
|
||||
return [_page(doc_id, p) for p in doc.pages]
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
return WS.sub("", s)
|
||||
Reference in New Issue
Block a user