feat(server): 工程骨架后端——上传 → 解析 → 清单最小链路
- core:配置(prod 拒绝默认密钥 / dev 短信 / 假模型)、租户事务 + 行级安全策略、 problem+json 错误码、信封加密与签名下载、对象存储、事务内入队、Valkey stream 事件流、权限点 - identity:手机号 HMAC 存储、注册、登录(挤下线需确认,API3)、服务端会话 - billing:额度流水只追加,预扣 / 结算 / 退回,一笔预扣只有一个结局 - document:分片上传、按文件头识别格式、Gotenberg 转换、文字层与稳定句子编号 - checklist:标注规则识别、模型抽取、三道校验、标注类条款规则兜底、重跑对齐 - ai:提示词注册表(extract.substantive v1 来自 S4 v2)、假模型、OpenAI 兼容供应商、ai_call 流水 - 迁移:data-model 基线、套餐、Procrastinate、会话状态函数 - 测试 30 个:租户隔离、额度、失败退回、对象加密、契约一致性 - Compose(含 migrate)、镜像、CI Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
0
server/src/bid_doctor/__init__.py
Normal file
0
server/src/bid_doctor/__init__.py
Normal file
0
server/src/bid_doctor/ai/__init__.py
Normal file
0
server/src/bid_doctor/ai/__init__.py
Normal file
31
server/src/bid_doctor/ai/models.py
Normal file
31
server/src/bid_doctor/ai/models.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.orm import Mapped
|
||||
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class AiCall(Base):
|
||||
"""调用流水账:不存原文(架构 §3.6)。"""
|
||||
|
||||
__tablename__ = "ai_call"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
job_type: Mapped[str]
|
||||
job_id: Mapped[UuidCol]
|
||||
task: Mapped[str]
|
||||
capability: Mapped[str]
|
||||
provider: Mapped[str]
|
||||
model: Mapped[str]
|
||||
prompt_id: Mapped[str]
|
||||
prompt_version: Mapped[str]
|
||||
prompt_hash: Mapped[Bytes]
|
||||
input_hash: Mapped[Bytes]
|
||||
output_hash: Mapped[Bytes | None]
|
||||
tokens_in: Mapped[int | None]
|
||||
tokens_out: Mapped[int | None]
|
||||
latency_ms: Mapped[int | None]
|
||||
output_mode: Mapped[str | None]
|
||||
error_class: Mapped[str | None]
|
||||
created_at: Mapped[DbTs]
|
||||
38
server/src/bid_doctor/ai/prompts.py
Normal file
38
server/src/bid_doctor/ai/prompts.py
Normal file
@@ -0,0 +1,38 @@
|
||||
"""提示词注册表:启动时加载 prompts/ 下的文件并计算内容哈希(架构 §3.6)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_DIR = Path(__file__).resolve().parents[4] / "prompts"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Prompt:
|
||||
id: str
|
||||
version: str
|
||||
capability: str
|
||||
body: str
|
||||
sha256: bytes
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get(prompt_id: str, version: int) -> Prompt:
|
||||
base = Path(os.environ.get("BD_PROMPTS_DIR", DEFAULT_DIR))
|
||||
raw = (base / prompt_id / f"v{version}.md").read_text(encoding="utf-8")
|
||||
_, head, body = raw.split("---", 2)
|
||||
meta = dict(line.split(":", 1) for line in head.strip().splitlines())
|
||||
meta = {k.strip(): v.strip() for k, v in meta.items()}
|
||||
if meta["id"] != prompt_id or meta["version"] != str(version):
|
||||
raise ValueError(f"提示词文件头和路径不一致:{prompt_id} v{version}")
|
||||
return Prompt(
|
||||
id=prompt_id,
|
||||
version=f"v{version}",
|
||||
capability=meta["capability"],
|
||||
body=body.strip(),
|
||||
sha256=hashlib.sha256(raw.encode()).digest(),
|
||||
)
|
||||
0
server/src/bid_doctor/ai/providers/__init__.py
Normal file
0
server/src/bid_doctor/ai/providers/__init__.py
Normal file
28
server/src/bid_doctor/ai/providers/base.py
Normal file
28
server/src/bid_doctor/ai/providers/base.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Completion:
|
||||
data: dict[str, Any]
|
||||
raw: str
|
||||
model: str
|
||||
tokens_in: int | None = None
|
||||
tokens_out: int | None = None
|
||||
output_mode: str = "json_mode"
|
||||
|
||||
|
||||
class TransientError(Exception):
|
||||
"""限流、5xx、网络抖动:可以重试;重试后仍失败按基础设施错误处理(退回额度)。"""
|
||||
|
||||
def __init__(self, error_class: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.error_class = error_class
|
||||
|
||||
|
||||
class Provider(Protocol):
|
||||
name: str
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str) -> Completion: ...
|
||||
40
server/src/bid_doctor/ai/providers/fake.py
Normal file
40
server/src/bid_doctor/ai/providers/fake.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""确定性的假模型:开发和测试用,CI 不调用真实模型(架构 §8)。
|
||||
|
||||
只认 extract.substantive:把带 ★ 或 <iu> 的句子当作实质性条款,逐字摘一段原文作为引文。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from bid_doctor.ai.providers.base import Completion
|
||||
|
||||
LINE = re.compile(r"^\[([^\]]+)\] (.*)$")
|
||||
TAG = re.compile(r"</?(iu|u|i)>")
|
||||
|
||||
|
||||
class FakeProvider:
|
||||
name = "fake"
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str) -> Completion:
|
||||
del system, capability
|
||||
items = []
|
||||
for line in user.split("## 待审分块", 1)[-1].splitlines():
|
||||
m = LINE.match(line.strip())
|
||||
if not m:
|
||||
continue
|
||||
sid, marked = m.groups()
|
||||
plain = TAG.sub("", marked)
|
||||
if "★" in plain:
|
||||
basis = "star"
|
||||
elif "<iu>" in marked:
|
||||
basis = "italic_underline"
|
||||
else:
|
||||
continue
|
||||
if any(k in plain for k in ("标注", "实质性要求", "条款响应")) and len(plain) < 60:
|
||||
continue # 只是在说明标注规则的句子,不算条款(和提示词的「不要输出」一致)
|
||||
body = plain.lstrip("★** ").strip()
|
||||
items.append({"summary": body[:40], "basis": basis, "evidence": [{"sid": sid, "quote": body[:20]}]})
|
||||
data = {"items": items}
|
||||
return Completion(data=data, raw=json.dumps(data, ensure_ascii=False), model="fake-extractor")
|
||||
54
server/src/bid_doctor/ai/providers/openai_compat.py
Normal file
54
server/src/bid_doctor/ai/providers/openai_compat.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""OpenAI 兼容协议的国产模型(DeepSeek、通义千问、豆包、GLM、Kimi 等,T1)。
|
||||
|
||||
供应商地址和模型由运维在部署配置里设置,用户和租户都不能填(架构 §3.6)。
|
||||
TODO(路由表):按能力配候选列表、按「供应商 × 租户」限流;现在只有一个供应商一个模型。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
from bid_doctor.ai.providers.base import Completion, TransientError
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
class OpenAICompatProvider:
|
||||
name = "openai_compat"
|
||||
|
||||
async def complete_json(self, system: str, user: str, capability: str) -> Completion:
|
||||
del capability
|
||||
s = get_settings()
|
||||
body = {
|
||||
"model": s.ai_model,
|
||||
"temperature": 0,
|
||||
"response_format": {"type": "json_object"},
|
||||
"messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(base_url=s.ai_base_url, timeout=httpx.Timeout(180, connect=10)) as c:
|
||||
r = await c.post("/chat/completions", json=body, headers={"Authorization": f"Bearer {s.ai_api_key}"})
|
||||
except httpx.TimeoutException as e:
|
||||
raise TransientError("timeout", "模型调用超时") from e
|
||||
except httpx.TransportError as e:
|
||||
raise TransientError("network", "模型调用网络错误") from e
|
||||
if r.status_code == 429:
|
||||
raise TransientError("rate_limited", "模型接口限流")
|
||||
if r.status_code >= 500:
|
||||
raise TransientError("server", f"模型接口 {r.status_code}")
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
raw = payload["choices"][0]["message"]["content"]
|
||||
usage = payload.get("usage") or {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
data = {"items": []} # 内容质量问题:不算任务失败,相关条目由规则兜底进待确认区(架构 §3.6)
|
||||
return Completion(
|
||||
data=data,
|
||||
raw=raw,
|
||||
model=payload.get("model", s.ai_model),
|
||||
tokens_in=usage.get("prompt_tokens"),
|
||||
tokens_out=usage.get("completion_tokens"),
|
||||
)
|
||||
70
server/src/bid_doctor/ai/service.py
Normal file
70
server/src/bid_doctor/ai/service.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""模型调用的唯一入口:业务只声明任务和提示词版本,供应商由路由决定;每次调用写一条 ai_call(架构 §3.6)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from bid_doctor.ai import prompts
|
||||
from bid_doctor.ai.models import AiCall
|
||||
from bid_doctor.ai.providers.base import Completion, Provider, TransientError
|
||||
from bid_doctor.ai.providers.fake import FakeProvider
|
||||
from bid_doctor.ai.providers.openai_compat import OpenAICompatProvider
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import sha256
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.ids import uuid7
|
||||
|
||||
RETRIES = 2
|
||||
|
||||
|
||||
def provider() -> Provider:
|
||||
return OpenAICompatProvider() if get_settings().ai_provider == "openai_compat" else FakeProvider()
|
||||
|
||||
|
||||
async def run_json(
|
||||
ctx: TenantContext, *, job_type: str, job_id: UUID, task: str, prompt_id: str, version: int, user: str
|
||||
) -> dict[str, Any]:
|
||||
"""外部内容(招标文件)只放在 user 的数据区;系统提示词声明数据区里的指令一律不执行。"""
|
||||
p = prompts.get(prompt_id, version)
|
||||
prov = provider()
|
||||
started = time.monotonic()
|
||||
result: Completion | None = None
|
||||
error: TransientError | None = None
|
||||
for attempt in range(RETRIES + 1):
|
||||
try:
|
||||
result = await prov.complete_json(p.body, user, p.capability)
|
||||
error = None
|
||||
break
|
||||
except TransientError as e: # 只对临时错误重试
|
||||
error = e
|
||||
await asyncio.sleep(2**attempt)
|
||||
async with tenant_tx(ctx) as s:
|
||||
s.add(
|
||||
AiCall(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
job_type=job_type,
|
||||
job_id=job_id,
|
||||
task=task,
|
||||
capability=p.capability,
|
||||
provider=prov.name,
|
||||
model=result.model if result else get_settings().ai_model,
|
||||
prompt_id=p.id,
|
||||
prompt_version=p.version,
|
||||
prompt_hash=p.sha256,
|
||||
input_hash=sha256(user),
|
||||
output_hash=sha256(result.raw) if result else None,
|
||||
tokens_in=result.tokens_in if result else None,
|
||||
tokens_out=result.tokens_out if result else None,
|
||||
latency_ms=int((time.monotonic() - started) * 1000),
|
||||
output_mode=result.output_mode if result else None,
|
||||
error_class=error.error_class if error else None,
|
||||
)
|
||||
)
|
||||
if error:
|
||||
raise error
|
||||
assert result is not None
|
||||
return result.data
|
||||
62
server/src/bid_doctor/app.py
Normal file
62
server/src/bid_doctor/app.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""api 进程:只做请求处理和短事务,重活交给 worker(架构 §2)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from fastapi import FastAPI, Request, Response
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from bid_doctor.core import errors
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.ids import uuid7
|
||||
|
||||
API_PREFIX = "/api/v1"
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
from bid_doctor.billing.api import router as billing_router
|
||||
from bid_doctor.checklist.api import router as checklist_router
|
||||
from bid_doctor.document.api import router as document_router
|
||||
from bid_doctor.identity.api import router as identity_router
|
||||
from bid_doctor.notification.api import router as notification_router
|
||||
from bid_doctor.project.api import router as project_router
|
||||
|
||||
settings = get_settings()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s")
|
||||
app = FastAPI(
|
||||
title="标博士用户端接口",
|
||||
version="0.1.0",
|
||||
openapi_url=f"{API_PREFIX}/openapi.json",
|
||||
docs_url=f"{API_PREFIX}/docs" if settings.env != "prod" else None,
|
||||
redoc_url=None,
|
||||
)
|
||||
errors.install(app)
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_id(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response: # pyright: ignore[reportUnusedFunction]
|
||||
rid = request.headers.get("x-request-id") or f"req_{uuid7().hex}"
|
||||
request.state.request_id = rid
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-ID"] = rid
|
||||
return response
|
||||
|
||||
# 网页版默认同源;开发时 Vite 在另一个端口,桌面版用自定义协议(架构 §5.2)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
expose_headers=["X-Request-ID", "ETag"],
|
||||
)
|
||||
|
||||
for r in (identity_router, billing_router, project_router, document_router, checklist_router, notification_router):
|
||||
app.include_router(r, prefix=API_PREFIX)
|
||||
|
||||
@app.get("/healthz", include_in_schema=False)
|
||||
async def healthz() -> dict[str, str]: # pyright: ignore[reportUnusedFunction]
|
||||
return {"status": "ok"}
|
||||
|
||||
return app
|
||||
0
server/src/bid_doctor/audit/__init__.py
Normal file
0
server/src/bid_doctor/audit/__init__.py
Normal file
24
server/src/bid_doctor/audit/models.py
Normal file
24
server/src/bid_doctor/audit/models.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.orm import Mapped
|
||||
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, Json, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class AuditEvent(Base):
|
||||
__tablename__ = "audit_event"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
actor_kind: Mapped[str]
|
||||
actor_user_id: Mapped[UuidCol | None]
|
||||
actor_name: Mapped[str | None]
|
||||
action: Mapped[str]
|
||||
object_type: Mapped[str]
|
||||
object_id: Mapped[UuidCol | None]
|
||||
project_id: Mapped[UuidCol | None]
|
||||
before: Mapped[Json | None]
|
||||
after: Mapped[Json | None]
|
||||
content_digest: Mapped[Bytes | None]
|
||||
request_id: Mapped[str | None]
|
||||
created_at: Mapped[DbTs]
|
||||
50
server/src/bid_doctor/audit/service.py
Normal file
50
server/src/bid_doctor/audit/service.py
Normal file
@@ -0,0 +1,50 @@
|
||||
"""审计:和业务修改在同一个事务里写入,写失败整个操作就失败(架构 §3.10)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.audit.models import AuditEvent
|
||||
from bid_doctor.core.crypto import sha256
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.ids import uuid7
|
||||
|
||||
|
||||
async def record(
|
||||
session: AsyncSession,
|
||||
ctx: TenantContext,
|
||||
action: str,
|
||||
object_type: str,
|
||||
object_id: UUID | None,
|
||||
*,
|
||||
project_id: UUID | None = None,
|
||||
before: dict[str, Any] | None = None,
|
||||
after: dict[str, Any] | None = None,
|
||||
actor_kind: str = "user",
|
||||
) -> None:
|
||||
digest = (
|
||||
sha256(json.dumps([before, after], ensure_ascii=False, sort_keys=True, default=str))
|
||||
if before or after
|
||||
else None
|
||||
)
|
||||
session.add(
|
||||
AuditEvent(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
actor_kind=actor_kind,
|
||||
actor_user_id=ctx.user_id if actor_kind == "user" else None,
|
||||
actor_name=ctx.actor_name or None,
|
||||
action=action,
|
||||
object_type=object_type,
|
||||
object_id=object_id,
|
||||
project_id=project_id,
|
||||
before=before,
|
||||
after=after,
|
||||
content_digest=digest,
|
||||
request_id=ctx.request_id or None,
|
||||
)
|
||||
)
|
||||
0
server/src/bid_doctor/billing/__init__.py
Normal file
0
server/src/bid_doctor/billing/__init__.py
Normal file
31
server/src/bid_doctor/billing/api.py
Normal file
31
server/src/bid_doctor/billing/api.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter
|
||||
from sqlalchemy import select
|
||||
|
||||
from bid_doctor.billing import service
|
||||
from bid_doctor.billing.models import Plan
|
||||
from bid_doctor.core.db import anonymous_tx, tenant_tx
|
||||
from bid_doctor.identity.deps import Ctx
|
||||
|
||||
router = APIRouter(tags=["billing"])
|
||||
|
||||
|
||||
@router.get("/plans")
|
||||
async def list_plans() -> dict[str, Any]:
|
||||
async with anonymous_tx() as s:
|
||||
plans = (await s.execute(select(Plan).where(Plan.active).order_by(Plan.code))).scalars().all()
|
||||
return {
|
||||
"items": [
|
||||
{"code": p.code, "version": p.version, "name": p.name, "quotas": p.quotas, "limits": p.limits}
|
||||
for p in plans
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/quota")
|
||||
async def get_quota(ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.overview(s, ctx)
|
||||
59
server/src/bid_doctor/billing/models.py
Normal file
59
server/src/bid_doctor/billing/models.py
Normal file
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, DbTs, Json, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class Plan(Base):
|
||||
__tablename__ = "plan"
|
||||
code: Mapped[str] = mapped_column(primary_key=True)
|
||||
version: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str]
|
||||
quotas: Mapped[Json]
|
||||
limits: Mapped[Json]
|
||||
active: Mapped[bool] = mapped_column(server_default=FetchedValue())
|
||||
|
||||
|
||||
class Subscription(Base):
|
||||
__tablename__ = "subscription"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
plan_code: Mapped[str]
|
||||
plan_version: Mapped[int]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
source: Mapped[str]
|
||||
seats: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
starts_at: Mapped[Ts]
|
||||
ends_at: Mapped[Ts | None]
|
||||
|
||||
|
||||
class QuotaAccount(Base):
|
||||
__tablename__ = "quota_account"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
unit: Mapped[str]
|
||||
balance: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class QuotaEntry(Base):
|
||||
"""只追加。amount 是对余额的影响(data-model §4.2)。"""
|
||||
|
||||
__tablename__ = "quota_entry"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
account_id: Mapped[UuidCol]
|
||||
subscription_id: Mapped[UuidCol | None]
|
||||
kind: Mapped[str]
|
||||
amount: Mapped[int]
|
||||
idempotency_key: Mapped[str]
|
||||
hold_entry_id: Mapped[UuidCol | None]
|
||||
ref_type: Mapped[str | None]
|
||||
ref_id: Mapped[UuidCol | None]
|
||||
project_id: Mapped[UuidCol | None]
|
||||
reason: Mapped[str | None]
|
||||
actor_user_id: Mapped[UuidCol | None]
|
||||
created_at: Mapped[DbTs]
|
||||
191
server/src/bid_doctor/billing/service.py
Normal file
191
server/src/bid_doctor/billing/service.py
Normal file
@@ -0,0 +1,191 @@
|
||||
"""额度(架构 §3.9、data-model §4.2)。流水只追加;余额缓存和流水在同一个事务里更新。
|
||||
|
||||
任务的三种结局(成功结算、失败退回、取消退回)都调用这里;同一笔预扣只能有一个结局(数据库唯一索引保证)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.billing.models import Plan, QuotaAccount, QuotaEntry, Subscription
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
|
||||
|
||||
async def start_free_subscription(session: AsyncSession, ctx: TenantContext) -> None:
|
||||
"""注册时:免费版订阅 + 额度账户 + 发放。"""
|
||||
plan = (
|
||||
await session.execute(
|
||||
select(Plan).where(Plan.code == "free", Plan.active).order_by(Plan.version.desc()).limit(1)
|
||||
)
|
||||
).scalar_one()
|
||||
sub = Subscription(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
plan_code=plan.code,
|
||||
plan_version=plan.version,
|
||||
source="signup",
|
||||
starts_at=datetime.now(UTC),
|
||||
ends_at=None,
|
||||
)
|
||||
session.add(sub)
|
||||
await session.flush()
|
||||
for unit, amount in plan.quotas.items():
|
||||
acct = QuotaAccount(id=uuid7(), tenant_id=ctx.tenant_id, unit=unit, balance=0)
|
||||
session.add(acct)
|
||||
await session.flush()
|
||||
await _append(
|
||||
session,
|
||||
ctx,
|
||||
acct,
|
||||
kind="grant",
|
||||
amount=amount,
|
||||
key=f"grant:{sub.id}:{unit}",
|
||||
subscription_id=sub.id,
|
||||
reason="注册赠送",
|
||||
)
|
||||
|
||||
|
||||
async def current_plan(session: AsyncSession, ctx: TenantContext) -> tuple[Subscription, Plan]:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(Subscription, Plan)
|
||||
.join(Plan, (Plan.code == Subscription.plan_code) & (Plan.version == Subscription.plan_version))
|
||||
.where(Subscription.tenant_id == ctx.tenant_id, Subscription.status == "active")
|
||||
)
|
||||
).one()
|
||||
return row[0], row[1]
|
||||
|
||||
|
||||
async def hold(
|
||||
session: AsyncSession, ctx: TenantContext, *, unit: str, ref_type: str, ref_id: UUID, project_id: UUID
|
||||
) -> UUID:
|
||||
"""预扣 1 次。先锁账户行,同一企业的并发预扣因此串行,不会扣成负数。"""
|
||||
acct = (
|
||||
await session.execute(
|
||||
select(QuotaAccount)
|
||||
.where(QuotaAccount.tenant_id == ctx.tenant_id, QuotaAccount.unit == unit)
|
||||
.with_for_update()
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if acct is None or acct.balance < 1:
|
||||
raise AppError("quota_exhausted", "本期额度已用完。专业版续费或加量请联系客服。")
|
||||
entry = await _append(
|
||||
session,
|
||||
ctx,
|
||||
acct,
|
||||
kind="hold",
|
||||
amount=-1,
|
||||
key=f"{ref_type}:{ref_id}:hold",
|
||||
ref_type=ref_type,
|
||||
ref_id=ref_id,
|
||||
project_id=project_id,
|
||||
)
|
||||
return entry.id
|
||||
|
||||
|
||||
async def settle(session: AsyncSession, ctx: TenantContext, hold_id: UUID) -> None:
|
||||
await _finish(session, ctx, hold_id, "settle", 0)
|
||||
|
||||
|
||||
async def release(session: AsyncSession, ctx: TenantContext, hold_id: UUID) -> None:
|
||||
await _finish(session, ctx, hold_id, "release", 1)
|
||||
|
||||
|
||||
async def _finish(session: AsyncSession, ctx: TenantContext, hold_id: UUID, kind: str, amount: int) -> None:
|
||||
h = (await session.execute(select(QuotaEntry).where(QuotaEntry.id == hold_id))).scalar_one()
|
||||
acct = (
|
||||
await session.execute(select(QuotaAccount).where(QuotaAccount.id == h.account_id).with_for_update())
|
||||
).scalar_one()
|
||||
done = (
|
||||
await session.execute(
|
||||
text(
|
||||
"SELECT 1 FROM quota_entry WHERE tenant_id = :t AND hold_entry_id = :h"
|
||||
" AND kind IN ('settle', 'release')"
|
||||
),
|
||||
{"t": ctx.tenant_id, "h": hold_id},
|
||||
)
|
||||
).first()
|
||||
if done: # 重复执行(worker 重试)时什么都不做
|
||||
return
|
||||
await _append(
|
||||
session,
|
||||
ctx,
|
||||
acct,
|
||||
kind=kind,
|
||||
amount=amount,
|
||||
key=f"{h.ref_type}:{h.ref_id}:{kind}",
|
||||
hold_entry_id=hold_id,
|
||||
ref_type=h.ref_type,
|
||||
ref_id=h.ref_id,
|
||||
project_id=h.project_id,
|
||||
)
|
||||
|
||||
|
||||
async def _append(
|
||||
session: AsyncSession, ctx: TenantContext, acct: QuotaAccount, *, kind: str, amount: int, key: str, **fields: Any
|
||||
) -> QuotaEntry:
|
||||
entry = QuotaEntry(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
account_id=acct.id,
|
||||
kind=kind,
|
||||
amount=amount,
|
||||
idempotency_key=key,
|
||||
actor_user_id=ctx.user_id,
|
||||
**fields,
|
||||
)
|
||||
session.add(entry)
|
||||
acct.balance += amount
|
||||
await session.flush()
|
||||
return entry
|
||||
|
||||
|
||||
async def overview(session: AsyncSession, ctx: TenantContext) -> dict[str, Any]:
|
||||
sub, plan = await current_plan(session, ctx)
|
||||
# 每笔预扣扣 1 次:已用 = 预扣数 − 退回数;正在预扣 = 预扣数 − 退回数 − 结算数。
|
||||
# TODO(专业版续费):按订阅周期切分「本期」;MVP 的免费版只有一个周期。
|
||||
rows = (
|
||||
await session.execute(
|
||||
text(
|
||||
"SELECT a.unit, a.balance,"
|
||||
" coalesce(sum(e.amount) FILTER (WHERE e.kind = 'grant' AND e.subscription_id = :s), 0),"
|
||||
" count(e.id) FILTER (WHERE e.kind = 'hold'),"
|
||||
" count(e.id) FILTER (WHERE e.kind = 'release'),"
|
||||
" count(e.id) FILTER (WHERE e.kind = 'settle')"
|
||||
" FROM quota_account a LEFT JOIN quota_entry e ON e.account_id = a.id"
|
||||
" WHERE a.tenant_id = :t GROUP BY a.unit, a.balance ORDER BY a.unit"
|
||||
),
|
||||
{"t": ctx.tenant_id, "s": sub.id},
|
||||
)
|
||||
).all()
|
||||
accounts: list[dict[str, Any]] = []
|
||||
for unit, balance, granted, holds, releases, settles in rows:
|
||||
used = holds - releases
|
||||
accounts.append(
|
||||
{
|
||||
"unit": unit,
|
||||
"balance": balance,
|
||||
"held": used - settles,
|
||||
"period_granted": granted,
|
||||
"period_used": used,
|
||||
"used_ratio": round(used / granted, 3) if granted else 0.0,
|
||||
}
|
||||
)
|
||||
return {"plan": plan_summary(sub, plan), "accounts": accounts}
|
||||
|
||||
|
||||
def plan_summary(sub: Subscription, plan: Plan) -> dict[str, Any]:
|
||||
return {
|
||||
"code": plan.code,
|
||||
"name": plan.name,
|
||||
"starts_at": sub.starts_at,
|
||||
"ends_at": sub.ends_at,
|
||||
"read_only": False,
|
||||
}
|
||||
0
server/src/bid_doctor/checklist/__init__.py
Normal file
0
server/src/bid_doctor/checklist/__init__.py
Normal file
28
server/src/bid_doctor/checklist/api.py
Normal file
28
server/src/bid_doctor/checklist/api.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from bid_doctor.checklist import service
|
||||
from bid_doctor.core.db import tenant_tx
|
||||
from bid_doctor.identity.deps import Ctx
|
||||
from bid_doctor.project import service as projects
|
||||
|
||||
router = APIRouter(tags=["checklist"])
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/checklist")
|
||||
async def get_checklist(project_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
await projects.get(s, ctx, project_id)
|
||||
return await service.overview(s, project_id)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/checklist/items")
|
||||
async def list_items(project_id: UUID, ctx: Ctx, zone: Literal["pending", "formal"] | None = None) -> dict[str, Any]:
|
||||
# TODO(读标):category、lot_scope、include_deleted 筛选,以及修改、采纳、删除、确认清单等接口
|
||||
async with tenant_tx(ctx) as s:
|
||||
await projects.get(s, ctx, project_id)
|
||||
return {"items": await service.items(s, project_id, zone)}
|
||||
183
server/src/bid_doctor/checklist/extract.py
Normal file
183
server/src/bid_doctor/checklist/extract.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""实质性条款抽取:标注规则识别 → 模型抽取 → 三道校验 → 标注类条款规则兜底(架构 §3.6,技术验证 S4)。
|
||||
|
||||
只有三道校验都通过的条目才进正式清单;其余降级进待确认区。没被任何条目覆盖的标注句,系统自动生成一条「待确认」,
|
||||
所以 ★ 和斜体下划线条款的召回不完全依赖模型(TV1:标注类召回 ≥ 99%)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from bid_doctor.ai import service as ai
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.document.pipeline.text_layer import Sentence, norm
|
||||
|
||||
PROMPT_ID, PROMPT_VERSION = "extract.substantive", 1
|
||||
CHUNK_CHARS = 6000
|
||||
INVALID = re.compile(r"投标无效|响应无效|无效投标|作无效处理|按无效|否决其投标|否决投标|不予受理")
|
||||
RULE_NOTE = re.compile(r"(★|*|\*|斜体|下划线).{0,30}(实质性|无效|否决)|(实质性).{0,20}(★|*|\*|斜体|下划线)")
|
||||
DECLARED = re.compile(r"★[^。;]{0,12}?共\s*(\d+)\s*[条项处]")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Extracted:
|
||||
summary: str
|
||||
basis: str
|
||||
sentences: list[Sentence]
|
||||
quotes: list[str]
|
||||
zone: str
|
||||
pending_reason: str | None = None
|
||||
checks_passed: list[str] = field(default_factory=list)
|
||||
ai_values: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class MarkingNote:
|
||||
marker: str
|
||||
sentence: Sentence
|
||||
declared_count: int | None
|
||||
|
||||
|
||||
def is_marked(s: Sentence, declared: set[str]) -> str | None:
|
||||
"""确定性识别的标注:含 ★;文件声明了斜体下划线标注时,整句大部分是斜体且下划线。
|
||||
|
||||
斜体下划线只在文件的「标注规则说明」声明过时才算(07 F1):没声明的文件里,标题、表头也常是斜体或下划线。
|
||||
"""
|
||||
if "★" in s.text:
|
||||
return "star"
|
||||
iu = sum(e - b for b, e, f in s.fmt_runs if "i" in f and "u" in f)
|
||||
if "italic_underline" in declared and iu >= max(6, len(s.text) // 2):
|
||||
return "italic_underline"
|
||||
return None
|
||||
|
||||
|
||||
def marking_notes(sentences: list[Sentence]) -> list[MarkingNote]:
|
||||
notes: list[MarkingNote] = []
|
||||
for s in sentences:
|
||||
if not RULE_NOTE.search(s.text):
|
||||
continue
|
||||
marker = (
|
||||
"italic_underline" if ("斜体" in s.text and "下划线" in s.text) else "star" if "★" in s.text else "asterisk"
|
||||
)
|
||||
m = DECLARED.search(s.text)
|
||||
notes.append(MarkingNote(marker=marker, sentence=s, declared_count=int(m.group(1)) if m else None))
|
||||
return notes[:10]
|
||||
|
||||
|
||||
def _chunks(sentences: list[Sentence]) -> list[list[Sentence]]:
|
||||
chunks: list[list[Sentence]] = []
|
||||
cur: list[Sentence] = []
|
||||
size = 0
|
||||
for s in sentences:
|
||||
if not norm(s.text):
|
||||
continue
|
||||
line = len(s.marked) + 12
|
||||
if cur and size + line > CHUNK_CHARS:
|
||||
chunks.append(cur)
|
||||
cur, size = [], 0
|
||||
cur.append(s)
|
||||
size += line
|
||||
if cur:
|
||||
chunks.append(cur)
|
||||
return chunks
|
||||
|
||||
|
||||
def _sid(s: Sentence) -> str:
|
||||
"""给模型看的短编号(p12s3);完整编号带文档编号,太长。"""
|
||||
return f"p{s.page_no}s{s.seq}"
|
||||
|
||||
|
||||
def check_evidence(ev: dict[str, Any], by_id: dict[str, Sentence]) -> str | None:
|
||||
"""第一、二道:编号在本分块里;引文是原句的子串,且不少于 8 个字。"""
|
||||
sid, quote = str(ev.get("sid", "")), norm(str(ev.get("quote", "")))
|
||||
if sid not in by_id:
|
||||
return "id"
|
||||
text = norm(by_id[sid].text)
|
||||
if not quote or len(quote) < min(8, len(text)) or quote not in text:
|
||||
return "quote"
|
||||
return None
|
||||
|
||||
|
||||
def check_basis(basis: str, sents: list[Sentence], chunk: list[Sentence], declared: set[str]) -> bool:
|
||||
"""第三道:模型声称的依据,用确定性信号核对(S4:不加这一道,采纳条目只有 36.7% 属实)。"""
|
||||
if basis == "star":
|
||||
return any("★" in s.text for s in sents)
|
||||
if basis == "asterisk":
|
||||
return any(re.search(r"[**]", s.text) for s in sents)
|
||||
if basis == "italic_underline":
|
||||
return "italic_underline" in declared and any("<iu>" in s.marked for s in sents)
|
||||
if basis == "explicit_invalid":
|
||||
idx = {s.key: i for i, s in enumerate(chunk)}
|
||||
near = [chunk[j] for s in sents for j in (idx[s.key] - 1, idx[s.key], idx[s.key] + 1) if 0 <= j < len(chunk)]
|
||||
return any(INVALID.search(s.text) for s in near)
|
||||
return False
|
||||
|
||||
|
||||
async def run(ctx: TenantContext, job_id: UUID, sentences: list[Sentence]) -> tuple[list[Extracted], list[MarkingNote]]:
|
||||
notes = marking_notes(sentences)
|
||||
note_text = (
|
||||
"\n".join(f"- {n.sentence.text[:200]}" for n in notes)
|
||||
or "(未找到标注规则说明,请按常见标注方式和「投标无效 / 否决投标」字样判断)"
|
||||
)
|
||||
note_keys = {n.sentence.key for n in notes}
|
||||
declared = {n.marker for n in notes}
|
||||
out: list[Extracted] = []
|
||||
for chunk in _chunks(sentences):
|
||||
by_id = {_sid(s): s for s in chunk}
|
||||
user = (
|
||||
"## 标注规则说明\n" + note_text + "\n\n## 待审分块\n" + "\n".join(f"[{_sid(s)}] {s.marked}" for s in chunk)
|
||||
)
|
||||
data = await ai.run_json(
|
||||
ctx,
|
||||
job_type="parse_job",
|
||||
job_id=job_id,
|
||||
task="extract.substantive",
|
||||
prompt_id=PROMPT_ID,
|
||||
version=PROMPT_VERSION,
|
||||
user=user,
|
||||
)
|
||||
for it in data.get("items") or []:
|
||||
if not isinstance(it, dict):
|
||||
continue
|
||||
evs = [e for e in it.get("evidence") or [] if isinstance(e, dict)]
|
||||
if evs and all(by_id.get(str(e.get("sid"))) and by_id[str(e.get("sid"))].key in note_keys for e in evs):
|
||||
continue # 只是「标注规则说明」本身,不是条款(提示词要求不输出,这里再兜一道)
|
||||
fails = [check_evidence(e, by_id) for e in evs]
|
||||
ok = [by_id[str(e["sid"])] for e, f in zip(evs, fails, strict=True) if f is None]
|
||||
passed = [c for c in ("id", "quote") if ok and c not in fails]
|
||||
basis = str(it.get("basis", ""))
|
||||
basis_ok = bool(ok) and check_basis(basis, ok, chunk, declared)
|
||||
if basis_ok:
|
||||
passed.append("basis")
|
||||
accepted = bool(ok) and len(ok) == len(evs) and basis_ok
|
||||
out.append(
|
||||
Extracted(
|
||||
summary=str(it.get("summary", ""))[:200] or (ok[0].text[:40] if ok else "(模型未给出摘要)"),
|
||||
basis=basis,
|
||||
sentences=ok,
|
||||
quotes=[str(e.get("quote", "")) for e in evs],
|
||||
zone="formal" if accepted else "pending",
|
||||
pending_reason=None if accepted else ("no_citation" if not ok else "verify_failed"),
|
||||
checks_passed=passed,
|
||||
ai_values=it,
|
||||
)
|
||||
)
|
||||
covered = {s.key for e in out if e.zone == "formal" for s in e.sentences}
|
||||
for s in sentences: # 规则兜底:标注句没被正式条目覆盖 → 生成一条待确认
|
||||
marker = is_marked(s, declared)
|
||||
if marker and s.key not in covered and s.key not in note_keys and len(norm(s.text)) >= 6:
|
||||
out.append(
|
||||
Extracted(
|
||||
summary=s.text.lstrip("★ ")[:40],
|
||||
basis=marker,
|
||||
sentences=[s],
|
||||
quotes=[s.text],
|
||||
zone="pending",
|
||||
pending_reason="marker_uncovered",
|
||||
checks_passed=["id", "quote", "basis"],
|
||||
)
|
||||
)
|
||||
return out, notes
|
||||
72
server/src/bid_doctor/checklist/models.py
Normal file
72
server/src/bid_doctor/checklist/models.py
Normal file
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, DbTs, Json, TextArray, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class Checklist(Base):
|
||||
__tablename__ = "checklist"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
last_version_no: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
confirmed_version_id: Mapped[UuidCol | None]
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class MarkingRule(Base):
|
||||
__tablename__ = "marking_rule"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
document_id: Mapped[UuidCol]
|
||||
marker: Mapped[str]
|
||||
marker_text: Mapped[str | None]
|
||||
meaning: Mapped[str]
|
||||
declared_count: Mapped[int | None]
|
||||
citation_id: Mapped[UuidCol]
|
||||
origin: Mapped[str]
|
||||
|
||||
|
||||
class ChecklistItem(Base):
|
||||
__tablename__ = "checklist_item"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
checklist_id: Mapped[UuidCol]
|
||||
categories: Mapped[TextArray]
|
||||
title: Mapped[str]
|
||||
requirement_text: Mapped[str | None]
|
||||
summary: Mapped[str | None]
|
||||
importance: Mapped[str]
|
||||
importance_basis: Mapped[str | None]
|
||||
marking_rule_id: Mapped[UuidCol | None]
|
||||
lot_id: Mapped[UuidCol | None]
|
||||
evidence_required: Mapped[str | None]
|
||||
spec: Mapped[Json] = mapped_column(server_default=FetchedValue())
|
||||
origin: Mapped[str]
|
||||
origin_document_id: Mapped[UuidCol | None]
|
||||
zone: Mapped[str]
|
||||
pending_reason: Mapped[str | None]
|
||||
ai_values: Mapped[Json | None]
|
||||
edited_fields: Mapped[TextArray]
|
||||
confidence: Mapped[Decimal | None]
|
||||
last_parse_job_id: Mapped[UuidCol | None]
|
||||
anchor_sentence_key: Mapped[str | None]
|
||||
sort_key: Mapped[str | None]
|
||||
deleted_at: Mapped[Ts | None]
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class ChecklistItemCitation(Base):
|
||||
__tablename__ = "checklist_item_citation"
|
||||
tenant_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
item_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
citation_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
position: Mapped[int]
|
||||
258
server/src/bid_doctor/checklist/service.py
Normal file
258
server/src/bid_doctor/checklist/service.py
Normal file
@@ -0,0 +1,258 @@
|
||||
"""项目检查清单(FR-M2-05~08,架构 §3.8)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.checklist.extract import Extracted, MarkingNote
|
||||
from bid_doctor.checklist.models import Checklist, ChecklistItem, ChecklistItemCitation, MarkingRule
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.document import citations
|
||||
from bid_doctor.document.models import Citation, SourceDocument
|
||||
|
||||
BASIS_TO_IMPORTANCE = {
|
||||
"star": "marker",
|
||||
"asterisk": "marker",
|
||||
"italic_underline": "marker",
|
||||
"explicit_invalid": "tender_text",
|
||||
}
|
||||
|
||||
|
||||
async def get_or_create(s: AsyncSession, ctx: TenantContext, project_id: UUID) -> Checklist:
|
||||
cl = (await s.execute(select(Checklist).where(Checklist.project_id == project_id))).scalar_one_or_none()
|
||||
if cl is None:
|
||||
cl = Checklist(id=uuid7(), tenant_id=ctx.tenant_id, project_id=project_id)
|
||||
s.add(cl)
|
||||
await s.flush()
|
||||
return cl
|
||||
|
||||
|
||||
async def merge(
|
||||
s: AsyncSession,
|
||||
ctx: TenantContext,
|
||||
doc: SourceDocument,
|
||||
job_id: UUID,
|
||||
extracted: list[Extracted],
|
||||
notes: list[MarkingNote],
|
||||
) -> dict[str, int]:
|
||||
"""并入清单。重新解析时按首句编号对齐旧条目:对上的只更新 AI 字段,人工改过的不动;
|
||||
对不上的旧条目移到待确认区(本次解析未出现),不静默删除(架构 §3.8)。"""
|
||||
cl = await get_or_create(s, ctx, doc.project_id)
|
||||
old = {
|
||||
i.anchor_sentence_key: i
|
||||
for i in (
|
||||
await s.execute(
|
||||
select(ChecklistItem).where(
|
||||
ChecklistItem.origin_document_id == doc.id,
|
||||
ChecklistItem.origin != "human",
|
||||
ChecklistItem.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
rules: dict[str, UUID] = {}
|
||||
for n in notes:
|
||||
c = await citations.from_sentences(s, ctx, doc, [n.sentence], "rule", ["id", "quote"])
|
||||
r = MarkingRule(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
project_id=doc.project_id,
|
||||
document_id=doc.id,
|
||||
marker=n.marker,
|
||||
meaning="substantive",
|
||||
declared_count=n.declared_count,
|
||||
citation_id=c.id,
|
||||
origin="rule",
|
||||
)
|
||||
s.add(r)
|
||||
rules.setdefault(n.marker, r.id)
|
||||
await s.flush() # 模型不声明外键:先写标注规则,清单项才能引用它
|
||||
stats = {"items_added": 0, "items_updated": 0, "items_pending": 0, "items_absent": 0}
|
||||
seen: set[str] = set()
|
||||
for e in extracted:
|
||||
anchor = e.sentences[0].key if e.sentences else None
|
||||
if anchor and anchor in seen:
|
||||
continue # 同一句被模型重复抽成两条:只留第一条
|
||||
if anchor:
|
||||
seen.add(anchor)
|
||||
values: dict[str, Any] = {
|
||||
"title": e.summary,
|
||||
"summary": e.summary,
|
||||
"requirement_text": "".join(x.text for x in e.sentences) or None,
|
||||
"categories": ["invalidity"],
|
||||
"importance": "substantive",
|
||||
}
|
||||
item = old.pop(anchor, None) if anchor else None
|
||||
if item is None:
|
||||
item = ChecklistItem(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
project_id=doc.project_id,
|
||||
checklist_id=cl.id,
|
||||
origin="ai" if e.pending_reason != "marker_uncovered" else "rule",
|
||||
origin_document_id=doc.id,
|
||||
edited_fields=[],
|
||||
**values,
|
||||
)
|
||||
s.add(item)
|
||||
stats["items_added"] += 1
|
||||
else:
|
||||
for k, v in values.items():
|
||||
if k not in item.edited_fields:
|
||||
setattr(item, k, v)
|
||||
stats["items_updated"] += 1
|
||||
item.importance_basis = BASIS_TO_IMPORTANCE.get(e.basis)
|
||||
item.marking_rule_id = rules.get(e.basis)
|
||||
item.zone, item.pending_reason = e.zone, e.pending_reason
|
||||
item.ai_values, item.last_parse_job_id, item.anchor_sentence_key = e.ai_values, job_id, anchor
|
||||
item.sort_key = f"{e.sentences[0].page_no:05d}.{e.sentences[0].seq:05d}" if e.sentences else "99999"
|
||||
stats["items_pending"] += e.zone == "pending"
|
||||
await s.flush()
|
||||
await s.execute(delete(ChecklistItemCitation).where(ChecklistItemCitation.item_id == item.id))
|
||||
if e.sentences:
|
||||
c = await citations.from_sentences(
|
||||
s, ctx, doc, e.sentences, "ai" if item.origin == "ai" else "rule", e.checks_passed
|
||||
)
|
||||
s.add(ChecklistItemCitation(tenant_id=ctx.tenant_id, item_id=item.id, citation_id=c.id, position=0))
|
||||
for item in old.values():
|
||||
item.zone, item.pending_reason = "pending", "absent_in_rerun"
|
||||
stats["items_absent"] += 1
|
||||
if cl.status == "confirmed":
|
||||
cl.status = "draft" # 清单有变化:回到草稿,历史版本保留(FR-M2-08)
|
||||
await s.flush()
|
||||
return stats
|
||||
|
||||
|
||||
async def overview(s: AsyncSession, project_id: UUID) -> dict[str, Any]:
|
||||
cl = (await s.execute(select(Checklist).where(Checklist.project_id == project_id))).scalar_one_or_none()
|
||||
counts = dict(
|
||||
(
|
||||
await s.execute(
|
||||
select(ChecklistItem.zone, func.count())
|
||||
.where(ChecklistItem.project_id == project_id, ChecklistItem.deleted_at.is_(None))
|
||||
.group_by(ChecklistItem.zone)
|
||||
)
|
||||
)
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
rules = (
|
||||
await s.execute(
|
||||
select(MarkingRule, Citation, SourceDocument.role, SourceDocument.seq_no)
|
||||
.join(Citation, Citation.id == MarkingRule.citation_id)
|
||||
.join(SourceDocument, SourceDocument.id == MarkingRule.document_id)
|
||||
.where(MarkingRule.project_id == project_id)
|
||||
)
|
||||
).all()
|
||||
found = counts.get("formal", 0) + counts.get("pending", 0)
|
||||
warnings = [
|
||||
{
|
||||
"code": "declared_count_mismatch",
|
||||
"message": f"文件写明 ★ 项共 {r.declared_count} 条,实际抽到 {found} 条,请核对",
|
||||
}
|
||||
for r, *_ in rules
|
||||
if r.declared_count is not None and r.declared_count != found
|
||||
]
|
||||
return {
|
||||
"status": cl.status if cl else "draft",
|
||||
"version_no": cl.last_version_no if cl else 0,
|
||||
"confirmed_by": None,
|
||||
"confirmed_at": None,
|
||||
"counts": {"formal": counts.get("formal", 0), "pending": counts.get("pending", 0), "deleted": 0},
|
||||
"marking_rules": [
|
||||
{
|
||||
"id": r.id,
|
||||
"marker": r.marker,
|
||||
"marker_text": r.marker_text,
|
||||
"meaning": r.meaning,
|
||||
"declared_count": r.declared_count,
|
||||
"found_count": found,
|
||||
"citation": citations.to_api(c, citations.document_label(role, seq)),
|
||||
}
|
||||
for r, c, role, seq in rules
|
||||
],
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
async def items(s: AsyncSession, project_id: UUID, zone: str | None) -> list[dict[str, Any]]:
|
||||
q = select(ChecklistItem).where(ChecklistItem.project_id == project_id, ChecklistItem.deleted_at.is_(None))
|
||||
if zone:
|
||||
q = q.where(ChecklistItem.zone == zone)
|
||||
rows = (await s.execute(q.order_by(ChecklistItem.sort_key, ChecklistItem.id))).scalars().all()
|
||||
links = (
|
||||
await s.execute(
|
||||
select(ChecklistItemCitation.item_id, Citation, SourceDocument.role, SourceDocument.seq_no)
|
||||
.join(Citation, Citation.id == ChecklistItemCitation.citation_id)
|
||||
.join(SourceDocument, SourceDocument.id == Citation.document_id)
|
||||
.where(ChecklistItemCitation.item_id.in_([r.id for r in rows]))
|
||||
.order_by(ChecklistItemCitation.position)
|
||||
)
|
||||
).all()
|
||||
by_item: dict[UUID, list[dict[str, Any]]] = {}
|
||||
for item_id, c, role, seq in links:
|
||||
by_item.setdefault(item_id, []).append(citations.to_api(c, citations.document_label(role, seq)))
|
||||
return [_item(r, by_item.get(r.id, [])) for r in rows]
|
||||
|
||||
|
||||
def _item(r: ChecklistItem, cites: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": r.id,
|
||||
"categories": r.categories,
|
||||
"title": r.title,
|
||||
"requirement_text": r.requirement_text,
|
||||
"summary": r.summary,
|
||||
"importance": r.importance,
|
||||
"importance_basis": r.importance_basis,
|
||||
"marking_rule_id": r.marking_rule_id,
|
||||
"lot_id": r.lot_id,
|
||||
"lot_no": None,
|
||||
"evidence_required": r.evidence_required,
|
||||
"spec": r.spec,
|
||||
"origin": r.origin,
|
||||
"origin_label": {"ai": "AI", "rule": "规则", "human": "人工添加"}[r.origin],
|
||||
"zone": r.zone,
|
||||
"pending_reason": r.pending_reason,
|
||||
"no_citation_ack": False,
|
||||
"citations": cites,
|
||||
"ai_values": r.ai_values,
|
||||
"edited_fields": r.edited_fields,
|
||||
"confidence": float(r.confidence) if r.confidence is not None else None,
|
||||
"superseded_by": None,
|
||||
"accepted_by": None,
|
||||
"deleted_at": None,
|
||||
"updated_at": r.updated_at,
|
||||
}
|
||||
|
||||
|
||||
async def statuses(s: AsyncSession, project_ids: list[UUID]) -> dict[UUID, str]:
|
||||
"""各项目清单的状态(草稿 / 已确认);还没有清单的项目不在结果里。"""
|
||||
if not project_ids:
|
||||
return {}
|
||||
rows = await s.execute(select(Checklist.project_id, Checklist.status).where(Checklist.project_id.in_(project_ids)))
|
||||
return dict(rows.tuples().all())
|
||||
|
||||
|
||||
async def pending_counts(s: AsyncSession, project_ids: list[UUID]) -> dict[UUID, int]:
|
||||
if not project_ids:
|
||||
return {}
|
||||
return dict(
|
||||
(
|
||||
await s.execute(
|
||||
select(ChecklistItem.project_id, func.count())
|
||||
.where(
|
||||
ChecklistItem.project_id.in_(project_ids),
|
||||
ChecklistItem.zone == "pending",
|
||||
ChecklistItem.deleted_at.is_(None),
|
||||
)
|
||||
.group_by(ChecklistItem.project_id)
|
||||
)
|
||||
)
|
||||
.tuples()
|
||||
.all()
|
||||
)
|
||||
45
server/src/bid_doctor/cli.py
Normal file
45
server/src/bid_doctor/cli.py
Normal file
@@ -0,0 +1,45 @@
|
||||
"""命令行入口:bid-doctor api | worker | migrate。同一套代码、两种进程(T2)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _alembic_ini() -> Path:
|
||||
"""开发时在 server/ 下运行;镜像里工作目录是 /app/server。也可以用 BD_ALEMBIC_INI 指定。"""
|
||||
import os
|
||||
|
||||
if env := os.environ.get("BD_ALEMBIC_INI"):
|
||||
return Path(env)
|
||||
for base in (Path.cwd(), Path(__file__).resolve().parents[2]):
|
||||
if (base / "alembic.ini").exists():
|
||||
return base / "alembic.ini"
|
||||
raise SystemExit("找不到 alembic.ini:请在 server/ 目录下运行,或设置 BD_ALEMBIC_INI")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(prog="bid-doctor")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
api = sub.add_parser("api", help="启动接口服务")
|
||||
api.add_argument("--host", default="127.0.0.1")
|
||||
api.add_argument("--port", type=int, default=8009)
|
||||
api.add_argument("--reload", action="store_true")
|
||||
sub.add_parser("worker", help="启动后台任务进程")
|
||||
sub.add_parser("migrate", help="把数据库升级到最新版本")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.cmd == "api":
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run("bid_doctor.app:create_app", factory=True, host=args.host, port=args.port, reload=args.reload)
|
||||
elif args.cmd == "worker":
|
||||
from bid_doctor.worker import run_worker
|
||||
|
||||
asyncio.run(run_worker())
|
||||
elif args.cmd == "migrate":
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
command.upgrade(Config(str(_alembic_ini())), "head")
|
||||
0
server/src/bid_doctor/core/__init__.py
Normal file
0
server/src/bid_doctor/core/__init__.py
Normal file
37
server/src/bid_doctor/core/authz.py
Normal file
37
server/src/bid_doctor/core/authz.py
Normal file
@@ -0,0 +1,37 @@
|
||||
"""权限检查点(AR12)。MVP 只有所有者,拥有全部权限点,检查总是通过;
|
||||
以后加角色时只改这里的对应关系,接口代码不用改。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
from bid_doctor.core.db import TenantContext
|
||||
|
||||
|
||||
class Permission(StrEnum):
|
||||
PROJECT_CREATE = "project.create"
|
||||
PROJECT_EDIT = "project.edit"
|
||||
PROJECT_DELETE = "project.delete"
|
||||
CHECKLIST_EDIT = "checklist.edit"
|
||||
CHECKLIST_CONFIRM = "checklist.confirm"
|
||||
CHECK_RUN = "check.run"
|
||||
FINDING_RESOLVE = "finding.resolve"
|
||||
REPORT_EXPORT = "report.export"
|
||||
QUOTA_VIEW = "quota.view"
|
||||
BILLING_MANAGE = "billing.manage"
|
||||
QUALIFICATION_EDIT = "qualification.edit"
|
||||
TENANT_CLOSE = "tenant.close"
|
||||
|
||||
|
||||
ROLE_PERMISSIONS: dict[str, frozenset[Permission]] = {
|
||||
"owner": frozenset(Permission),
|
||||
}
|
||||
|
||||
|
||||
def permissions_for(role: str) -> frozenset[Permission]:
|
||||
return ROLE_PERMISSIONS.get(role, frozenset())
|
||||
|
||||
|
||||
def require(ctx: TenantContext, permission: Permission, obj: object = None) -> None:
|
||||
"""service 层统一调用。MVP 里 ctx 都是所有者,直接通过。"""
|
||||
del ctx, permission, obj
|
||||
23
server/src/bid_doctor/core/columns.py
Normal file
23
server/src/bid_doctor/core/columns.py
Normal file
@@ -0,0 +1,23 @@
|
||||
"""ORM 列类型的简写。表结构以迁移脚本为准(不用自动建表),这里只做映射。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Annotated, Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, FetchedValue, LargeBinary, Text, Uuid
|
||||
from sqlalchemy.dialects.postgresql import ARRAY, INET, JSONB
|
||||
from sqlalchemy.orm import mapped_column
|
||||
|
||||
PK = Annotated[UUID, mapped_column(Uuid, primary_key=True)]
|
||||
UuidCol = Annotated[UUID, mapped_column(Uuid)]
|
||||
Ts = Annotated[datetime, mapped_column(DateTime(timezone=True))]
|
||||
# 由数据库默认值或触发器填写的时间
|
||||
DbTs = Annotated[
|
||||
datetime, mapped_column(DateTime(timezone=True), server_default=FetchedValue(), server_onupdate=FetchedValue())
|
||||
]
|
||||
Bytes = Annotated[bytes, mapped_column(LargeBinary)]
|
||||
Json = Annotated[Any, mapped_column(JSONB)]
|
||||
TextArray = Annotated[list[str], mapped_column(ARRAY(Text), server_default=FetchedValue())]
|
||||
Inet = Annotated[str, mapped_column(INET)]
|
||||
73
server/src/bid_doctor/core/config.py
Normal file
73
server/src/bid_doctor/core/config.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""配置:只从环境变量(前缀 BD_)和 .env 读取。密钥不进代码和镜像(架构 §3.11)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
DEV_SECRET = "dev-insecure-secret-change-me-0000000000000000" # noqa: S105 - 只用于 dev,生产环境检测到会拒绝启动
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="BD_", env_file=".env", extra="ignore")
|
||||
|
||||
env: Literal["dev", "test", "prod"] = "dev"
|
||||
|
||||
# api / worker 用 bd_app(受行级安全策略约束);迁移用 bd_owner
|
||||
database_url: str = "postgresql+psycopg://bd_app:dev-app@127.0.0.1:54329/bid_doctor"
|
||||
migrate_database_url: str = "postgresql+psycopg://bd_owner:dev-owner@127.0.0.1:54329/bid_doctor"
|
||||
valkey_url: str = "redis://127.0.0.1:63799/0"
|
||||
gotenberg_url: str = "http://127.0.0.1:3109"
|
||||
|
||||
storage_dir: Path = Path(".data/objects")
|
||||
public_base_url: str = "http://127.0.0.1:8009"
|
||||
cors_origins: list[str] = Field(default_factory=lambda: ["http://127.0.0.1:5173", "http://localhost:5173"])
|
||||
|
||||
# 32 字节以上的随机串;分别用于:签名下载地址、手机号 HMAC、手机号加密、包装数据密钥(本地 KMS 替身)
|
||||
secret_key: str = DEV_SECRET
|
||||
phone_pepper: str = DEV_SECRET
|
||||
master_key: str = DEV_SECRET
|
||||
|
||||
session_ttl_hours: int = 24 * 14
|
||||
cookie_secure: bool = False
|
||||
|
||||
# 短信:dev / test 用固定验证码并打日志;prod 必须接真实通道(还没接)
|
||||
sms_provider: Literal["dev", "aliyun"] = "dev"
|
||||
sms_dev_code: str = "000000"
|
||||
|
||||
# 模型路由(T1):业务只声明能力,这里决定供应商。dev 默认用确定性的假模型
|
||||
ai_provider: Literal["fake", "openai_compat"] = "fake"
|
||||
ai_base_url: str = ""
|
||||
ai_api_key: str = ""
|
||||
ai_model: str = ""
|
||||
|
||||
upload_part_size: int = 8 * 1024 * 1024
|
||||
download_url_ttl_s: int = 300
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _refuse_insecure_prod(self) -> Settings:
|
||||
"""生产环境检测到默认密钥或调试开关,拒绝启动(架构 §3.11)。"""
|
||||
if self.env == "prod":
|
||||
weak = [
|
||||
n
|
||||
for n in ("secret_key", "phone_pepper", "master_key")
|
||||
if getattr(self, n) == DEV_SECRET or len(getattr(self, n)) < 32
|
||||
]
|
||||
if weak:
|
||||
raise ValueError(f"生产环境的密钥不安全:{', '.join(weak)}")
|
||||
if self.sms_provider == "dev":
|
||||
raise ValueError("生产环境不能使用 dev 短信通道")
|
||||
if self.ai_provider == "fake":
|
||||
raise ValueError("生产环境不能使用假模型")
|
||||
if not self.cookie_secure:
|
||||
raise ValueError("生产环境必须开启 cookie_secure")
|
||||
return self
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
94
server/src/bid_doctor/core/crypto.py
Normal file
94
server/src/bid_doctor/core/crypto.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""加密工具:会话令牌、手机号 HMAC(DM5)、信封加密(DM4)、签名下载地址。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
"""256 位随机令牌,只发给客户端一次;数据库只存它的 SHA-256(AR8)。"""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def sha256(data: bytes | str) -> bytes:
|
||||
return hashlib.sha256(data.encode() if isinstance(data, str) else data).digest()
|
||||
|
||||
|
||||
def _key(material: str, purpose: str) -> bytes:
|
||||
return hashlib.sha256(f"{purpose}:{material}".encode()).digest()
|
||||
|
||||
|
||||
def phone_lookup_key(phone: str) -> str:
|
||||
"""手机号的查找键:HMAC,不可逆;数据库里不存明文号码。"""
|
||||
return hmac.new(_key(get_settings().phone_pepper, "phone"), phone.encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def phone_hint(phone: str) -> str:
|
||||
return f"{phone[:3]}****{phone[-4:]}"
|
||||
|
||||
|
||||
def _seal(key: bytes, plaintext: bytes, aad: bytes = b"") -> bytes:
|
||||
nonce = os.urandom(12)
|
||||
return nonce + AESGCM(key).encrypt(nonce, plaintext, aad)
|
||||
|
||||
|
||||
def _open(key: bytes, sealed: bytes, aad: bytes = b"") -> bytes:
|
||||
return AESGCM(key).decrypt(sealed[:12], sealed[12:], aad)
|
||||
|
||||
|
||||
def encrypt_contact(value: str) -> bytes:
|
||||
return _seal(_key(get_settings().phone_pepper, "contact"), value.encode())
|
||||
|
||||
|
||||
def decrypt_contact(sealed: bytes) -> str:
|
||||
return _open(_key(get_settings().phone_pepper, "contact"), sealed).decode()
|
||||
|
||||
|
||||
# ---- 信封加密:每个项目一把数据密钥,用主密钥包装后存库;删除时销毁(DM4)
|
||||
# 主密钥现在来自环境变量,是 KMS 的本地替身;上云后换成云厂商 KMS,接口不变。
|
||||
|
||||
KEK_REF = "env:v1"
|
||||
|
||||
|
||||
def new_data_key() -> tuple[bytes, bytes]:
|
||||
"""返回 (明文数据密钥, 包装后的数据密钥)。"""
|
||||
dk = AESGCM.generate_key(bit_length=256)
|
||||
return dk, _seal(_key(get_settings().master_key, "kek"), dk)
|
||||
|
||||
|
||||
def unwrap_data_key(wrapped: bytes) -> bytes:
|
||||
return _open(_key(get_settings().master_key, "kek"), wrapped)
|
||||
|
||||
|
||||
def seal_object(data_key: bytes, plaintext: bytes, object_key: str) -> bytes:
|
||||
# 对象键作为附加数据:密文被挪到别的键下就解不开
|
||||
return _seal(data_key, plaintext, object_key.encode())
|
||||
|
||||
|
||||
def open_object(data_key: bytes, sealed: bytes, object_key: str) -> bytes:
|
||||
return _open(data_key, sealed, object_key.encode())
|
||||
|
||||
|
||||
# ---- 签名下载地址:pdf.js 直接加载,不带鉴权头(api-design §4.4)
|
||||
|
||||
|
||||
def sign_download(payload: str, ttl_s: int) -> tuple[str, int]:
|
||||
exp = int(time.time()) + ttl_s
|
||||
mac = hmac.new(_key(get_settings().secret_key, "download"), f"{payload}|{exp}".encode(), hashlib.sha256).digest()
|
||||
return base64.urlsafe_b64encode(mac).decode().rstrip("="), exp
|
||||
|
||||
|
||||
def verify_download(payload: str, exp: int, sig: str) -> bool:
|
||||
if exp < time.time():
|
||||
return False
|
||||
mac = hmac.new(_key(get_settings().secret_key, "download"), f"{payload}|{exp}".encode(), hashlib.sha256).digest()
|
||||
return hmac.compare_digest(base64.urlsafe_b64encode(mac).decode().rstrip("="), sig)
|
||||
68
server/src/bid_doctor/core/db.py
Normal file
68
server/src/bid_doctor/core/db.py
Normal file
@@ -0,0 +1,68 @@
|
||||
"""数据库:所有业务访问都在「带租户的事务」里进行(架构 §3.3)。
|
||||
|
||||
事务开始时 set_config('app.tenant_id', …, true),行级安全策略按它过滤;没设租户时什么都查不到。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TenantContext:
|
||||
"""数据访问层的第一个参数。没有「传空就查全部」的写法。"""
|
||||
|
||||
tenant_id: UUID
|
||||
user_id: UUID
|
||||
membership_id: UUID | None = None
|
||||
request_id: str = ""
|
||||
actor_name: str = ""
|
||||
|
||||
|
||||
@lru_cache
|
||||
def engine() -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
get_settings().database_url,
|
||||
pool_size=10,
|
||||
pool_pre_ping=True,
|
||||
connect_args={"options": "-c search_path=app,public"},
|
||||
)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _sessions() -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(engine(), expire_on_commit=False)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def tenant_tx(ctx: TenantContext) -> AsyncGenerator[AsyncSession]:
|
||||
"""一个带租户的事务:正常结束提交,出错回滚。"""
|
||||
async with _sessions()() as session, session.begin():
|
||||
await session.execute(text("SELECT set_config('app.tenant_id', :t, true)"), {"t": str(ctx.tenant_id)})
|
||||
yield session
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def anonymous_tx() -> AsyncGenerator[AsyncSession]:
|
||||
"""不带租户的事务:只用于登录、注册这类还不知道租户的操作(identity 模块专用)。"""
|
||||
async with _sessions()() as session, session.begin():
|
||||
yield session
|
||||
|
||||
|
||||
async def set_tenant(session: AsyncSession, tenant_id: UUID) -> None:
|
||||
"""注册时先建租户再写它的数据:在同一个事务里切换到新租户。"""
|
||||
await session.execute(text("SELECT set_config('app.tenant_id', :t, true)"), {"t": str(tenant_id)})
|
||||
91
server/src/bid_doctor/core/errors.py
Normal file
91
server/src/bid_doctor/core/errors.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""错误:统一用 RFC 9457 problem+json,带稳定的 code(api-design §5)。错误码只在这里定义。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
# code → (HTTP 状态, 给用户看的标题)
|
||||
CODES: dict[str, tuple[int, str]] = {
|
||||
"unauthenticated": (401, "请先登录"),
|
||||
"session_expired": (401, "登录已过期,请重新登录"),
|
||||
"session_revoked": (401, "登录已失效,请重新登录"),
|
||||
"csrf_failed": (403, "请求校验失败,请刷新页面后重试"),
|
||||
"forbidden": (403, "没有权限进行这个操作"),
|
||||
"not_found": (404, "没有找到"),
|
||||
"phone_not_registered": (404, "这个手机号还没有注册"),
|
||||
"phone_already_registered": (409, "这个手机号已经注册"),
|
||||
"concurrent_session": (409, "账号已在其他设备登录"),
|
||||
"quota_exhausted": (409, "额度不足"),
|
||||
"billing_changed": (409, "扣点情况有变化,请确认后再试"),
|
||||
"plan_limit_exceeded": (409, "超出套餐限制"),
|
||||
"job_already_running": (409, "任务正在进行"),
|
||||
"job_finished": (409, "任务已经结束"),
|
||||
"project_read_only": (409, "项目已递交或已归档,要先撤销才能修改"),
|
||||
"document_role_not_parsable": (409, "这类文件不需要单独解析"),
|
||||
"document_not_ready": (409, "文件还在处理"),
|
||||
"upload_incomplete": (409, "文件还没有上传完"),
|
||||
"precondition_failed": (412, "内容已被修改,请刷新后再改"),
|
||||
"validation_failed": (422, "填写的内容有误"),
|
||||
"sms_code_invalid": (422, "验证码不正确或已过期"),
|
||||
"file_format_unsupported": (422, "不支持这种文件格式"),
|
||||
"file_encrypted": (422, "文件已加密,请上传未加密的版本"),
|
||||
"file_corrupted": (422, "文件已损坏,请换一个文件"),
|
||||
"bid_file_platform_encrypted": (422, "这是电子投标平台加密后的文件,请上传加密前的 Word 或 PDF"),
|
||||
"upload_checksum_mismatch": (422, "文件校验不一致,请重新上传"),
|
||||
"ocr_required": (422, "这份文件大部分页面没有文字(可能是扫描件),扫描件识别还没有上线"),
|
||||
"rate_limited": (429, "操作太频繁,请稍后再试"),
|
||||
"internal_error": (500, "系统出错了,请稍后再试"),
|
||||
}
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
def __init__(self, code: str, detail: str | None = None, **extra: Any) -> None:
|
||||
if code not in CODES:
|
||||
raise ValueError(f"未定义的错误码:{code}")
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.status, self.title = CODES[code]
|
||||
self.detail = detail
|
||||
self.extra = extra
|
||||
|
||||
|
||||
def problem(request: Request, code: str, detail: str | None = None, **extra: Any) -> JSONResponse:
|
||||
status, title = CODES[code]
|
||||
body: dict[str, Any] = {
|
||||
"type": "about:blank",
|
||||
"title": title,
|
||||
"status": status,
|
||||
"code": code,
|
||||
"request_id": getattr(request.state, "request_id", ""),
|
||||
**({"detail": detail} if detail else {}),
|
||||
**extra,
|
||||
}
|
||||
headers = {"Retry-After": str(extra["retry_after_s"])} if "retry_after_s" in extra else None
|
||||
return JSONResponse(body, status_code=status, media_type="application/problem+json", headers=headers)
|
||||
|
||||
|
||||
def install(app: FastAPI) -> None:
|
||||
@app.exception_handler(AppError)
|
||||
async def _app_error(request: Request, exc: AppError) -> JSONResponse: # pyright: ignore[reportUnusedFunction]
|
||||
return problem(request, exc.code, exc.detail, **exc.extra)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation(request: Request, exc: RequestValidationError) -> JSONResponse: # pyright: ignore[reportUnusedFunction]
|
||||
errors = [
|
||||
{
|
||||
"field": ".".join(str(p) for p in e["loc"][1:]) or str(e["loc"][0]),
|
||||
"code": e["type"],
|
||||
"message": e["msg"],
|
||||
}
|
||||
for e in exc.errors()
|
||||
]
|
||||
return problem(request, "validation_failed", errors=errors)
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def _http(request: Request, exc: StarletteHTTPException) -> JSONResponse: # pyright: ignore[reportUnusedFunction]
|
||||
return problem(request, "not_found" if exc.status_code == 404 else "internal_error")
|
||||
72
server/src/bid_doctor/core/events.py
Normal file
72
server/src/bid_doctor/core/events.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""事件流(api-design §4.2):用 Valkey stream 存最近的事件,SSE 读它。
|
||||
|
||||
事件只是「有变化」的提示,权威状态以资源的 GET 为准;丢了也没关系,前端会轮询兜底。
|
||||
用 stream 而不是发布 / 订阅:断线重连时可以按 Last-Event-ID 补发,多个 api 实例也都能读到。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
from bid_doctor.core.valkey import stream_valkey, valkey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
MAXLEN = 500
|
||||
BUFFER_TTL_S = 600
|
||||
HEARTBEAT_S = 15
|
||||
|
||||
|
||||
def _key(tenant_id: UUID) -> str:
|
||||
return f"ev:{tenant_id}"
|
||||
|
||||
|
||||
async def publish(tenant_id: UUID, type_: str, **data: Any) -> None:
|
||||
"""发布失败只记日志,不影响业务事务。"""
|
||||
event = {"type": type_, "at": datetime.now(UTC).isoformat(), **data}
|
||||
try:
|
||||
r = valkey()
|
||||
await r.xadd(_key(tenant_id), {"e": json.dumps(event, default=str)}, maxlen=MAXLEN, approximate=True)
|
||||
await r.expire(_key(tenant_id), BUFFER_TTL_S)
|
||||
except Exception:
|
||||
log.warning("event publish failed", exc_info=True)
|
||||
|
||||
|
||||
async def stream(tenant_id: UUID, last_id: str | None) -> AsyncIterator[str]:
|
||||
"""产出 SSE 文本块。last_id 已经过期时先发一条 resync,让前端重新拉数据。"""
|
||||
r = stream_valkey()
|
||||
key = _key(tenant_id)
|
||||
cursor = last_id or "$"
|
||||
if last_id:
|
||||
first = cast(list[tuple[str, dict[str, str]]], await r.xrange(key, count=1))
|
||||
if not first or _sid(first[0][0]) > _sid(last_id):
|
||||
yield _sse("0-0", {"type": "resync", "at": datetime.now(UTC).isoformat()})
|
||||
cursor = "$"
|
||||
while True:
|
||||
raw = await r.xread({key: cursor}, block=HEARTBEAT_S * 1000, count=50)
|
||||
batch = cast(list[tuple[str, list[tuple[str, dict[str, str]]]]], raw)
|
||||
if not batch:
|
||||
yield ": heartbeat\n\n"
|
||||
continue
|
||||
for _, entries in batch:
|
||||
for entry_id, fields in entries:
|
||||
cursor = entry_id
|
||||
event = json.loads(fields["e"])
|
||||
yield _sse(entry_id, event)
|
||||
|
||||
|
||||
def _sid(entry_id: str) -> tuple[int, int]:
|
||||
"""stream 编号是「毫秒-序号」,要按数字比较。格式不对的当作最旧。"""
|
||||
ms, _, seq = entry_id.partition("-")
|
||||
try:
|
||||
return int(ms), int(seq or 0)
|
||||
except ValueError:
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _sse(entry_id: str, event: dict[str, Any]) -> str:
|
||||
return f"id: {entry_id}\nevent: {event['type']}\ndata: {json.dumps(event, ensure_ascii=False)}\n\n"
|
||||
16
server/src/bid_doctor/core/ids.py
Normal file
16
server/src/bid_doctor/core/ids.py
Normal file
@@ -0,0 +1,16 @@
|
||||
"""UUIDv7:按时间有序、不可猜测(架构 §9)。Python 3.14 起标准库自带 uuid.uuid7,这里兼容 3.12。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
def uuid7() -> uuid.UUID:
|
||||
ms = time.time_ns() // 1_000_000
|
||||
rand = int.from_bytes(os.urandom(10), "big")
|
||||
value = (ms & 0xFFFF_FFFF_FFFF) << 80 | rand
|
||||
value = (value & ~(0xF << 76)) | (0x7 << 76) # 版本 7
|
||||
value = (value & ~(0x3 << 62)) | (0x2 << 62) # RFC 4122 变体
|
||||
return uuid.UUID(int=value)
|
||||
47
server/src/bid_doctor/core/keys.py
Normal file
47
server/src/bid_doctor/core/keys.py
Normal file
@@ -0,0 +1,47 @@
|
||||
"""数据密钥(DM4):每个项目一把,资质附件和上传暂存用企业级的一把;删除时销毁。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.core.crypto import KEK_REF, new_data_key, unwrap_data_key
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.ids import uuid7
|
||||
|
||||
|
||||
async def data_key(session: AsyncSession, ctx: TenantContext, project_id: UUID | None = None) -> bytes:
|
||||
"""取明文数据密钥;还没有就建一把。project_id 为空表示企业级密钥。"""
|
||||
scope = "project" if project_id else "tenant"
|
||||
row = (
|
||||
await session.execute(
|
||||
text(
|
||||
"SELECT wrapped_key FROM data_key WHERE tenant_id = :t AND scope = :s "
|
||||
"AND project_id IS NOT DISTINCT FROM :p AND destroyed_at IS NULL"
|
||||
),
|
||||
{"t": ctx.tenant_id, "s": scope, "p": project_id},
|
||||
)
|
||||
).first()
|
||||
if row:
|
||||
return unwrap_data_key(row[0])
|
||||
plain, wrapped = new_data_key()
|
||||
await session.execute(
|
||||
text(
|
||||
"INSERT INTO data_key (id, tenant_id, scope, project_id, wrapped_key, kek_ref) "
|
||||
"VALUES (:id, :t, :s, :p, :w, :k)"
|
||||
),
|
||||
{"id": uuid7(), "t": ctx.tenant_id, "s": scope, "p": project_id, "w": wrapped, "k": KEK_REF},
|
||||
)
|
||||
return plain
|
||||
|
||||
|
||||
async def destroy_project_key(session: AsyncSession, ctx: TenantContext, project_id: UUID) -> None:
|
||||
await session.execute(
|
||||
text(
|
||||
"UPDATE data_key SET wrapped_key = NULL, destroyed_at = now() "
|
||||
"WHERE tenant_id = :t AND scope = 'project' AND project_id = :p AND destroyed_at IS NULL"
|
||||
),
|
||||
{"t": ctx.tenant_id, "p": project_id},
|
||||
)
|
||||
25
server/src/bid_doctor/core/queue.py
Normal file
25
server/src/bid_doctor/core/queue.py
Normal file
@@ -0,0 +1,25 @@
|
||||
"""在业务事务里入队(架构 §3.4):「预扣额度 → 建任务 → 入队」一起提交或一起回滚。
|
||||
|
||||
直接调用 Procrastinate 的 SQL 函数 procrastinate_defer_jobs_v1(它的 defer 接口用的是自己的连接,进不了我们的事务)。
|
||||
这依赖 Procrastinate 的 SQL 接口,所以 pyproject 里锁定了大版本,升级时要核对 queries.sql 的 defer_jobs。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
DEFER = text(
|
||||
"SELECT unnest(procrastinate_defer_jobs_v1(ARRAY[ROW(:queue, :task, 0, :lock, NULL, CAST(:args AS jsonb), NULL)"
|
||||
"::procrastinate_job_to_defer_v1])) AS id"
|
||||
)
|
||||
|
||||
|
||||
async def enqueue(
|
||||
s: AsyncSession, task: str, args: dict[str, Any], *, lock: str | None = None, queue: str = "default"
|
||||
) -> int:
|
||||
row = (await s.execute(DEFER, {"queue": queue, "task": task, "lock": lock, "args": json.dumps(args)})).one()
|
||||
return int(row[0])
|
||||
67
server/src/bid_doctor/core/storage.py
Normal file
67
server/src/bid_doctor/core/storage.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""对象存储:S3 协议 + 本地磁盘驱动(AR5)。业务代码只用这个接口,不关心落在哪里。
|
||||
|
||||
所有对象都按所属项目(或企业)的数据密钥加密后再写入(DM4),布局见 data-model §7。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
class ObjectStorage(Protocol):
|
||||
def put(self, key: str, data: bytes) -> None: ...
|
||||
def get(self, key: str) -> bytes: ...
|
||||
def exists(self, key: str) -> bool: ...
|
||||
def delete_prefix(self, prefix: str) -> None: ...
|
||||
|
||||
|
||||
class LocalDiskStorage:
|
||||
"""单机部署和开发环境用。键里不允许 .. 和绝对路径。"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = root.resolve()
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
p = (self.root / key).resolve()
|
||||
if self.root not in p.parents:
|
||||
raise ValueError(f"非法的对象键:{key}")
|
||||
return p
|
||||
|
||||
def put(self, key: str, data: bytes) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = p.with_suffix(p.suffix + ".tmp")
|
||||
tmp.write_bytes(data)
|
||||
os.replace(tmp, p)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self._path(key).read_bytes()
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self._path(key).exists()
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
p = self._path(prefix)
|
||||
if p.is_dir():
|
||||
shutil.rmtree(p)
|
||||
elif p.exists():
|
||||
p.unlink()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def storage() -> ObjectStorage:
|
||||
return LocalDiskStorage(get_settings().storage_dir)
|
||||
|
||||
|
||||
def project_prefix(tenant_id: object, project_id: object) -> str:
|
||||
return f"t/{tenant_id}/p/{project_id}"
|
||||
|
||||
|
||||
def upload_prefix(tenant_id: object, upload_id: object) -> str:
|
||||
return f"t/{tenant_id}/uploads/{upload_id}"
|
||||
32
server/src/bid_doctor/core/valkey.py
Normal file
32
server/src/bid_doctor/core/valkey.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""Valkey:只存丢了也没关系的数据(AR4)——验证码、限流、事件缓冲。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
@lru_cache
|
||||
def valkey() -> Redis:
|
||||
return Redis.from_url(get_settings().valkey_url, decode_responses=True)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def stream_valkey() -> Redis:
|
||||
"""事件流的长阻塞读(XREAD BLOCK)用单独的连接池:读超时要比阻塞时间长,否则连接被当成超时断开。"""
|
||||
return Redis.from_url(get_settings().valkey_url, decode_responses=True, socket_timeout=60)
|
||||
|
||||
|
||||
async def hit_limit(key: str, limit: int, window_s: int) -> int | None:
|
||||
"""固定窗口限流。超限时返回还要等几秒,否则返回 None。"""
|
||||
r = valkey()
|
||||
n = await r.incr(key)
|
||||
if n == 1:
|
||||
await r.expire(key, window_s)
|
||||
if n > limit:
|
||||
ttl = await r.ttl(key)
|
||||
return max(ttl, 1)
|
||||
return None
|
||||
0
server/src/bid_doctor/document/__init__.py
Normal file
0
server/src/bid_doctor/document/__init__.py
Normal file
190
server/src/bid_doctor/document/api.py
Normal file
190
server/src/bid_doctor/document/api.py
Normal file
@@ -0,0 +1,190 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Annotated, Any, Literal
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Body, Query, Request
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import open_object, sign_download, verify_download
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document import service, uploads
|
||||
from bid_doctor.document.models import DocumentPage, SourceDocument
|
||||
from bid_doctor.identity.deps import Ctx
|
||||
|
||||
router = APIRouter(tags=["uploads", "documents"])
|
||||
|
||||
|
||||
class UploadIn(BaseModel):
|
||||
filename: str = Field(max_length=255)
|
||||
size_bytes: int = Field(ge=1)
|
||||
sha256: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$")
|
||||
purpose: Literal["project_document", "qualification_file"]
|
||||
|
||||
|
||||
class DocumentIn(BaseModel):
|
||||
upload_id: UUID
|
||||
role: Literal["tender", "addendum", "clarification", "bid_volume"]
|
||||
volume_name: str | None = Field(default=None, max_length=40)
|
||||
|
||||
|
||||
class ParseIn(BaseModel):
|
||||
expected_charge: int | None = None
|
||||
|
||||
|
||||
@router.post("/uploads", status_code=201)
|
||||
async def create_upload(body: UploadIn, ctx: Ctx) -> dict[str, Any]:
|
||||
return await uploads.create(ctx, body.filename, body.size_bytes, body.purpose, body.sha256)
|
||||
|
||||
|
||||
@router.get("/uploads/{upload_id}")
|
||||
async def get_upload(upload_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
return await uploads.describe(ctx, upload_id)
|
||||
|
||||
|
||||
@router.put("/uploads/{upload_id}/parts/{part_no}", status_code=204)
|
||||
async def put_part(upload_id: UUID, part_no: int, request: Request, ctx: Ctx) -> None:
|
||||
await uploads.put_part(ctx, upload_id, part_no, await request.body())
|
||||
|
||||
|
||||
@router.post("/uploads/{upload_id}/complete")
|
||||
async def complete_upload(upload_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
return await uploads.complete(ctx, upload_id)
|
||||
|
||||
|
||||
@router.delete("/uploads/{upload_id}", status_code=204)
|
||||
async def abort_upload(upload_id: UUID, ctx: Ctx) -> None:
|
||||
await uploads.discard(ctx, upload_id)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/documents")
|
||||
async def list_documents(project_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
docs = (
|
||||
(
|
||||
await s.execute(
|
||||
select(SourceDocument)
|
||||
.where(SourceDocument.project_id == project_id)
|
||||
.order_by(SourceDocument.created_at)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return {"items": [await service.doc_to_api(s, d) for d in docs]}
|
||||
|
||||
|
||||
@router.post("/projects/{project_id}/documents", status_code=201)
|
||||
async def create_document(project_id: UUID, body: DocumentIn, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = await service.create(s, ctx, project_id, body.upload_id, body.role, body.volume_name)
|
||||
return await service.doc_to_api(s, doc)
|
||||
|
||||
|
||||
@router.get("/documents/{document_id}")
|
||||
async def get_document(document_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await service.doc_to_api(s, await service.get(s, document_id))
|
||||
|
||||
|
||||
@router.get("/documents/{document_id}/pages")
|
||||
async def list_pages(document_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = await service.get(s, document_id)
|
||||
pages = (
|
||||
(
|
||||
await s.execute(
|
||||
select(DocumentPage).where(DocumentPage.document_id == doc.id).order_by(DocumentPage.page_no)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return {
|
||||
"normalized_sha256": doc.normalized_sha256.hex() if doc.normalized_sha256 else None,
|
||||
"items": [
|
||||
{
|
||||
"page_no": p.page_no,
|
||||
"width_pt": p.width_pt,
|
||||
"height_pt": p.height_pt,
|
||||
"rotation": p.rotation,
|
||||
"text_source": p.text_source,
|
||||
}
|
||||
for p in pages
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/documents/{document_id}/pages/{page_no}/sentences")
|
||||
async def list_sentences(document_id: UUID, page_no: int, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = await service.get(s, document_id)
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
if not doc.text_layer_key:
|
||||
raise AppError("document_not_ready")
|
||||
pages = json.loads(open_object(dk, storage().get(doc.text_layer_key), doc.text_layer_key))
|
||||
page = next((p for p in pages if p["page_no"] == page_no), None)
|
||||
if page is None:
|
||||
raise AppError("not_found")
|
||||
return {"items": page["sentences"]}
|
||||
|
||||
|
||||
@router.get("/documents/{document_id}/file")
|
||||
async def get_file_url(
|
||||
document_id: UUID, ctx: Ctx, variant: Literal["normalized", "original"] = Query()
|
||||
) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = await service.get(s, document_id)
|
||||
if variant == "normalized" and not doc.normalized_key:
|
||||
raise AppError("document_not_ready")
|
||||
payload = f"{ctx.tenant_id}:{doc.id}:{variant}"
|
||||
sig, exp = sign_download(payload, get_settings().download_url_ttl_s)
|
||||
from datetime import UTC, datetime
|
||||
|
||||
url = f"{get_settings().public_base_url}/api/v1/files/{ctx.tenant_id}/{doc.id}/{variant}?exp={exp}&sig={sig}"
|
||||
return {"url": url, "expires_at": datetime.fromtimestamp(exp, UTC)}
|
||||
|
||||
|
||||
@router.get("/files/{tenant_id}/{document_id}/{variant}", include_in_schema=False)
|
||||
async def download(
|
||||
tenant_id: UUID, document_id: UUID, variant: Literal["normalized", "original"], exp: int, sig: str
|
||||
) -> Response:
|
||||
"""签名下载(本地磁盘驱动用)。不带鉴权头,靠短时签名;S3 驱动时直接返回对象存储的预签名地址。"""
|
||||
if not verify_download(f"{tenant_id}:{document_id}:{variant}", exp, sig):
|
||||
raise AppError("not_found")
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
doc = await service.get(s, document_id)
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
key = doc.normalized_key if variant == "normalized" else doc.original_key
|
||||
assert key is not None
|
||||
media = "application/pdf" if variant == "normalized" else "application/octet-stream"
|
||||
return Response(
|
||||
open_object(dk, storage().get(key), key), media_type=media, headers={"Cache-Control": "private, max-age=300"}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/documents/{document_id}/parse-jobs", status_code=201)
|
||||
async def start_parse(document_id: UUID, ctx: Ctx, body: Annotated[ParseIn | None, Body()] = None) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = await service.start_parse(s, ctx, document_id, body.expected_charge if body else None)
|
||||
return service.job_to_api(job)
|
||||
|
||||
|
||||
@router.get("/parse-jobs/{job_id}")
|
||||
async def get_parse_job(job_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return service.job_to_api(await service.get_job(s, job_id))
|
||||
|
||||
|
||||
@router.post("/parse-jobs/{job_id}/cancel", status_code=202)
|
||||
async def cancel_parse_job(job_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return service.job_to_api(await service.request_cancel(s, job_id))
|
||||
59
server/src/bid_doctor/document/citations.py
Normal file
59
server/src/bid_doctor/document/citations.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""出处(data-model §5.1):不可变,清单项、问题、项目字段、标注规则都引用它。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.document.models import Citation, SourceDocument
|
||||
from bid_doctor.document.pipeline.text_layer import Sentence
|
||||
|
||||
LABELS = {"tender": "招标文件", "addendum": "补遗", "clarification": "澄清", "bid_volume": "投标文件"}
|
||||
|
||||
|
||||
def document_label(role: str, seq_no: int) -> str:
|
||||
return f"{LABELS[role]} {seq_no}" if role in ("addendum", "clarification") else LABELS.get(role, role)
|
||||
|
||||
|
||||
async def from_sentences(
|
||||
s: AsyncSession, ctx: TenantContext, doc: SourceDocument, sentences: list[Sentence], kind: str, checks: list[str]
|
||||
) -> Citation:
|
||||
"""一条出处只能在同一页上:跨页的要求取第一页的句子(跨页拆成多条出处,TODO)。"""
|
||||
assert doc.normalized_sha256 is not None
|
||||
page = sentences[0].page_no
|
||||
same = [x for x in sentences if x.page_no == page]
|
||||
c = Citation(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
project_id=doc.project_id,
|
||||
document_id=doc.id,
|
||||
normalized_sha256=doc.normalized_sha256,
|
||||
page_no=page,
|
||||
sentence_keys=[x.key for x in same],
|
||||
quads=[q for x in same for q in x.quads],
|
||||
quote="".join(x.text for x in same),
|
||||
created_by_kind=kind,
|
||||
checks_passed=checks,
|
||||
)
|
||||
s.add(c)
|
||||
await s.flush()
|
||||
return c
|
||||
|
||||
|
||||
def to_api(c: Citation, label: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": c.id,
|
||||
"document_id": c.document_id,
|
||||
"document_label": label,
|
||||
"normalized_sha256": c.normalized_sha256.hex(),
|
||||
"page_no": c.page_no,
|
||||
"sentence_keys": c.sentence_keys,
|
||||
"char_start": c.char_start,
|
||||
"char_end": c.char_end,
|
||||
"quote": c.quote,
|
||||
"quads": c.quads,
|
||||
"created_by_kind": c.created_by_kind,
|
||||
}
|
||||
64
server/src/bid_doctor/document/formats.py
Normal file
64
server/src/bid_doctor/document/formats.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""按文件头识别真实格式,不看扩展名(FR-M2-01、07 F2)。文件是不可信输入:只读结构,不执行任何内容。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
from bid_doctor.core.errors import AppError
|
||||
|
||||
OLE_MAGIC = bytes.fromhex("D0CF11E0A1B11AE1")
|
||||
EXTENSIONS = {"pdf": {"pdf"}, "docx": {"docx", "doc", "wps"}, "doc": {"doc", "wps"}, "ofd": {"ofd"}}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Detected:
|
||||
format: str # pdf / docx / doc / ofd
|
||||
page_count: int | None # 只有 PDF 能直接读出页数
|
||||
extension_mismatch: bool
|
||||
|
||||
|
||||
def detect(data: bytes, filename: str) -> Detected:
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
if data.startswith(b"%PDF"):
|
||||
fmt, pages = "pdf", _pdf_pages(data)
|
||||
elif data.startswith(b"PK\x03\x04"):
|
||||
fmt, pages = _zip_kind(data), None
|
||||
elif data.startswith(OLE_MAGIC):
|
||||
# 旧版 DOC 和 WPS 都是 OLE 复合文档;转换时一样交给 LibreOffice。加密的 DOC 在转换阶段报错
|
||||
fmt, pages = "doc", None
|
||||
else:
|
||||
raise AppError("file_format_unsupported", "支持 PDF、Word(DOCX / DOC)和 WPS 文件")
|
||||
if fmt == "ofd":
|
||||
raise AppError("file_format_unsupported", "暂不支持 OFD 版式文件,请上传 PDF 或 Word")
|
||||
return Detected(format=fmt, page_count=pages, extension_mismatch=ext not in EXTENSIONS.get(fmt, set()))
|
||||
|
||||
|
||||
def _pdf_pages(data: bytes) -> int:
|
||||
try:
|
||||
doc = pdfium.PdfDocument(data)
|
||||
except pdfium.PdfiumError as e:
|
||||
if "password" in str(e).lower():
|
||||
raise AppError("file_encrypted") from e
|
||||
raise AppError("file_corrupted") from e
|
||||
try:
|
||||
return len(doc)
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
|
||||
def _zip_kind(data: bytes) -> str:
|
||||
try:
|
||||
names = set(zipfile.ZipFile(io.BytesIO(data)).namelist())
|
||||
except zipfile.BadZipFile as e:
|
||||
raise AppError("file_corrupted") from e
|
||||
if "word/document.xml" in names:
|
||||
return "docx"
|
||||
if "OFD.xml" in names:
|
||||
return "ofd"
|
||||
if "EncryptionInfo" in names or "EncryptedPackage" in names:
|
||||
raise AppError("file_encrypted")
|
||||
raise AppError("file_format_unsupported", "支持 PDF、Word(DOCX / DOC)和 WPS 文件")
|
||||
87
server/src/bid_doctor/document/models.py
Normal file
87
server/src/bid_doctor/document/models.py
Normal file
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, Json, TextArray, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class SourceDocument(Base):
|
||||
__tablename__ = "source_document"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
role: Mapped[str]
|
||||
seq_no: Mapped[int]
|
||||
volume_name: Mapped[str | None]
|
||||
lineage_id: Mapped[UuidCol]
|
||||
original_filename: Mapped[str]
|
||||
detected_format: Mapped[str]
|
||||
size_bytes: Mapped[int]
|
||||
original_key: Mapped[str]
|
||||
original_sha256: Mapped[Bytes]
|
||||
normalized_key: Mapped[str | None]
|
||||
normalized_sha256: Mapped[Bytes | None]
|
||||
page_count: Mapped[int | None]
|
||||
text_layer_key: Mapped[str | None]
|
||||
pipeline_profile: Mapped[Json | None]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
uploaded_by: Mapped[UuidCol]
|
||||
created_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class DocumentPage(Base):
|
||||
__tablename__ = "document_page"
|
||||
tenant_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
document_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
page_no: Mapped[int] = mapped_column(primary_key=True)
|
||||
width_pt: Mapped[float]
|
||||
height_pt: Mapped[float]
|
||||
rotation: Mapped[int]
|
||||
text_source: Mapped[str]
|
||||
char_count: Mapped[int]
|
||||
sentence_count: Mapped[int]
|
||||
|
||||
|
||||
class ParseJob(Base):
|
||||
__tablename__ = "parse_job"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
document_id: Mapped[UuidCol]
|
||||
attempt_no: Mapped[int]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
stage: Mapped[str | None]
|
||||
checkpoints: Mapped[Json] = mapped_column(server_default=FetchedValue())
|
||||
progress: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
cancel_requested_at: Mapped[Ts | None]
|
||||
error_class: Mapped[str | None]
|
||||
error_code: Mapped[str | None]
|
||||
error_detail: Mapped[str | None]
|
||||
prompt_versions: Mapped[Json | None]
|
||||
quota_hold_id: Mapped[UuidCol | None]
|
||||
queue_job_id: Mapped[int | None]
|
||||
created_by: Mapped[UuidCol]
|
||||
created_at: Mapped[DbTs]
|
||||
started_at: Mapped[Ts | None]
|
||||
finished_at: Mapped[Ts | None]
|
||||
|
||||
|
||||
class Citation(Base):
|
||||
"""出处:不可变(data-model §5.1)。"""
|
||||
|
||||
__tablename__ = "citation"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
document_id: Mapped[UuidCol]
|
||||
normalized_sha256: Mapped[Bytes]
|
||||
page_no: Mapped[int]
|
||||
sentence_keys: Mapped[TextArray]
|
||||
char_start: Mapped[int | None]
|
||||
char_end: Mapped[int | None]
|
||||
quads: Mapped[Json]
|
||||
quote: Mapped[str]
|
||||
created_by_kind: Mapped[str]
|
||||
checks_passed: Mapped[TextArray]
|
||||
0
server/src/bid_doctor/document/pipeline/__init__.py
Normal file
0
server/src/bid_doctor/document/pipeline/__init__.py
Normal file
60
server/src/bid_doctor/document/pipeline/convert.py
Normal file
60
server/src/bid_doctor/document/pipeline/convert.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""转换:Word → 规范化 PDF(架构 §3.5,技术验证 S1)。PDF 原样作为规范化 PDF。
|
||||
|
||||
TODO(S1):
|
||||
- 转换镜像要带按宋体行高调整过的字体包(BD Song / BD Hei)和 fontconfig 替换规则,现在用的是 Gotenberg 默认字体,
|
||||
分页偏差会偏大(S1:默认配置 +17%~+37%);
|
||||
- WPS 生成的旧版 DOC 要先转 DOCX、修正字体字符集再转 PDF,否则正文可能是空白(S1 §1.3)。现在直接转。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
import httpx
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.errors import AppError
|
||||
|
||||
CJK_LATIN = {"SimSun", "NSimSun", "SimHei", "FangSong", "KaiTi", "Microsoft YaHei", "DengXian"}
|
||||
TIMEOUT = httpx.Timeout(300, connect=10) # 服务端 240s 先超时,错误由转换服务明确返回
|
||||
|
||||
|
||||
def fix_wps_charset(docx: bytes) -> bytes:
|
||||
"""WPS 把中文字体标成 SYMBOL(02) 字符集时改成 GB2312(86),否则 LibreOffice 会用符号字体排版(S1)。"""
|
||||
src = zipfile.ZipFile(io.BytesIO(docx))
|
||||
if "word/fontTable.xml" not in src.namelist():
|
||||
return docx
|
||||
|
||||
def repl(m: re.Match[str]) -> str:
|
||||
name, body = m.group(1), m.group(2)
|
||||
if (re.search(r"[一-鿿]", name) or name in CJK_LATIN) and 'w:charset w:val="02"' in body:
|
||||
body = body.replace('w:charset w:val="02"', 'w:charset w:val="86"')
|
||||
return f'<w:font w:name="{name}">{body}</w:font>'
|
||||
|
||||
out = io.BytesIO()
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst:
|
||||
for item in src.infolist():
|
||||
data = src.read(item.filename)
|
||||
if item.filename == "word/fontTable.xml":
|
||||
data = re.sub(
|
||||
r'<w:font w:name="([^"]+)">(.*?)</w:font>', repl, data.decode("utf-8"), flags=re.S
|
||||
).encode()
|
||||
dst.writestr(item, data)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
async def to_pdf(data: bytes, fmt: str) -> bytes:
|
||||
if fmt == "pdf":
|
||||
return data
|
||||
if fmt == "docx":
|
||||
data = fix_wps_charset(data)
|
||||
async with httpx.AsyncClient(base_url=get_settings().gotenberg_url, timeout=TIMEOUT) as c:
|
||||
r = await c.post("/forms/libreoffice/convert", files={"files": (f"input.{fmt}", data)})
|
||||
if r.status_code == 200:
|
||||
return r.content
|
||||
if r.status_code in (400, 422): # 文件本身转不了(加密、损坏):输入问题,不扣点
|
||||
raise AppError("file_corrupted", "这份 Word 文件无法转换,可能已加密或损坏")
|
||||
r.raise_for_status()
|
||||
raise AssertionError("unreachable")
|
||||
194
server/src/bid_doctor/document/pipeline/runner.py
Normal file
194
server/src/bid_doctor/document/pipeline/runner.py
Normal file
@@ -0,0 +1,194 @@
|
||||
"""解析流水线(架构 §3.4、§3.5):格式识别 → 转换 → 文字层 → OCR 补全 → 抽取 → 证据校验 → 清单合并。
|
||||
|
||||
- 每个阶段结束写检查点;重跑时已完成的阶段直接跳过(规范化 PDF 只生成一次,本来也不能重写)。
|
||||
- 阶段之间检查取消标记。
|
||||
- 结局由错误类别决定扣不扣点:基础设施错误、输入问题、取消 → 退回;成功 → 结算(架构 §3.6 的错误分类表)。
|
||||
TODO(结构识别):章节、表格、分包识别还没做,现在只抽实质性条款。TODO(S5):OCR 还没接,没有文字层的页直接报「需要 OCR」。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from bid_doctor.ai.providers.base import TransientError
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.checklist import extract
|
||||
from bid_doctor.checklist import service as checklist
|
||||
from bid_doctor.core import events, keys
|
||||
from bid_doctor.core.crypto import open_object, seal_object, sha256
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import storage
|
||||
from bid_doctor.document.models import DocumentPage, ParseJob, SourceDocument
|
||||
from bid_doctor.document.pipeline import convert, text_layer
|
||||
from bid_doctor.document.service import doc_prefix
|
||||
from bid_doctor.notification.models import Notification
|
||||
from bid_doctor.project import service as projects
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
PROGRESS = {"detect": 5, "convert": 15, "text_layer": 40, "ocr": 45, "extract": 55, "verify": 90, "merge": 95}
|
||||
PROFILE = {"converter": "gotenberg-8", "fonts": "gotenberg-default", "text_layer": "v1", "sentence_split": "s3-v1"}
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def run(tenant_id: UUID, job_id: UUID) -> None:
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=UUID(int=0))
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
if job.status not in ("queued", "running"):
|
||||
return # 重复派发:已经有结局了
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=job.created_by, actor_name="系统")
|
||||
job.status, job.started_at = "running", job.started_at or datetime.now(UTC)
|
||||
try:
|
||||
result = await _stages(ctx, job_id)
|
||||
except Cancelled:
|
||||
await _finish(ctx, job_id, "cancelled", "cancelled", None)
|
||||
except AppError as e:
|
||||
await _finish(ctx, job_id, "failed", "input", e.code)
|
||||
except (TransientError, httpx.HTTPError) as e:
|
||||
log.warning("parse job %s infra error: %s", job_id, e)
|
||||
code = "model_unavailable" if isinstance(e, TransientError) else "converter_unavailable"
|
||||
await _finish(ctx, job_id, "failed", "infra", code)
|
||||
except Exception:
|
||||
log.exception("parse job %s crashed", job_id)
|
||||
await _finish(ctx, job_id, "failed", "infra", "internal_error")
|
||||
else:
|
||||
await _finish(ctx, job_id, "succeeded", None, None, result)
|
||||
|
||||
|
||||
async def _stage(ctx: TenantContext, job_id: UUID, stage: str) -> dict[str, Any]:
|
||||
"""进入一个阶段:写进度、检查取消标记,返回已有的检查点。"""
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
if job.cancel_requested_at:
|
||||
raise Cancelled
|
||||
job.stage, job.progress = stage, PROGRESS[stage]
|
||||
checkpoints = dict(job.checkpoints or {})
|
||||
await events.publish(
|
||||
ctx.tenant_id,
|
||||
"job.progress",
|
||||
project_id=str(job.project_id),
|
||||
job={"kind": "parse_job", "id": str(job_id), "status": "running", "stage": stage, "progress": PROGRESS[stage]},
|
||||
)
|
||||
return checkpoints
|
||||
|
||||
|
||||
async def _stages(ctx: TenantContext, job_id: UUID) -> dict[str, int]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == job.document_id))).scalar_one()
|
||||
dk = await keys.data_key(s, ctx, doc.project_id)
|
||||
|
||||
await _stage(ctx, job_id, "detect") # 上传完成时已识别过格式;这里只核对原件没被替换
|
||||
original = open_object(dk, storage().get(doc.original_key), doc.original_key)
|
||||
if sha256(original) != doc.original_sha256:
|
||||
raise AppError("file_corrupted")
|
||||
|
||||
await _stage(ctx, job_id, "convert")
|
||||
if doc.normalized_key is None:
|
||||
pdf = await convert.to_pdf(original, doc.detected_format)
|
||||
key = f"{doc_prefix(doc)}/normalized.pdf"
|
||||
storage().put(key, seal_object(dk, pdf, key))
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.normalized_key, d.normalized_sha256, d.status = key, sha256(pdf), "normalizing"
|
||||
d.pipeline_profile = PROFILE
|
||||
doc.normalized_key, doc.normalized_sha256 = key, sha256(pdf)
|
||||
else:
|
||||
pdf = open_object(dk, storage().get(doc.normalized_key), doc.normalized_key)
|
||||
|
||||
await _stage(ctx, job_id, "text_layer")
|
||||
pages = text_layer.extract(str(doc.id), pdf)
|
||||
text_key = f"{doc_prefix(doc)}/text.json"
|
||||
storage().put(
|
||||
text_key, seal_object(dk, json.dumps([p.to_json() for p in pages], ensure_ascii=False).encode(), text_key)
|
||||
)
|
||||
async with tenant_tx(ctx) as s:
|
||||
await s.execute(delete(DocumentPage).where(DocumentPage.document_id == doc.id)) # 阶段幂等:先清残留
|
||||
for p in pages:
|
||||
s.add(
|
||||
DocumentPage(
|
||||
tenant_id=ctx.tenant_id,
|
||||
document_id=doc.id,
|
||||
page_no=p.page_no,
|
||||
width_pt=p.width,
|
||||
height_pt=p.height,
|
||||
rotation=p.rotation,
|
||||
text_source=p.text_source,
|
||||
char_count=p.char_count,
|
||||
sentence_count=len(p.sentences),
|
||||
)
|
||||
)
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
d.text_layer_key, d.page_count, d.status = text_key, len(pages), "ready"
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
if len(pages) > plan.limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded")
|
||||
|
||||
await _stage(ctx, job_id, "ocr")
|
||||
empty = sum(p.text_source == "empty" for p in pages)
|
||||
if pages and empty / len(pages) > 0.5:
|
||||
raise AppError("ocr_required")
|
||||
|
||||
await _stage(ctx, job_id, "extract")
|
||||
sentences = [x for p in pages for x in p.sentences]
|
||||
extracted, notes = await extract.run(ctx, job_id, sentences)
|
||||
|
||||
await _stage(ctx, job_id, "verify") # 三道校验在抽取时逐条做完;这里只是进度节点
|
||||
await _stage(ctx, job_id, "merge")
|
||||
async with tenant_tx(ctx) as s:
|
||||
d = (await s.execute(select(SourceDocument).where(SourceDocument.id == doc.id))).scalar_one()
|
||||
stats = await checklist.merge(s, ctx, d, job_id, extracted, notes)
|
||||
return {**stats, "lots_found": 0}
|
||||
|
||||
|
||||
async def _finish(
|
||||
ctx: TenantContext,
|
||||
job_id: UUID,
|
||||
status: str,
|
||||
error_class: str | None,
|
||||
code: str | None,
|
||||
result: dict[str, int] | None = None,
|
||||
) -> None:
|
||||
async with tenant_tx(ctx) as s:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one()
|
||||
job.status, job.finished_at, job.error_class, job.error_code = status, datetime.now(UTC), error_class, code
|
||||
job.progress = 100 if status == "succeeded" else job.progress
|
||||
if result is not None:
|
||||
job.checkpoints = {**(job.checkpoints or {}), "result": result}
|
||||
if job.quota_hold_id:
|
||||
if status == "succeeded":
|
||||
await billing.settle(s, ctx, job.quota_hold_id)
|
||||
else:
|
||||
await billing.release(s, ctx, job.quota_hold_id) # 失败、超时、取消都不扣点(红线 9)
|
||||
if status == "succeeded":
|
||||
await projects.mark_drafting(s, await projects.get(s, ctx, job.project_id))
|
||||
if status != "cancelled":
|
||||
s.add(
|
||||
Notification(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
user_id=job.created_by,
|
||||
kind="parse_succeeded" if status == "succeeded" else "parse_failed",
|
||||
project_id=job.project_id,
|
||||
payload={"job_id": str(job.id)},
|
||||
)
|
||||
)
|
||||
project_id = job.project_id
|
||||
await events.publish(
|
||||
ctx.tenant_id,
|
||||
"job.finished",
|
||||
project_id=str(project_id),
|
||||
job={"kind": "parse_job", "id": str(job_id), "status": status, "progress": 100},
|
||||
)
|
||||
225
server/src/bid_doctor/document/pipeline/text_layer.py
Normal file
225
server/src/bid_doctor/document/pipeline/text_layer.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""文字层:从规范化 PDF 取逐字坐标和格式 → 切句 → 每句生成 quads 和稳定编号(架构 §3.5,技术验证 S2、S3)。
|
||||
|
||||
切句规则:
|
||||
1. 同一行里字间空隙大于 1.5 个字号时断开(表格单元格、制表位),得到「行片段」;
|
||||
2. 句子在 。!?; 处结束;行片段没写到右边距附近(段落末行)时也结束;
|
||||
3. 句子不跨页。编号 <文档>:p<页>:s<页内序号>;规范化 PDF 不再重新生成,所以编号稳定。
|
||||
格式:斜体看字体名或字形倾斜;下划线看字下方的横线;加粗看字体名(S2:斜体且下划线按字符 100% 一致)。
|
||||
TODO(S3 改进):表格按单元格切句;页眉页脚带单独识别(现在都算正文)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import itertools
|
||||
import logging
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pdfplumber
|
||||
|
||||
END = set("。!?;")
|
||||
WS = re.compile(r"[\s ]")
|
||||
# pdfminer 对缺 FontBBox 的字体逐字报警告,没有信息量
|
||||
logging.getLogger("pdfminer").setLevel(logging.ERROR)
|
||||
|
||||
|
||||
# 部首补充区没有 NFKC 映射,简化字形的部首要单独查表
|
||||
RADICAL_SUPPLEMENT = dict(
|
||||
zip(
|
||||
"⺟⺠⻅⻆⻉⻋⻓⻔⻘⻚⻛⻜⻝⻢⻣⻤⻥⻦⻧⻨⻩⻬⻮⻰⻳",
|
||||
"母民见角贝车长门青页风飞食马骨鬼鱼鸟卤麦黄齐齿龙龟",
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _char(t: str) -> str:
|
||||
"""有的 PDF(如 Chromium 生成的)把汉字映射成康熙部首或部首补充区的码位(「目」成了 U+2F6C)。
|
||||
只把这两个区换回常用汉字;全角标点等其他字符保持原样,否则引文和原文对不上。"""
|
||||
if len(t) != 1:
|
||||
return t
|
||||
if "⼀" <= t <= "":
|
||||
return unicodedata.normalize("NFKC", t)
|
||||
return RADICAL_SUPPLEMENT.get(t, t)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Sentence:
|
||||
key: str
|
||||
page_no: int
|
||||
seq: int
|
||||
text: str
|
||||
quads: list[list[float]]
|
||||
fmt_runs: list[tuple[int, int, str]] = field(default_factory=list) # [起, 止, 格式],格式取 b / i / u 的组合
|
||||
location: str = "body"
|
||||
|
||||
@property
|
||||
def marked(self) -> str:
|
||||
"""给模型看的文本:斜体且下划线 → <iu>,只下划线 → <u>,只斜体 → <i>(S4 的输入格式)。"""
|
||||
out, pos = [], 0
|
||||
for start, end, f in self.fmt_runs:
|
||||
tag = "iu" if "i" in f and "u" in f else "u" if "u" in f else "i" if "i" in f else ""
|
||||
if not tag:
|
||||
continue
|
||||
out.append(self.text[pos:start])
|
||||
out.append(f"<{tag}>{self.text[start:end]}</{tag}>")
|
||||
pos = end
|
||||
out.append(self.text[pos:])
|
||||
return "".join(out)
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"key": self.key,
|
||||
"seq": self.seq,
|
||||
"text": self.text,
|
||||
"quads": self.quads,
|
||||
"format_runs": [list(r) for r in self.fmt_runs],
|
||||
"location": self.location,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, page_no: int, d: dict[str, Any]) -> Sentence:
|
||||
return cls(
|
||||
key=d["key"],
|
||||
page_no=page_no,
|
||||
seq=d["seq"],
|
||||
text=d["text"],
|
||||
quads=d["quads"],
|
||||
fmt_runs=[(a, b, f) for a, b, f in d["format_runs"]],
|
||||
location=d["location"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Page:
|
||||
page_no: int
|
||||
width: float
|
||||
height: float
|
||||
rotation: int
|
||||
char_count: int
|
||||
sentences: list[Sentence]
|
||||
|
||||
@property
|
||||
def text_source(self) -> str:
|
||||
return "text" if self.char_count >= 20 else "empty"
|
||||
|
||||
def to_json(self) -> dict[str, Any]:
|
||||
return {
|
||||
"page_no": self.page_no,
|
||||
"width_pt": self.width,
|
||||
"height_pt": self.height,
|
||||
"rotation": self.rotation,
|
||||
"char_count": self.char_count,
|
||||
"sentences": [s.to_json() for s in self.sentences],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, d: dict[str, Any]) -> Page:
|
||||
return cls(
|
||||
page_no=d["page_no"],
|
||||
width=d["width_pt"],
|
||||
height=d["height_pt"],
|
||||
rotation=d["rotation"],
|
||||
char_count=d["char_count"],
|
||||
sentences=[Sentence.from_json(d["page_no"], s) for s in d["sentences"]],
|
||||
)
|
||||
|
||||
|
||||
def _flags(c: dict[str, Any], hlines: list[tuple[float, float, float]]) -> str:
|
||||
name = str(c["fontname"]).split("+")[-1]
|
||||
italic = "Italic" in name or "Oblique" in name or abs(c["matrix"][2]) > 0.1
|
||||
mid = (c["x0"] + c["x1"]) / 2
|
||||
# 下划线一般在基线下 1~2pt;字体下行部分小时会落在字符框内,所以窗口向上放宽 0.25 个字号
|
||||
under = any(x0 - 0.5 <= mid <= x1 + 0.5 and -0.25 * c["size"] <= y - c["bottom"] <= 4 for x0, x1, y in hlines)
|
||||
return ("b" if "Bold" in name else "") + ("i" if italic else "") + ("u" if under else "")
|
||||
|
||||
|
||||
def _fragments(chars: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
|
||||
chars = sorted(chars, key=lambda c: (round(c["bottom"]), c["x0"]))
|
||||
lines: list[list[dict[str, Any]]] = []
|
||||
cur: list[dict[str, Any]] = []
|
||||
base: float | None = None
|
||||
for c in chars:
|
||||
if base is not None and abs(c["bottom"] - base) > 0.45 * c["size"]:
|
||||
lines.append(cur)
|
||||
cur = []
|
||||
if not cur:
|
||||
base = c["bottom"]
|
||||
cur.append(c)
|
||||
if cur:
|
||||
lines.append(cur)
|
||||
frags: list[list[dict[str, Any]]] = []
|
||||
for ln in lines:
|
||||
ln.sort(key=lambda c: c["x0"])
|
||||
f = [ln[0]]
|
||||
for a, b in itertools.pairwise(ln):
|
||||
if b["x0"] - a["x1"] > 1.5 * a["size"]:
|
||||
frags.append(f)
|
||||
f = []
|
||||
f.append(b)
|
||||
frags.append(f)
|
||||
return frags
|
||||
|
||||
|
||||
def _page(doc_id: str, page: Any) -> Page:
|
||||
hlines = [(ln["x0"], ln["x1"], ln["top"]) for ln in page.lines if abs(ln["top"] - ln["bottom"]) < 0.5]
|
||||
hlines += [(r["x0"], r["x1"], r["top"]) for r in page.rects if r["height"] < 1.5 and r["width"] > 2]
|
||||
chars = [c for c in page.chars if str(c["text"]).strip()]
|
||||
right = max((c["x1"] for c in chars), default=page.width)
|
||||
groups: list[list[tuple[int, dict[str, Any]]]] = []
|
||||
cur: list[tuple[int, dict[str, Any]]] = []
|
||||
for fi, frag in enumerate(_fragments(chars)):
|
||||
for c in frag:
|
||||
cur.append((fi, c))
|
||||
if c["text"] in END:
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
if cur and frag[-1]["x1"] < right - 2 * frag[-1]["size"]:
|
||||
groups.append(cur)
|
||||
cur = []
|
||||
if cur:
|
||||
groups.append(cur)
|
||||
sentences: list[Sentence] = []
|
||||
for seq, g in enumerate(groups, 1):
|
||||
quads: dict[int, list[float]] = {}
|
||||
runs: list[tuple[int, int, str]] = []
|
||||
text_parts: list[str] = []
|
||||
for pos, (fi, c) in enumerate(g):
|
||||
q = quads.setdefault(fi, [c["x0"], c["top"], c["x1"], c["bottom"]])
|
||||
q[0], q[1], q[2], q[3] = min(q[0], c["x0"]), min(q[1], c["top"]), max(q[2], c["x1"]), max(q[3], c["bottom"])
|
||||
text_parts.append(_char(str(c["text"])))
|
||||
f = _flags(c, hlines)
|
||||
if f and runs and runs[-1][1] == pos and runs[-1][2] == f:
|
||||
runs[-1] = (runs[-1][0], pos + 1, f)
|
||||
elif f:
|
||||
runs.append((pos, pos + 1, f))
|
||||
sentences.append(
|
||||
Sentence(
|
||||
key=f"{doc_id}:p{page.page_number}:s{seq}",
|
||||
page_no=page.page_number,
|
||||
seq=seq,
|
||||
text="".join(text_parts),
|
||||
quads=[[round(v, 2) for v in q] for q in quads.values()],
|
||||
fmt_runs=runs,
|
||||
)
|
||||
)
|
||||
return Page(
|
||||
page_no=page.page_number,
|
||||
width=float(page.width),
|
||||
height=float(page.height),
|
||||
rotation=int(page.rotation or 0) % 360,
|
||||
char_count=len(chars),
|
||||
sentences=sentences,
|
||||
)
|
||||
|
||||
|
||||
def extract(doc_id: str, pdf: bytes) -> list[Page]:
|
||||
with pdfplumber.open(io.BytesIO(pdf)) as doc:
|
||||
return [_page(doc_id, p) for p in doc.pages]
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
return WS.sub("", s)
|
||||
229
server/src/bid_doctor/document/service.py
Normal file
229
server/src/bid_doctor/document/service.py
Normal file
@@ -0,0 +1,229 @@
|
||||
"""文档、解析任务(FR-M2-01、02,FR-M3-01)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.audit import service as audit
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.core import keys, queue
|
||||
from bid_doctor.core.crypto import seal_object, sha256
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import project_prefix, storage
|
||||
from bid_doctor.document import uploads
|
||||
from bid_doctor.document.citations import document_label
|
||||
from bid_doctor.document.models import ParseJob, SourceDocument
|
||||
from bid_doctor.project import service as projects
|
||||
|
||||
PARSABLE = {"tender", "addendum", "clarification"}
|
||||
PARSE_TASK = "parse_document"
|
||||
|
||||
|
||||
def doc_prefix(doc: SourceDocument) -> str:
|
||||
return f"{project_prefix(doc.tenant_id, doc.project_id)}/doc/{doc.id}"
|
||||
|
||||
|
||||
async def create(
|
||||
s: AsyncSession, ctx: TenantContext, project_id: UUID, upload_id: UUID, role: str, volume_name: str | None
|
||||
) -> SourceDocument:
|
||||
p = await projects.get(s, ctx, project_id)
|
||||
projects.ensure_editable(p)
|
||||
meta, data = await uploads.take(ctx, upload_id)
|
||||
assert meta.detected_format is not None
|
||||
seq = (
|
||||
await s.execute(
|
||||
select(func.coalesce(func.max(SourceDocument.seq_no), -1) + 1).where(
|
||||
SourceDocument.project_id == project_id, SourceDocument.role == role
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
if role == "tender":
|
||||
seq = 0
|
||||
doc_id = uuid7()
|
||||
doc = SourceDocument(
|
||||
id=doc_id,
|
||||
tenant_id=ctx.tenant_id,
|
||||
project_id=project_id,
|
||||
role=role,
|
||||
seq_no=seq,
|
||||
volume_name=volume_name,
|
||||
lineage_id=doc_id,
|
||||
original_filename=meta.filename,
|
||||
detected_format=meta.detected_format,
|
||||
size_bytes=meta.size_bytes,
|
||||
original_key="",
|
||||
original_sha256=sha256(data),
|
||||
page_count=meta.page_count,
|
||||
uploaded_by=ctx.user_id,
|
||||
)
|
||||
doc.original_key = f"{doc_prefix(doc)}/original"
|
||||
dk = await keys.data_key(s, ctx, project_id)
|
||||
storage().put(doc.original_key, seal_object(dk, data, doc.original_key))
|
||||
s.add(doc)
|
||||
await audit.record(
|
||||
s,
|
||||
ctx,
|
||||
"document.create",
|
||||
"source_document",
|
||||
doc.id,
|
||||
project_id=project_id,
|
||||
after={"role": role, "filename": meta.filename},
|
||||
)
|
||||
await s.flush()
|
||||
await s.refresh(doc)
|
||||
await uploads.discard(ctx, upload_id)
|
||||
return doc
|
||||
|
||||
|
||||
async def get(s: AsyncSession, document_id: UUID) -> SourceDocument:
|
||||
doc = (await s.execute(select(SourceDocument).where(SourceDocument.id == document_id))).scalar_one_or_none()
|
||||
if doc is None:
|
||||
raise AppError("not_found")
|
||||
return doc
|
||||
|
||||
|
||||
async def start_parse(s: AsyncSession, ctx: TenantContext, document_id: UUID, expected_charge: int | None) -> ParseJob:
|
||||
"""预扣 1 次解析 → 建任务 → 入队,在同一个事务里(架构 §3.4)。"""
|
||||
doc = await get(s, document_id)
|
||||
if doc.role not in PARSABLE:
|
||||
raise AppError("document_role_not_parsable")
|
||||
projects.ensure_editable(await projects.get(s, ctx, doc.project_id))
|
||||
if expected_charge is not None and expected_charge != 1:
|
||||
raise AppError("billing_changed", charge=1) # 用户看到的扣点和实际不一致:拒绝,不扣点(API4)
|
||||
running = (
|
||||
await s.execute(
|
||||
select(ParseJob.id).where(ParseJob.document_id == doc.id, ParseJob.status.in_(("queued", "running")))
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if running:
|
||||
raise AppError("job_already_running", job_id=str(running))
|
||||
attempt = (
|
||||
await s.execute(
|
||||
select(func.coalesce(func.max(ParseJob.attempt_no), 0) + 1).where(ParseJob.document_id == doc.id)
|
||||
)
|
||||
).scalar_one()
|
||||
job = ParseJob(
|
||||
id=uuid7(),
|
||||
tenant_id=ctx.tenant_id,
|
||||
project_id=doc.project_id,
|
||||
document_id=doc.id,
|
||||
attempt_no=attempt,
|
||||
created_by=ctx.user_id,
|
||||
)
|
||||
s.add(job)
|
||||
await s.flush()
|
||||
job.quota_hold_id = await billing.hold(
|
||||
s, ctx, unit="parse", ref_type="parse_job", ref_id=job.id, project_id=doc.project_id
|
||||
)
|
||||
job.queue_job_id = await queue.enqueue(
|
||||
s, PARSE_TASK, {"tenant_id": str(ctx.tenant_id), "job_id": str(job.id)}, lock=f"doc:{doc.id}"
|
||||
)
|
||||
await audit.record(s, ctx, "parse_job.start", "parse_job", job.id, project_id=doc.project_id)
|
||||
await s.flush()
|
||||
await s.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
async def request_cancel(s: AsyncSession, job_id: UUID) -> ParseJob:
|
||||
job = await get_job(s, job_id)
|
||||
if job.status not in ("queued", "running"):
|
||||
raise AppError("job_finished")
|
||||
job.cancel_requested_at = datetime.now(UTC)
|
||||
await s.flush()
|
||||
return job
|
||||
|
||||
|
||||
async def get_job(s: AsyncSession, job_id: UUID) -> ParseJob:
|
||||
job = (await s.execute(select(ParseJob).where(ParseJob.id == job_id))).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise AppError("not_found")
|
||||
return job
|
||||
|
||||
|
||||
ERROR_MESSAGES = {
|
||||
"file_corrupted": "这份文件无法转换,可能已加密或损坏",
|
||||
"file_encrypted": "文件已加密,请上传未加密的版本",
|
||||
"ocr_required": "这份文件大部分页面没有文字层(可能是扫描件),扫描件识别还没有上线",
|
||||
"plan_limit_exceeded": "页数超出套餐上限",
|
||||
"converter_unavailable": "文档转换服务暂时不可用",
|
||||
"model_unavailable": "模型服务暂时不可用",
|
||||
"internal_error": "系统出错了",
|
||||
}
|
||||
|
||||
|
||||
def job_to_api(job: ParseJob) -> dict[str, Any]:
|
||||
error = None
|
||||
if job.status in ("failed", "cancelled") and job.error_code:
|
||||
error = {
|
||||
"code": job.error_code,
|
||||
"message": ERROR_MESSAGES.get(job.error_code, job.error_code),
|
||||
"refunded": True,
|
||||
}
|
||||
return {
|
||||
"id": job.id,
|
||||
"document_id": job.document_id,
|
||||
"attempt_no": job.attempt_no,
|
||||
"status": job.status,
|
||||
"stage": job.stage,
|
||||
"progress": job.progress,
|
||||
"cancel_requested": job.cancel_requested_at is not None,
|
||||
"error": error,
|
||||
"result": (job.checkpoints or {}).get("result"),
|
||||
"created_at": job.created_at,
|
||||
"finished_at": job.finished_at,
|
||||
}
|
||||
|
||||
|
||||
async def doc_to_api(s: AsyncSession, doc: SourceDocument) -> dict[str, Any]:
|
||||
latest = (
|
||||
await s.execute(
|
||||
select(ParseJob).where(ParseJob.document_id == doc.id).order_by(ParseJob.attempt_no.desc()).limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return {
|
||||
"id": doc.id,
|
||||
"role": doc.role,
|
||||
"label": document_label(doc.role, doc.seq_no),
|
||||
"seq_no": doc.seq_no,
|
||||
"volume_name": doc.volume_name,
|
||||
"lineage_id": doc.lineage_id,
|
||||
"is_latest_version": True,
|
||||
"original_filename": doc.original_filename,
|
||||
"detected_format": doc.detected_format,
|
||||
"size_bytes": doc.size_bytes,
|
||||
"page_count": doc.page_count,
|
||||
"normalized_sha256": doc.normalized_sha256.hex() if doc.normalized_sha256 else None,
|
||||
"status": doc.status,
|
||||
"latest_parse_job": job_to_api(latest) if latest else None,
|
||||
"uploaded_by": {"id": doc.uploaded_by, "name": ""},
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
|
||||
async def summaries(s: AsyncSession, project_ids: list[UUID]) -> dict[UUID, dict[str, Any]]:
|
||||
"""工作台用:每个项目的招标文件数、正在跑的解析。"""
|
||||
out: dict[UUID, dict[str, Any]] = {pid: {"tenders": 0, "job": None} for pid in project_ids}
|
||||
if not project_ids:
|
||||
return out
|
||||
for pid, n in (
|
||||
await s.execute(
|
||||
select(SourceDocument.project_id, func.count())
|
||||
.where(SourceDocument.project_id.in_(project_ids), SourceDocument.role == "tender")
|
||||
.group_by(SourceDocument.project_id)
|
||||
)
|
||||
).tuples():
|
||||
out[pid]["tenders"] = n
|
||||
for job in (
|
||||
await s.execute(
|
||||
select(ParseJob).where(ParseJob.project_id.in_(project_ids), ParseJob.status.in_(("queued", "running")))
|
||||
)
|
||||
).scalars():
|
||||
out[job.project_id]["job"] = {"kind": "parse_job", "id": job.id, "progress": job.progress, "label": "解析中"}
|
||||
return out
|
||||
162
server/src/bid_doctor/document/uploads.py
Normal file
162
server/src/bid_doctor/document/uploads.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""分片上传、断点续传(架构 §5.2 第 7 条,api-design §4.3)。
|
||||
|
||||
上传会话的元数据放在 Valkey(24 小时过期,丢了就重新上传);分片按企业级数据密钥加密后写进对象存储。
|
||||
TODO(大文件):现在合并时整份读进内存;200MB 以上改成流式合并、分段加密。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import open_object, seal_object
|
||||
from bid_doctor.core.db import TenantContext, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.core.storage import storage, upload_prefix
|
||||
from bid_doctor.core.valkey import valkey
|
||||
from bid_doctor.document.formats import detect
|
||||
|
||||
TTL_S = 24 * 3600
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class UploadMeta:
|
||||
id: str
|
||||
filename: str
|
||||
size_bytes: int
|
||||
purpose: str
|
||||
part_size: int
|
||||
part_count: int
|
||||
sha256: str | None = None
|
||||
detected_format: str | None = None
|
||||
page_count: int | None = None
|
||||
extension_mismatch: bool = False
|
||||
completed_sha256: str | None = None
|
||||
|
||||
|
||||
def _key(ctx: TenantContext, upload_id: UUID | str) -> str:
|
||||
return f"up:{ctx.tenant_id}:{upload_id}"
|
||||
|
||||
|
||||
async def _limits(ctx: TenantContext) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
return plan.limits
|
||||
|
||||
|
||||
async def create(
|
||||
ctx: TenantContext, filename: str, size_bytes: int, purpose: str, sha256: str | None
|
||||
) -> dict[str, Any]:
|
||||
limits = await _limits(ctx)
|
||||
if size_bytes > limits["max_file_bytes"]:
|
||||
raise AppError("plan_limit_exceeded", f"单个文件最大 {limits['max_file_bytes'] // (1024 * 1024)}MB")
|
||||
part = get_settings().upload_part_size
|
||||
meta = UploadMeta(
|
||||
id=str(uuid7()),
|
||||
filename=filename,
|
||||
size_bytes=size_bytes,
|
||||
purpose=purpose,
|
||||
part_size=part,
|
||||
part_count=max(1, -(-size_bytes // part)),
|
||||
sha256=sha256,
|
||||
)
|
||||
await valkey().set(_key(ctx, meta.id), json.dumps(asdict(meta)), ex=TTL_S)
|
||||
return await describe(ctx, meta.id)
|
||||
|
||||
|
||||
async def load(ctx: TenantContext, upload_id: UUID | str) -> UploadMeta:
|
||||
raw = await valkey().get(_key(ctx, upload_id))
|
||||
if raw is None:
|
||||
raise AppError("not_found")
|
||||
return UploadMeta(**json.loads(raw))
|
||||
|
||||
|
||||
async def describe(ctx: TenantContext, upload_id: UUID | str) -> dict[str, Any]:
|
||||
meta = await load(ctx, upload_id)
|
||||
parts = sorted(int(p) for p in await valkey().smembers(_key(ctx, upload_id) + ":parts"))
|
||||
ttl = await valkey().ttl(_key(ctx, upload_id))
|
||||
return {
|
||||
"id": meta.id,
|
||||
"part_size": meta.part_size,
|
||||
"part_count": meta.part_count,
|
||||
"received_parts": parts,
|
||||
"expires_at": datetime.now(UTC) + timedelta(seconds=max(ttl, 0)),
|
||||
}
|
||||
|
||||
|
||||
async def put_part(ctx: TenantContext, upload_id: UUID, part_no: int, data: bytes) -> None:
|
||||
meta = await load(ctx, upload_id)
|
||||
last = part_no == meta.part_count
|
||||
expected = meta.size_bytes - meta.part_size * (meta.part_count - 1) if last else meta.part_size
|
||||
if not 1 <= part_no <= meta.part_count or len(data) != expected:
|
||||
raise AppError("validation_failed", f"第 {part_no} 片的大小应为 {expected} 字节")
|
||||
async with tenant_tx(ctx) as s:
|
||||
dk = await keys.data_key(s, ctx)
|
||||
key = f"{upload_prefix(ctx.tenant_id, upload_id)}/part-{part_no:05d}"
|
||||
storage().put(key, seal_object(dk, data, key))
|
||||
await valkey().sadd(_key(ctx, upload_id) + ":parts", part_no)
|
||||
await valkey().expire(_key(ctx, upload_id) + ":parts", TTL_S)
|
||||
|
||||
|
||||
async def complete(ctx: TenantContext, upload_id: UUID) -> dict[str, Any]:
|
||||
meta = await load(ctx, upload_id)
|
||||
got = {int(p) for p in await valkey().smembers(_key(ctx, upload_id) + ":parts")}
|
||||
if got != set(range(1, meta.part_count + 1)):
|
||||
raise AppError("upload_incomplete", received_parts=sorted(got))
|
||||
async with tenant_tx(ctx) as s:
|
||||
dk = await keys.data_key(s, ctx)
|
||||
prefix = upload_prefix(ctx.tenant_id, upload_id)
|
||||
chunks = [
|
||||
open_object(dk, storage().get(k), k) for k in (f"{prefix}/part-{n:05d}" for n in range(1, meta.part_count + 1))
|
||||
]
|
||||
data = b"".join(chunks)
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if meta.sha256 and meta.sha256 != digest:
|
||||
raise AppError("upload_checksum_mismatch")
|
||||
d = detect(data, meta.filename) # 格式、加密、损坏都在这里拦下,不建任务、不扣点
|
||||
limits = await _limits(ctx)
|
||||
if d.page_count and d.page_count > limits["max_pages"]:
|
||||
raise AppError("plan_limit_exceeded", f"单个文件最多 {limits['max_pages']} 页,这份有 {d.page_count} 页")
|
||||
storage().put(f"{prefix}/file", seal_object(dk, data, f"{prefix}/file"))
|
||||
for n in range(1, meta.part_count + 1): # 分片用完即删
|
||||
storage().delete_prefix(f"{prefix}/part-{n:05d}")
|
||||
meta.detected_format, meta.page_count, meta.extension_mismatch, meta.completed_sha256 = (
|
||||
d.format,
|
||||
d.page_count,
|
||||
d.extension_mismatch,
|
||||
digest,
|
||||
)
|
||||
await valkey().set(_key(ctx, upload_id), json.dumps(asdict(meta)), ex=TTL_S)
|
||||
return {
|
||||
"id": meta.id,
|
||||
"filename": meta.filename,
|
||||
"size_bytes": meta.size_bytes,
|
||||
"sha256": digest,
|
||||
"detected_format": d.format,
|
||||
"extension_mismatch": d.extension_mismatch,
|
||||
"page_count": d.page_count,
|
||||
}
|
||||
|
||||
|
||||
async def take(ctx: TenantContext, upload_id: UUID) -> tuple[UploadMeta, bytes]:
|
||||
"""取走已完成的上传(明文),挂到项目上以后删除暂存。"""
|
||||
meta = await load(ctx, upload_id)
|
||||
if not meta.completed_sha256:
|
||||
raise AppError("upload_incomplete")
|
||||
async with tenant_tx(ctx) as s:
|
||||
dk = await keys.data_key(s, ctx)
|
||||
key = f"{upload_prefix(ctx.tenant_id, upload_id)}/file"
|
||||
return meta, open_object(dk, storage().get(key), key)
|
||||
|
||||
|
||||
async def discard(ctx: TenantContext, upload_id: UUID) -> None:
|
||||
storage().delete_prefix(upload_prefix(ctx.tenant_id, upload_id))
|
||||
await valkey().delete(_key(ctx, upload_id), _key(ctx, upload_id) + ":parts")
|
||||
0
server/src/bid_doctor/identity/__init__.py
Normal file
0
server/src/bid_doctor/identity/__init__.py
Normal file
81
server/src/bid_doctor/identity/api.py
Normal file
81
server/src/bid_doctor/identity/api.py
Normal file
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from bid_doctor.identity import service, sms
|
||||
from bid_doctor.identity.deps import Auth, clear_session_cookies, set_session_cookies
|
||||
|
||||
router = APIRouter(tags=["auth", "me"])
|
||||
PHONE = r"^1[3-9][0-9]{9}$"
|
||||
CODE = r"^[0-9]{6}$"
|
||||
|
||||
|
||||
class SmsCodeIn(BaseModel):
|
||||
phone: str = Field(pattern=PHONE)
|
||||
purpose: Literal["login", "register", "close_tenant"]
|
||||
|
||||
|
||||
class RegisterIn(BaseModel):
|
||||
phone: str = Field(pattern=PHONE)
|
||||
sms_code: str = Field(pattern=CODE)
|
||||
tenant_name: str = Field(min_length=2, max_length=100)
|
||||
display_name: str = Field(min_length=1, max_length=40)
|
||||
terms_version: str
|
||||
client_kind: Literal["web", "desktop"]
|
||||
|
||||
|
||||
class LoginIn(BaseModel):
|
||||
phone: str = Field(pattern=PHONE)
|
||||
sms_code: str = Field(pattern=CODE)
|
||||
client_kind: Literal["web", "desktop"]
|
||||
force: bool = False
|
||||
|
||||
|
||||
def _device(request: Request, client_kind: str) -> service.Device:
|
||||
ip = request.client.host if request.client else None
|
||||
return service.Device(client_kind=client_kind, user_agent=request.headers.get("user-agent"), ip=ip)
|
||||
|
||||
|
||||
async def _login_result(response: Response, token: str, p: service.Principal, client_kind: str) -> dict[str, Any]:
|
||||
if client_kind == "web":
|
||||
set_session_cookies(response, token)
|
||||
return {"me": await service.me(p)}
|
||||
return {"me": await service.me(p), "token": token} # 桌面版:令牌只返回这一次
|
||||
|
||||
|
||||
@router.post("/auth/sms-codes", status_code=202)
|
||||
async def send_sms_code(body: SmsCodeIn) -> dict[str, int]:
|
||||
return {"resend_after_s": await sms.send(body.phone, body.purpose)}
|
||||
|
||||
|
||||
@router.post("/auth/register", status_code=201)
|
||||
async def register(body: RegisterIn, request: Request, response: Response) -> dict[str, Any]:
|
||||
token, p = await service.register(
|
||||
body.phone,
|
||||
body.sms_code,
|
||||
body.tenant_name,
|
||||
body.display_name,
|
||||
body.terms_version,
|
||||
_device(request, body.client_kind),
|
||||
)
|
||||
return await _login_result(response, token, p, body.client_kind)
|
||||
|
||||
|
||||
@router.post("/auth/login")
|
||||
async def login(body: LoginIn, request: Request, response: Response) -> dict[str, Any]:
|
||||
token, p = await service.login(body.phone, body.sms_code, _device(request, body.client_kind), body.force)
|
||||
return await _login_result(response, token, p, body.client_kind)
|
||||
|
||||
|
||||
@router.post("/auth/logout", status_code=204)
|
||||
async def logout(p: Auth, response: Response) -> None:
|
||||
await service.logout(p)
|
||||
clear_session_cookies(response)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def get_me(p: Auth) -> dict[str, Any]:
|
||||
return await service.me(p)
|
||||
61
server/src/bid_doctor/identity/deps.py
Normal file
61
server/src/bid_doctor/identity/deps.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""鉴权依赖(api-design §2):网页版 cookie + CSRF 头,桌面版 Bearer 令牌。其他模块的接口都用它。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import Depends, Request, Response
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import new_token
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.identity.service import Principal, resolve, why_invalid
|
||||
|
||||
SESSION_COOKIE = "bd_session"
|
||||
CSRF_COOKIE = "bd_csrf"
|
||||
SAFE_METHODS = {"GET", "HEAD", "OPTIONS"}
|
||||
|
||||
|
||||
async def current_principal(request: Request) -> Principal:
|
||||
auth = request.headers.get("authorization", "")
|
||||
if auth.lower().startswith("bearer "):
|
||||
token = auth[7:].strip()
|
||||
else:
|
||||
token = request.cookies.get(SESSION_COOKIE, "")
|
||||
if token and request.method not in SAFE_METHODS:
|
||||
sent, expected = request.headers.get("x-csrf-token", ""), request.cookies.get(CSRF_COOKIE, "")
|
||||
if not expected or not hmac.compare_digest(sent, expected):
|
||||
raise AppError("csrf_failed")
|
||||
if not token:
|
||||
raise AppError("unauthenticated")
|
||||
p = await resolve(token, getattr(request.state, "request_id", ""))
|
||||
if p is None:
|
||||
code, reason = await why_invalid(token)
|
||||
raise AppError(code, **({"reason": reason} if reason else {}))
|
||||
return p
|
||||
|
||||
|
||||
async def current_ctx(p: Annotated[Principal, Depends(current_principal)]) -> TenantContext:
|
||||
return p.ctx
|
||||
|
||||
|
||||
Auth = Annotated[Principal, Depends(current_principal)]
|
||||
Ctx = Annotated[TenantContext, Depends(current_ctx)]
|
||||
|
||||
|
||||
def set_session_cookies(response: Response, token: str) -> None:
|
||||
s = get_settings()
|
||||
max_age = s.session_ttl_hours * 3600
|
||||
response.set_cookie(
|
||||
SESSION_COOKIE, token, max_age=max_age, httponly=True, secure=s.cookie_secure, samesite="lax", path="/"
|
||||
)
|
||||
response.set_cookie(
|
||||
CSRF_COOKIE, new_token(), max_age=max_age, httponly=False, secure=s.cookie_secure, samesite="lax", path="/"
|
||||
)
|
||||
|
||||
|
||||
def clear_session_cookies(response: Response) -> None:
|
||||
response.delete_cookie(SESSION_COOKIE, path="/")
|
||||
response.delete_cookie(CSRF_COOKIE, path="/")
|
||||
67
server/src/bid_doctor/identity/models.py
Normal file
67
server/src/bid_doctor/identity/models.py
Normal file
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, Bytes, DbTs, Inet, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class Tenant(Base):
|
||||
__tablename__ = "tenant"
|
||||
id: Mapped[PK]
|
||||
name: Mapped[str]
|
||||
uscc: Mapped[str | None]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
created_at: Mapped[DbTs]
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class UserAccount(Base):
|
||||
"""全局表:不带 tenant_id,不受行级安全策略约束,只有 identity 模块能读写。"""
|
||||
|
||||
__tablename__ = "user_account"
|
||||
id: Mapped[PK]
|
||||
display_name: Mapped[str]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
created_at: Mapped[DbTs]
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class LoginIdentity(Base):
|
||||
__tablename__ = "login_identity"
|
||||
id: Mapped[PK]
|
||||
user_id: Mapped[UuidCol]
|
||||
provider: Mapped[str]
|
||||
external_org_id: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
external_user_id: Mapped[str]
|
||||
contact_cipher: Mapped[Bytes | None]
|
||||
contact_hint: Mapped[str | None]
|
||||
verified_at: Mapped[Ts | None]
|
||||
last_login_at: Mapped[Ts | None]
|
||||
|
||||
|
||||
class Membership(Base):
|
||||
__tablename__ = "membership"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
user_id: Mapped[UuidCol]
|
||||
role: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
|
||||
|
||||
class UserSession(Base):
|
||||
__tablename__ = "user_session"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
membership_id: Mapped[UuidCol]
|
||||
user_id: Mapped[UuidCol]
|
||||
token_hash: Mapped[Bytes]
|
||||
client_kind: Mapped[str]
|
||||
user_agent: Mapped[str | None]
|
||||
ip: Mapped[Inet | None]
|
||||
created_at: Mapped[DbTs]
|
||||
last_seen_at: Mapped[DbTs]
|
||||
expires_at: Mapped[Ts]
|
||||
revoked_at: Mapped[Ts | None]
|
||||
revoke_reason: Mapped[str | None]
|
||||
261
server/src/bid_doctor/identity/service.py
Normal file
261
server/src/bid_doctor/identity/service.py
Normal file
@@ -0,0 +1,261 @@
|
||||
"""人、登录方式、企业主体、成员身份、会话(架构 §3.14,AR11)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, text, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.audit import service as audit
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.core import keys
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import encrypt_contact, new_token, phone_hint, phone_lookup_key, sha256
|
||||
from bid_doctor.core.db import TenantContext, anonymous_tx, set_tenant, tenant_tx
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.identity import sms
|
||||
from bid_doctor.identity.models import LoginIdentity, Membership, Tenant, UserAccount, UserSession
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Principal:
|
||||
ctx: TenantContext
|
||||
session_id: UUID
|
||||
role: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Device:
|
||||
client_kind: str
|
||||
user_agent: str | None
|
||||
ip: str | None
|
||||
|
||||
|
||||
async def register(
|
||||
phone: str, code: str, tenant_name: str, display_name: str, terms_version: str, device: Device
|
||||
) -> tuple[str, Principal]:
|
||||
await sms.verify(phone, "register", code)
|
||||
lk = phone_lookup_key(phone)
|
||||
async with anonymous_tx() as s:
|
||||
exists = (
|
||||
await s.execute(
|
||||
select(LoginIdentity.id).where(
|
||||
LoginIdentity.provider == "phone",
|
||||
LoginIdentity.external_org_id == "",
|
||||
LoginIdentity.external_user_id == lk,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if exists:
|
||||
raise AppError("phone_already_registered")
|
||||
user = UserAccount(id=uuid7(), display_name=display_name)
|
||||
s.add(user)
|
||||
await s.flush() # 模型不声明外键(表结构以迁移为准),写入顺序要自己保证
|
||||
s.add(
|
||||
LoginIdentity(
|
||||
id=uuid7(),
|
||||
user_id=user.id,
|
||||
provider="phone",
|
||||
external_user_id=lk,
|
||||
contact_cipher=encrypt_contact(phone),
|
||||
contact_hint=phone_hint(phone),
|
||||
verified_at=datetime.now(UTC),
|
||||
last_login_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
await s.flush()
|
||||
tenant_id = uuid7()
|
||||
await set_tenant(s, tenant_id) # 行级安全策略:先切到新租户,才能写它的数据
|
||||
s.add(Tenant(id=tenant_id, name=tenant_name))
|
||||
await s.flush()
|
||||
m = Membership(id=uuid7(), tenant_id=tenant_id, user_id=user.id, role="owner")
|
||||
s.add(m)
|
||||
await s.flush()
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=user.id, membership_id=m.id, actor_name=display_name)
|
||||
await keys.data_key(s, ctx)
|
||||
await billing.start_free_subscription(s, ctx)
|
||||
token, session_id = await _new_session(s, ctx, device)
|
||||
# 同意的服务协议版本(含围串标禁止条款,红线 8)记进审计,可追溯
|
||||
await audit.record(s, ctx, "tenant.register", "tenant", tenant_id, after={"terms_version": terms_version})
|
||||
return token, Principal(ctx=ctx, session_id=session_id, role="owner")
|
||||
|
||||
|
||||
async def login(phone: str, code: str, device: Device, force: bool) -> tuple[str, Principal]:
|
||||
await sms.verify(phone, "login", code, consume=False)
|
||||
lk = phone_lookup_key(phone)
|
||||
async with anonymous_tx() as s:
|
||||
row = (
|
||||
await s.execute(
|
||||
select(LoginIdentity, UserAccount)
|
||||
.join(UserAccount, UserAccount.id == LoginIdentity.user_id)
|
||||
.where(
|
||||
LoginIdentity.provider == "phone",
|
||||
LoginIdentity.external_org_id == "",
|
||||
LoginIdentity.external_user_id == lk,
|
||||
UserAccount.status == "active",
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
# 验证码保留:用户可以直接用它注册(api-design §2)
|
||||
raise AppError("phone_not_registered")
|
||||
ident, user = row
|
||||
ms = (
|
||||
await s.execute(
|
||||
text("SELECT membership_id, tenant_id, role FROM auth_list_memberships(:u) WHERE status = 'active'"),
|
||||
{"u": user.id},
|
||||
)
|
||||
).all()
|
||||
if not ms:
|
||||
raise AppError("forbidden", "这个账号没有可用的企业")
|
||||
membership_id, tenant_id, role = ms[0] # MVP 一人只属于一个企业(FR-M6-04)
|
||||
ident.last_login_at = datetime.now(UTC)
|
||||
ctx = TenantContext(tenant_id=tenant_id, user_id=user.id, membership_id=membership_id, actor_name=user.display_name)
|
||||
async with tenant_tx(ctx) as s:
|
||||
others = (
|
||||
(
|
||||
await s.execute(
|
||||
select(UserSession).where(
|
||||
UserSession.user_id == user.id,
|
||||
UserSession.revoked_at.is_(None),
|
||||
UserSession.expires_at > datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if others and not force:
|
||||
raise AppError("concurrent_session", sessions=[session_info(o, None) for o in others])
|
||||
for o in others: # 用户确认后挤掉旧设备(API3)
|
||||
o.revoked_at, o.revoke_reason = datetime.now(UTC), "concurrent_login"
|
||||
token, session_id = await _new_session(s, ctx, device)
|
||||
await audit.record(
|
||||
s,
|
||||
ctx,
|
||||
"session.login",
|
||||
"user_session",
|
||||
session_id,
|
||||
after={"client_kind": device.client_kind, "revoked_others": len(others)},
|
||||
)
|
||||
# 会话建好才作废验证码:遇到「已在其他设备登录」时,用户确认后可以用同一个验证码重发(API3)
|
||||
await sms.consume(phone, "login")
|
||||
return token, Principal(ctx=ctx, session_id=session_id, role=role)
|
||||
|
||||
|
||||
async def _new_session(s: AsyncSession, ctx: TenantContext, device: Device) -> tuple[str, UUID]:
|
||||
assert ctx.membership_id is not None
|
||||
token = new_token()
|
||||
sid = uuid7()
|
||||
s.add(
|
||||
UserSession(
|
||||
id=sid,
|
||||
tenant_id=ctx.tenant_id,
|
||||
membership_id=ctx.membership_id,
|
||||
user_id=ctx.user_id,
|
||||
token_hash=sha256(token),
|
||||
client_kind=device.client_kind,
|
||||
user_agent=device.user_agent,
|
||||
ip=device.ip,
|
||||
expires_at=datetime.now(UTC) + timedelta(hours=get_settings().session_ttl_hours),
|
||||
)
|
||||
)
|
||||
await s.flush()
|
||||
return token, sid
|
||||
|
||||
|
||||
async def resolve(token: str, request_id: str) -> Principal | None:
|
||||
"""按令牌哈希找会话:登录前还不知道租户,走 SECURITY DEFINER 函数(data-model §2.3)。"""
|
||||
async with anonymous_tx() as s:
|
||||
row = (
|
||||
await s.execute(
|
||||
text(
|
||||
"SELECT r.session_id, r.tenant_id, r.membership_id, r.user_id, u.display_name, m.role "
|
||||
"FROM auth_resolve_session(:h) r JOIN user_account u ON u.id = r.user_id "
|
||||
"JOIN auth_list_memberships(r.user_id) m ON m.membership_id = r.membership_id"
|
||||
),
|
||||
{"h": sha256(token)},
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
return None
|
||||
sid, tenant_id, membership_id, user_id, name, role = row
|
||||
ctx = TenantContext(
|
||||
tenant_id=tenant_id, user_id=user_id, membership_id=membership_id, request_id=request_id, actor_name=name
|
||||
)
|
||||
async with tenant_tx(ctx) as s: # 「最近活跃时间」最多每 5 分钟写一次(架构 §3.11)
|
||||
await s.execute(
|
||||
text(
|
||||
"UPDATE user_session SET last_seen_at = now()"
|
||||
" WHERE id = :id AND last_seen_at < now() - interval '5 minutes'"
|
||||
),
|
||||
{"id": sid},
|
||||
)
|
||||
return Principal(ctx=ctx, session_id=sid, role=role)
|
||||
|
||||
|
||||
async def why_invalid(token: str) -> tuple[str, str | None]:
|
||||
"""令牌查不到有效会话时,区分被吊销(含被挤下线)和过期,前端据此提示(API3)。"""
|
||||
async with anonymous_tx() as s:
|
||||
row = (
|
||||
await s.execute(text("SELECT revoke_reason, expired FROM auth_session_state(:h)"), {"h": sha256(token)})
|
||||
).first()
|
||||
if row is not None and row[0] is not None and row[0] != "logout":
|
||||
return "session_revoked", row[0]
|
||||
return "session_expired", None
|
||||
|
||||
|
||||
async def logout(p: Principal) -> None:
|
||||
async with tenant_tx(p.ctx) as s:
|
||||
await s.execute(
|
||||
update(UserSession)
|
||||
.where(UserSession.id == p.session_id)
|
||||
.values(revoked_at=datetime.now(UTC), revoke_reason="logout")
|
||||
)
|
||||
|
||||
|
||||
async def me(p: Principal) -> dict[str, Any]:
|
||||
from bid_doctor.core.authz import permissions_for
|
||||
|
||||
async with tenant_tx(p.ctx) as s:
|
||||
tenant = (await s.execute(select(Tenant).where(Tenant.id == p.ctx.tenant_id))).scalar_one()
|
||||
user = (await s.execute(select(UserAccount).where(UserAccount.id == p.ctx.user_id))).scalar_one()
|
||||
hint = (
|
||||
await s.execute(
|
||||
select(LoginIdentity.contact_hint).where(
|
||||
LoginIdentity.user_id == user.id, LoginIdentity.provider == "phone"
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
sub, plan = await billing.current_plan(s, p.ctx)
|
||||
return {
|
||||
"user": {"id": user.id, "display_name": user.display_name, "phone_hint": hint or ""},
|
||||
"tenant": {"id": tenant.id, "name": tenant.name},
|
||||
"role": p.role,
|
||||
"plan": billing.plan_summary(sub, plan),
|
||||
"permissions": sorted(permissions_for(p.role)),
|
||||
}
|
||||
|
||||
|
||||
def session_info(o: UserSession, current: UUID | None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": str(o.id),
|
||||
"client_kind": o.client_kind,
|
||||
"device": _device_label(o.user_agent),
|
||||
"created_at": o.created_at.isoformat(),
|
||||
"last_seen_at": o.last_seen_at.isoformat(),
|
||||
"current": o.id == current,
|
||||
}
|
||||
|
||||
|
||||
def _device_label(ua: str | None) -> str:
|
||||
if not ua:
|
||||
return "未知设备"
|
||||
browser = next((b for b in ("Edg", "Chrome", "Firefox", "Safari") if b in ua), "浏览器")
|
||||
system = next((n for k, n in (("Windows", "Windows"), ("Mac OS", "macOS"), ("Linux", "Linux")) if k in ua), "")
|
||||
return f"{browser.replace('Edg', 'Edge')} · {system}".strip(" ·")
|
||||
63
server/src/bid_doctor/identity/sms.py
Normal file
63
server/src/bid_doctor/identity/sms.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""短信验证码:存在 Valkey 里,带过期时间(架构 §3.11)。丢了最多是用户重新获取一次。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
from bid_doctor.core.crypto import phone_lookup_key
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.valkey import hit_limit, valkey
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
CODE_TTL_S = 300
|
||||
RESEND_S = 60
|
||||
MAX_ATTEMPTS = 5
|
||||
|
||||
|
||||
def _scope(purpose: str) -> str:
|
||||
"""登录和注册共用一个验证码:用户用登录的验证码发现「还没注册」时,可以直接拿它去注册。"""
|
||||
return "auth" if purpose in ("login", "register") else purpose
|
||||
|
||||
|
||||
async def send(phone: str, purpose: str) -> int:
|
||||
purpose = _scope(purpose)
|
||||
lk = phone_lookup_key(phone)
|
||||
wait = await hit_limit(f"sms:rate:{lk}", 1, RESEND_S) or await hit_limit(f"sms:hour:{lk}", 10, 3600)
|
||||
if wait:
|
||||
raise AppError("rate_limited", retry_after_s=wait)
|
||||
settings = get_settings()
|
||||
code = settings.sms_dev_code if settings.sms_provider == "dev" else f"{secrets.randbelow(10**6):06d}"
|
||||
await valkey().set(f"sms:code:{purpose}:{lk}", code, ex=CODE_TTL_S)
|
||||
await valkey().delete(f"sms:tries:{purpose}:{lk}")
|
||||
if settings.sms_provider == "dev":
|
||||
log.warning("dev 短信通道:%s 的验证码是 %s", phone[:3] + "****" + phone[-4:], code)
|
||||
else: # pragma: no cover - 真实通道上线前接入
|
||||
raise NotImplementedError("短信通道还没有接入")
|
||||
return RESEND_S
|
||||
|
||||
|
||||
async def verify(phone: str, purpose: str, code: str, *, consume: bool = True) -> None:
|
||||
"""校验验证码;连续错 5 次作废。consume=False 时校验通过也保留,由调用方稍后 consume()。"""
|
||||
purpose = _scope(purpose)
|
||||
lk = phone_lookup_key(phone)
|
||||
r = valkey()
|
||||
tries = await r.incr(f"sms:tries:{purpose}:{lk}")
|
||||
await r.expire(f"sms:tries:{purpose}:{lk}", CODE_TTL_S)
|
||||
expected = await r.get(f"sms:code:{purpose}:{lk}")
|
||||
if tries > MAX_ATTEMPTS or expected is None or not hmac.compare_digest(str(expected), code):
|
||||
if tries > MAX_ATTEMPTS:
|
||||
await r.delete(f"sms:code:{purpose}:{lk}")
|
||||
raise AppError("sms_code_invalid")
|
||||
if consume:
|
||||
await r.delete(f"sms:code:{purpose}:{lk}", f"sms:tries:{purpose}:{lk}")
|
||||
else:
|
||||
await r.decr(f"sms:tries:{purpose}:{lk}")
|
||||
|
||||
|
||||
async def consume(phone: str, purpose: str) -> None:
|
||||
lk = phone_lookup_key(phone)
|
||||
purpose = _scope(purpose)
|
||||
await valkey().delete(f"sms:code:{purpose}:{lk}", f"sms:tries:{purpose}:{lk}")
|
||||
0
server/src/bid_doctor/notification/__init__.py
Normal file
0
server/src/bid_doctor/notification/__init__.py
Normal file
57
server/src/bid_doctor/notification/api.py
Normal file
57
server/src/bid_doctor/notification/api.py
Normal file
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Header
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from bid_doctor.core import events
|
||||
from bid_doctor.core.db import tenant_tx
|
||||
from bid_doctor.identity.deps import Ctx
|
||||
from bid_doctor.notification.models import Notification
|
||||
|
||||
router = APIRouter(tags=["notifications"])
|
||||
|
||||
|
||||
@router.get("/events")
|
||||
async def stream_events(ctx: Ctx, last_event_id: Annotated[str | None, Header()] = None) -> StreamingResponse:
|
||||
"""SSE(API2:前端用 fetch 读,可以带 Bearer 头)。"""
|
||||
return StreamingResponse(
|
||||
events.stream(ctx.tenant_id, last_event_id),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/notifications")
|
||||
async def list_notifications(ctx: Ctx, unread: bool | None = None) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
q = select(Notification).where(Notification.user_id == ctx.user_id)
|
||||
if unread:
|
||||
q = q.where(Notification.read_at.is_(None))
|
||||
rows = (await s.execute(q.order_by(Notification.created_at.desc()).limit(50))).scalars().all()
|
||||
n = (
|
||||
await s.execute(
|
||||
select(func.count())
|
||||
.select_from(Notification)
|
||||
.where(Notification.user_id == ctx.user_id, Notification.read_at.is_(None))
|
||||
)
|
||||
).scalar_one()
|
||||
titles = {"parse_succeeded": "招标文件解析完成", "parse_failed": "招标文件解析失败,额度已退回"}
|
||||
return {
|
||||
"unread_count": n,
|
||||
"next_cursor": None,
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"kind": r.kind,
|
||||
"title": titles.get(r.kind, r.kind),
|
||||
"project_id": r.project_id,
|
||||
"link": f"/projects/{r.project_id}/checklist" if r.project_id else None,
|
||||
"read_at": r.read_at,
|
||||
"created_at": r.created_at,
|
||||
}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
19
server/src/bid_doctor/notification/models.py
Normal file
19
server/src/bid_doctor/notification/models.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, DbTs, Json, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class Notification(Base):
|
||||
__tablename__ = "notification"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
user_id: Mapped[UuidCol]
|
||||
kind: Mapped[str]
|
||||
project_id: Mapped[UuidCol | None]
|
||||
payload: Mapped[Json] = mapped_column(server_default=FetchedValue())
|
||||
read_at: Mapped[Ts | None]
|
||||
created_at: Mapped[DbTs]
|
||||
0
server/src/bid_doctor/project/__init__.py
Normal file
0
server/src/bid_doctor/project/__init__.py
Normal file
37
server/src/bid_doctor/project/api.py
Normal file
37
server/src/bid_doctor/project/api.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from bid_doctor.core.db import tenant_tx
|
||||
from bid_doctor.identity.deps import Ctx
|
||||
from bid_doctor.project import service, workbench
|
||||
|
||||
router = APIRouter(tags=["projects"])
|
||||
|
||||
|
||||
class ProjectIn(BaseModel):
|
||||
name: str | None = Field(default=None, max_length=200)
|
||||
|
||||
|
||||
@router.get("/projects")
|
||||
async def list_projects(ctx: Ctx, limit: int = Query(default=20, ge=1, le=100)) -> dict[str, Any]:
|
||||
# TODO(工作台筛选):status / deadline / q / 游标分页,按 openapi.yaml 补齐
|
||||
async with tenant_tx(ctx) as s:
|
||||
return {"items": await workbench.rows(s, ctx, limit), "next_cursor": None}
|
||||
|
||||
|
||||
@router.post("/projects", status_code=201)
|
||||
async def create_project(body: ProjectIn, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
p = await service.create(s, ctx, body.name)
|
||||
return await workbench.detail(s, ctx, p.id)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}")
|
||||
async def get_project(project_id: UUID, ctx: Ctx) -> dict[str, Any]:
|
||||
async with tenant_tx(ctx) as s:
|
||||
return await workbench.detail(s, ctx, project_id)
|
||||
50
server/src/bid_doctor/project/models.py
Normal file
50
server/src/bid_doctor/project/models.py
Normal file
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import FetchedValue
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from bid_doctor.core.columns import PK, DbTs, Json, TextArray, Ts, UuidCol
|
||||
from bid_doctor.core.db import Base
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "project"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
name: Mapped[str]
|
||||
code: Mapped[str | None]
|
||||
purchaser: Mapped[str | None]
|
||||
agency: Mapped[str | None]
|
||||
procurement_method: Mapped[str | None]
|
||||
legal_regime: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
deadline_at: Mapped[Ts | None]
|
||||
status: Mapped[str] = mapped_column(server_default=FetchedValue())
|
||||
latest_check_run_id: Mapped[UuidCol | None]
|
||||
open_fatal_count: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
created_by: Mapped[UuidCol]
|
||||
created_at: Mapped[DbTs]
|
||||
updated_at: Mapped[DbTs]
|
||||
|
||||
|
||||
class ProjectMember(Base):
|
||||
__tablename__ = "project_member"
|
||||
tenant_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
project_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[UuidCol] = mapped_column(primary_key=True)
|
||||
role: Mapped[str]
|
||||
added_by: Mapped[UuidCol]
|
||||
|
||||
|
||||
class Lot(Base):
|
||||
__tablename__ = "lot"
|
||||
id: Mapped[PK]
|
||||
tenant_id: Mapped[UuidCol]
|
||||
project_id: Mapped[UuidCol]
|
||||
lot_no: Mapped[str]
|
||||
name: Mapped[str | None]
|
||||
is_default: Mapped[bool] = mapped_column(server_default=FetchedValue())
|
||||
is_bidding: Mapped[bool] = mapped_column(server_default=FetchedValue())
|
||||
origin: Mapped[str]
|
||||
ai_values: Mapped[Json | None]
|
||||
edited_fields: Mapped[TextArray]
|
||||
sort_order: Mapped[int] = mapped_column(server_default=FetchedValue())
|
||||
85
server/src/bid_doctor/project/service.py
Normal file
85
server/src/bid_doctor/project/service.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""项目、项目成员、分包(FR-M6-01、02)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.audit import service as audit
|
||||
from bid_doctor.billing import service as billing
|
||||
from bid_doctor.core.authz import Permission, require
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.core.errors import AppError
|
||||
from bid_doctor.core.ids import uuid7
|
||||
from bid_doctor.project.models import Lot, Project, ProjectMember
|
||||
|
||||
UNNAMED = "未命名项目"
|
||||
|
||||
|
||||
async def create(s: AsyncSession, ctx: TenantContext, name: str | None) -> Project:
|
||||
require(ctx, Permission.PROJECT_CREATE)
|
||||
_, plan = await billing.current_plan(s, ctx)
|
||||
limit = plan.limits.get("max_projects")
|
||||
if limit is not None:
|
||||
n = (await s.execute(select(func.count()).select_from(Project))).scalar_one()
|
||||
if n >= limit:
|
||||
raise AppError("plan_limit_exceeded", f"{plan.name}最多 {limit} 个项目")
|
||||
p = Project(id=uuid7(), tenant_id=ctx.tenant_id, name=(name or "").strip() or UNNAMED, created_by=ctx.user_id)
|
||||
s.add(p)
|
||||
await s.flush()
|
||||
s.add(
|
||||
ProjectMember(tenant_id=ctx.tenant_id, project_id=p.id, user_id=ctx.user_id, role="lead", added_by=ctx.user_id)
|
||||
)
|
||||
# 没有分包时也有一个默认包(04 §2);解析出分包后替换
|
||||
s.add(Lot(id=uuid7(), tenant_id=ctx.tenant_id, project_id=p.id, lot_no="默认", is_default=True, origin="ai"))
|
||||
await audit.record(s, ctx, "project.create", "project", p.id, project_id=p.id, after={"name": p.name})
|
||||
await s.flush()
|
||||
await s.refresh(p)
|
||||
return p
|
||||
|
||||
|
||||
async def get(s: AsyncSession, ctx: TenantContext, project_id: UUID) -> Project:
|
||||
"""按可见范围取项目(AR12):MVP 只有租户这一层,交给行级安全策略;以后在这里加项目成员条件。"""
|
||||
del ctx
|
||||
p = (await s.execute(select(Project).where(Project.id == project_id))).scalar_one_or_none()
|
||||
if p is None:
|
||||
raise AppError("not_found")
|
||||
return p
|
||||
|
||||
|
||||
def ensure_editable(p: Project) -> None:
|
||||
if p.status in ("submitted", "archived"):
|
||||
raise AppError("project_read_only")
|
||||
|
||||
|
||||
async def mark_drafting(s: AsyncSession, p: Project) -> None:
|
||||
"""首份招标文件解析成功:跟进中 → 编制中(04 §3.1)。"""
|
||||
if p.status == "tracking":
|
||||
p.status = "drafting"
|
||||
|
||||
|
||||
async def list_projects(s: AsyncSession, ctx: TenantContext, limit: int) -> list[tuple[Project, list[str]]]:
|
||||
"""只读本模块的表;工作台要的其他模块信息由 workbench 组合(模块之间不直接读写对方的表,架构 §3.2)。"""
|
||||
del ctx
|
||||
projects = (
|
||||
(
|
||||
await s.execute(
|
||||
select(Project).order_by(Project.deadline_at.asc().nulls_last(), Project.created_at.desc()).limit(limit)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
lots: dict[Any, list[str]] = {}
|
||||
for pid, lot_no in (
|
||||
await s.execute(
|
||||
select(Lot.project_id, Lot.lot_no)
|
||||
.where(Lot.project_id.in_([p.id for p in projects]), Lot.is_bidding, Lot.is_default.is_(False))
|
||||
.order_by(Lot.sort_order)
|
||||
)
|
||||
).tuples():
|
||||
lots.setdefault(pid, []).append(lot_no)
|
||||
return [(p, lots.get(p.id, [])) for p in projects]
|
||||
74
server/src/bid_doctor/project/workbench.py
Normal file
74
server/src/bid_doctor/project/workbench.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""工作台的项目行(FR-M6-01):组合 project、document、checklist 三个模块的公开服务,不直接读别的模块的表。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from bid_doctor.checklist import service as checklist
|
||||
from bid_doctor.core.db import TenantContext
|
||||
from bid_doctor.document import service as documents
|
||||
from bid_doctor.project import service as projects
|
||||
from bid_doctor.project.models import Project
|
||||
|
||||
|
||||
async def rows(s: AsyncSession, ctx: TenantContext, limit: int) -> list[dict[str, Any]]:
|
||||
items = await projects.list_projects(s, ctx, limit)
|
||||
ids = [p.id for p, _ in items]
|
||||
docs = await documents.summaries(s, ids)
|
||||
pending = await checklist.pending_counts(s, ids)
|
||||
states = await checklist.statuses(s, ids)
|
||||
return [_row(p, lots, docs[p.id], pending.get(p.id, 0), states.get(p.id)) for p, lots in items]
|
||||
|
||||
|
||||
async def detail(s: AsyncSession, ctx: TenantContext, project_id: UUID) -> dict[str, Any]:
|
||||
p = await projects.get(s, ctx, project_id)
|
||||
lots = next((lots for q, lots in await projects.list_projects(s, ctx, 1000) if q.id == p.id), [])
|
||||
docs = (await documents.summaries(s, [p.id]))[p.id]
|
||||
pending = (await checklist.pending_counts(s, [p.id])).get(p.id, 0)
|
||||
state = (await checklist.statuses(s, [p.id])).get(p.id)
|
||||
return {
|
||||
**_row(p, lots, docs, pending, state),
|
||||
"legal_regime": p.legal_regime,
|
||||
"agency": p.agency,
|
||||
"procurement_method": p.procurement_method,
|
||||
"fields": [],
|
||||
"key_dates": [],
|
||||
"latest_check_run_id": p.latest_check_run_id,
|
||||
"created_by": {"id": p.created_by, "name": ctx.actor_name},
|
||||
"created_at": p.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _row(
|
||||
p: Project, lots: list[str], docs: dict[str, Any], pending: int, checklist_status: str | None
|
||||
) -> dict[str, Any]:
|
||||
ready = None if p.status != "checked" else p.open_fatal_count == 0
|
||||
return {
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"code": p.code,
|
||||
"purchaser": p.purchaser,
|
||||
"deadline_at": p.deadline_at,
|
||||
"status": p.status,
|
||||
"readiness": {"status": p.status, "open_fatal_count": p.open_fatal_count, "ready": ready},
|
||||
"bidding_lots": lots,
|
||||
"pending_item_count": pending,
|
||||
"active_job": docs["job"],
|
||||
"next_action": _next_action(p.status, docs["tenders"], docs["job"], checklist_status),
|
||||
"updated_at": p.updated_at,
|
||||
}
|
||||
|
||||
|
||||
def _next_action(status: str, tenders: int, job: Any, checklist_status: str | None) -> str:
|
||||
"""工作台「下一步」(api-design §6)。骨架阶段只覆盖解读这一段,查标阶段补齐。"""
|
||||
if job:
|
||||
return "wait_parse"
|
||||
if status == "tracking":
|
||||
return "upload_tender" if tenders == 0 else "wait_parse"
|
||||
if status == "drafting":
|
||||
# 清单确认之前先确认清单(原型工作台的「下一步」);待确认区有条目也算没确认完
|
||||
return "upload_bid" if checklist_status == "confirmed" else "confirm_checklist"
|
||||
return "none"
|
||||
33
server/src/bid_doctor/worker.py
Normal file
33
server/src/bid_doctor/worker.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""worker 进程:Procrastinate 领取任务,跑流水线(架构 §3.4)。
|
||||
|
||||
worker 直连数据库(Procrastinate 靠 LISTEN/NOTIFY 发现新任务,不能走事务模式的连接池)。
|
||||
收到 SIGTERM 后不再领新任务,等手头的任务跑完再退出(Procrastinate 自带)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
import procrastinate
|
||||
|
||||
from bid_doctor.core.config import get_settings
|
||||
|
||||
|
||||
def _conninfo() -> str:
|
||||
return get_settings().database_url.replace("postgresql+psycopg://", "postgresql://", 1)
|
||||
|
||||
|
||||
app = procrastinate.App(connector=procrastinate.PsycopgConnector(conninfo=_conninfo()))
|
||||
|
||||
|
||||
@app.task(name="parse_document", retry=False)
|
||||
async def parse_document(tenant_id: str, job_id: str) -> None:
|
||||
# 不让 Procrastinate 自动重试:流水线自己处理错误和退回额度,用户重试会建一条新任务(FR-M2-02)
|
||||
from bid_doctor.document.pipeline import runner
|
||||
|
||||
await runner.run(UUID(tenant_id), UUID(job_id))
|
||||
|
||||
|
||||
async def run_worker() -> None:
|
||||
async with app.open_async():
|
||||
await app.run_worker_async(concurrency=4, install_signal_handlers=True)
|
||||
Reference in New Issue
Block a user