@@ -15,6 +15,7 @@ from __future__ import annotations
import asyncio
import itertools
import re
from collections import Counter
from collections . abc import Awaitable , Callable
from dataclasses import dataclass , field
from datetime import datetime
@@ -29,6 +30,9 @@ from bid_doctor.document.pipeline import structure
from bid_doctor . document . pipeline . text_layer import Sentence , norm
CHUNK_CHARS = 6000
OVERLAP_CHARS = 800 # 相邻分块重叠的字数：跨块的条款在后一块里还能看到开头
OVERLAP_SENTENCES = 8
SPLIT_DEPTH = 3 # 模型输出被截断时把分块对半重跑，最多拆这么多层
CONCURRENCY = 8 # 每个解析任务同时调用模型的分块数； ×
INFO_CHUNKS = 3 # 项目基本信息只看文件开头几个分块（招标公告、须知前附表）
CN_TZ = ZoneInfo ( " Asia/Shanghai " )
@@ -54,6 +58,29 @@ CN_DIGIT = {"零": 0, "〇
CN_UNIT = { " 十 " : 10 , " 百 " : 100 , " 千 " : 1000 }
END_PUNCT = " 。；;， ！ ？ "
OUR_MARKUP = re . compile ( r " </?(?:iu|u|i)>|〔 〕 〕 " )
SID = re . compile ( r " ^p \ d+s \ d+$ " )
# 招标文件里待投标人填写的空白： × × （
PLACEHOLDER = re . compile ( r " _ { 2,}|＿ { 2,}|× { 2,}|[xX] { 2,}| \ * { 2,}|（ \ s*） \ ( \ s* \ )|待填|TBD " , re . IGNORECASE )
# 没通过校验的原因码（记在清单项 ai_values.failures 里，解析结果按原因计数）。每一条对应提示词里的一条规则，
# 见 prompts/README.md「提示词规则和服务端校验」
FAILURES = {
" evidence_missing " : " 没有给证据 " ,
" sid_format " : " 句子编号格式不对 " ,
" sid_unknown " : " 句子编号不在本分块里 " ,
" quote_too_short " : " 引文太短（不足 8 个字，短句没有整句复制） " ,
" quote_not_original " : " 引文不是这句话的原文 " ,
" basis_unmatched " : " 认定为实质性条款的依据在原文里找不到 " ,
" value_unverified " : " 取值在原文里找不到 " ,
" points_unverified " : " 分值在原文里找不到 " ,
" date_unverified " : " 日期时间在原文里找不到 " ,
" amount_unverified " : " 金额在原文里找不到 " ,
" placeholder_value " : " 取值是待填写的空白 " ,
" marker_uncovered " : " 标注了实质性条款，但模型没有抽到 " ,
" not_found_with_evidence " : " 说没找到却带了证据 " ,
" ambiguous " : " 模型认为有歧义 " ,
" missing_answer " : " 模型漏答了这一项 " ,
}
@dataclass ( slots = True )
@@ -72,6 +99,7 @@ class Extracted:
evidence_required : str | None = None
lot_no : str | None = None
ai_values : dict [ str , Any ] = field ( default_factory = dict )
failures : list [ str ] = field ( default_factory = list ) # 没通过校验的原因码（ ）
@dataclass ( slots = True )
@@ -92,6 +120,8 @@ class InfoField:
class ProjectInfo :
fields : dict [ str , InfoField ] = field ( default_factory = dict )
lots : list [ dict [ str , Any ] ] = field ( default_factory = list )
# 每个字段的结论： （ ） （ ） ，
status : dict [ str , dict [ str , Any ] ] = field ( default_factory = dict )
@dataclass ( slots = True )
@@ -184,7 +214,36 @@ def chunks(sentences: list[Sentence], continuations: dict[str, Sentence]) -> lis
prev = s
if cur :
out . append ( cur )
return out
return [ out [ 0 ] , * ( _tail ( a , continuations ) + b for a , b in itertools . pairwise ( out ) ) ] if out else out
def _tail ( chunk : list [ Sentence ] , continuations : dict [ str , Sentence ] ) - > list [ Sentence ] :
""" 上一块的结尾几句，放到下一块开头作重叠：跨块的条款（列表的前几条、表格的前几行）在后一块里也看得到。
重叠部分抽到的结果按首句合并，不会重复。跨页接续的两句不拆开。 """
out : list [ Sentence ] = [ ]
size = 0
for s in reversed ( chunk ) :
line = len ( s . marked ) + 12
if out and ( size + line > OVERLAP_CHARS or len ( out ) > = OVERLAP_SENTENCES ) :
break
out . insert ( 0 , s )
size + = line
while len ( out ) < len ( chunk ) and chunk [ len ( chunk ) - len ( out ) - 1 ] . key in continuations :
out . insert ( 0 , chunk [ len ( chunk ) - len ( out ) - 1 ] )
return out if len ( out ) < len ( chunk ) else [ ]
def is_placeholder ( value : Any ) - > bool :
return isinstance ( value , str ) and ( not value . strip ( ) or PLACEHOLDER . search ( value ) is not None )
def value_issue ( value : Any , sents : list [ Sentence ] , code : str = " value_unverified " ) - > str | None :
""" 结构化取值的问题：待填写的空白 → placeholder_value； ； """
if value in ( None , " " , [ ] ) :
return None
if is_placeholder ( str ( value ) ) :
return " placeholder_value "
return None if check_value ( value , sents ) else code
def _render ( chunk : list [ Sentence ] , doc : Doc , marks : bool = False ) - > str :
@@ -205,19 +264,21 @@ def _render(chunk: list[Sentence], doc: Doc, marks: bool = False) -> str:
def check_evidence (
ev : dict [ str , Any ] , by_id : dict [ str , Sentence ] , nxt : dict [ str , Sentence ] | None = None
) - > tuple [ str | None , list [ Sentence ] ] :
""" 第一、二道：编号在本分块里；引文是原句的子串，且不少于 8 个字。返回 (没通过的那一道 , 引文所在的句子)。
""" 第一、二道：编号在本分块里；引文是原句的子串，且不少于 8 个字。返回 (没通过的原因码 , 引文所在的句子)。
比对前去掉我们自己加进输入的格式标记和〔…〕附注（不是原文，模型偶尔会照抄）；原文文字本身不做任何放宽。
表格单元格、跨页会把一句话切成两段（ ） ，
下一句（同一分块里的下一句，或跨页接续句），但必须从这一句开始，两句都作为出处。
"""
sid , quote = str ( ev . get ( " sid " , " " ) ) , norm ( OUR_MARKUP . sub ( " " , str ( ev . get ( " quote " , " " ) ) ) )
sid , quote = str ( ev . get ( " sid " , " " ) ) . strip ( ) , norm ( OUR_MARKUP . sub ( " " , str ( ev . get ( " quote " , " " ) ) ) )
if not SID . match ( sid ) :
return " sid_format " , [ ]
if sid not in by_id :
return " id " , [ ]
return " s id_unknown " , [ ]
s = by_id [ sid ]
text = norm ( s . text )
if not quote or len ( quote ) < min ( 8 , len ( text . rstrip ( END_PUNCT ) ) ) : # 不足 8 个字的短句要整句复制，句末标点不算
return " quote " , [ ]
return " quote_too_short " if quote and quote in text else " quote_not_original " , [ ]
if quote in text :
return None , [ s ]
following = ( nxt or { } ) . get ( s . key )
@@ -226,22 +287,25 @@ def check_evidence(
at = joined . find ( quote )
if 0 < = at < len ( text ) and at + len ( quote ) > len ( text ) :
return None , [ s , following ]
return " quote " , [ ]
return " quote_not_original " , [ ]
def _evidence (
it : dict [ str , Any ] , by_id : dict [ str , Sentence ] , nxt : dict [ str , Sentence ] | None = None
) - > tuple [ list [ Sentence ] , list [ str ] , bool ] :
""" 返回 (通过前两道的句子, 通过了哪几道, 是否全部证据都通过)。 """
) - > tuple [ list [ Sentence ] , list [ str ] , bool , list [ str ] :
""" 返回 (通过前两道的句子, 通过了哪几道, 是否全部证据都通过, 没通过的原因码 )。 """
evs = [ e for e in it . get ( " evidence " ) or [ ] if isinstance ( e , dict ) ]
if not evs :
return [ ] , [ ] , False , [ " evidence_missing " ]
results = [ check_evidence ( e , by_id , nxt ) for e in evs ]
ok : list [ Sentence ] = [ ]
for fail , sents in results :
if fail is None :
ok + = [ x for x in sents if x not in ok ]
fails = [ f for f , _ in results ]
pass ed= [ c for c in ( " id" , " quote " ) if ok and c not in fails ]
return ok , passed , bool ( ok ) and all ( f is None for f in fails )
fails = [ f for f , _ in results if f ]
id_fail ed= any ( f . startswith ( " s id_ " ) for f in fails )
passed = [ c for c , bad in ( ( " id " , id_failed ) , ( " quote " , bool ( fails ) ) ) if ok and not bad ]
return ok , passed , bool ( ok ) and not fails , [ * dict . fromkeys ( fails ) ]
def _next_map ( chunk : list [ Sentence ] , doc : Doc ) - > dict [ str , Sentence ] :
@@ -385,8 +449,10 @@ def _item(
value_ok : bool ,
raw : dict [ str , Any ] ,
doc : Doc ,
value_fail : str | None = None ,
* * kw : Any ,
) - > Extracted :
""" value_fail： ； """
if value_ok and sents :
passed = [ * passed , " basis " ]
accepted = ok and value_ok
@@ -408,6 +474,7 @@ def _item(
importance = importance ,
importance_basis = imp_basis ,
ai_values = { task : raw } ,
failures = [ ] if value_ok or not sents else [ value_fail or " value_unverified " ] ,
* * kw ,
)
@@ -426,6 +493,7 @@ def _substantive(
bool ( sents ) and check_basis ( basis , sents , chunk , doc ) ,
it ,
doc ,
" basis_unmatched " ,
basis = basis ,
spec = { " invalidity " : { " source " : " substantive_marker " if basis != " explicit_invalid " else " tender_text " } } ,
)
@@ -463,6 +531,7 @@ def _scoring(
points = it . get ( " points " )
unit = " " if not points or str ( points ) . endswith ( ( " % " , " 分 " ) ) else " 分 " # 价格扣除是百分比，不加「分」
title = f " { it . get ( ' factor ' ) or ' 评分项 ' } " + ( f " （ { points } { unit } ） " if points else " " )
issue = value_issue ( points , ctx , " points_unverified " )
return _item (
" scoring " ,
[ " scoring " ] ,
@@ -470,9 +539,10 @@ def _scoring(
sents ,
passed ,
ok ,
check_value ( points , ctx ) ,
issue is None ,
it ,
doc ,
issue ,
spec = {
" scoring " : {
" factor " : it . get ( " factor " ) ,
@@ -499,6 +569,7 @@ def _schedule(
else None
)
spec = { " date_kind " : kind , " at " : at , " deposit_amount " : deposit , " submission_mode " : mode }
issue = " date_unverified " if not at_ok else value_issue ( deposit , ctx , " amount_unverified " )
return _item (
" schedule " ,
[ " schedule " ] ,
@@ -506,9 +577,10 @@ def _schedule(
sents ,
passed ,
ok ,
at_ok and check_value ( deposit , ctx ) ,
issue is None ,
it ,
doc ,
issue ,
spec = { " schedule " : { k : v for k , v in spec . items ( ) if v is not None } } ,
)
@@ -533,22 +605,23 @@ def _tech_param(
sents ,
passed ,
ok ,
check_val ue( it . get ( " value " ) , ctx ) ,
( issue := value_iss ue( it . get ( " value " ) , ctx ) ) is None
it ,
doc ,
issue ,
spec = { " tech_param " : { k : v for k , v in spec . items ( ) if v is not None } } ,
lot_no = it . get ( " lot_no " ) ,
)
TASKS = [
Task ( " extract.substantive " , 3 , _substantive , marks = True ) ,
Task ( " extract.qualification " , 2 , _qualification ) ,
Task ( " extract.scoring " , 2 , _scoring ) ,
Task ( " extract.schedule " , 2 , _schedule ) ,
Task ( " extract.tech_param " , 2 , _tech_param ) ,
Task ( " extract.substantive " , 4 , _substantive , marks = True ) ,
Task ( " extract.qualification " , 3 , _qualification ) ,
Task ( " extract.scoring " , 3 , _scoring ) ,
Task ( " extract.schedule " , 3 , _schedule ) ,
Task ( " extract.tech_param " , 3 , _tech_param ) ,
]
INFO_TASK = ( " extract.project_info " , 1 )
INFO_TASK = ( " extract.project_info " , 2 )
INFO_KEYS = (
" name " ,
" code " ,
@@ -573,54 +646,84 @@ def prompt_versions() -> dict[str, str]:
Progress = Callable [ [ float ] , Awaitable [ None ] ]
Parts = list [ tuple [ list [ Sentence ] , dict [ str , Any ] ] ] # 一次调用（可能被对半拆开）的 (分块, 模型输出) 列表
async def run (
ctx : TenantContext , job_id : UUID , doc : Doc , progress : Progress | None = None
) - > tuple [ list [ Extracted ] , ProjectInfo ] :
) - > tuple [ list [ Extracted ] , ProjectInfo , dict [ str , Any ] :
""" 返回 (清单项, 项目基本信息, 统计)。统计里有对半重跑的次数、拆到底仍不完整的分块数和各原因码的条数。 """
parts = chunks ( doc . sentences , doc . continuations )
gate = asyncio . Semaphore ( CONCURRENCY )
total = len ( parts ) * len ( TASKS ) + min ( INFO_CHUNKS , len ( parts ) )
done = 0
stats : Counter [ str ] = Counter ( )
async def call ( task : str , version : int , chunk : list [ Sentence ] , marks : bool = False ) - > dict [ str , Any ] :
nonlocal done
async with gate :
data = await ai . run_json (
ctx ,
job_type = " parse_job " ,
job_id = job_id ,
task = task ,
prompt _id= task ,
version = version ,
user = _render ( chunk , doc , marks ) ,
async def ask ( task : str , version : int , chunk : list [ Sentence ] , marks : bool , depth : int = 0 ) - > Parts :
""" 一个分块调用一次；输出被截断或不是合法 JSON 时对半拆开重跑（参考项目 BidEngine 的做法），
拆到底还不完整就记下来，不能当成「这一块没有结果」。 """
try :
async with gate :
data = await ai . run_json (
ctx ,
job_type = " parse_job " ,
job _id = job_id ,
task = task ,
prompt_id = task ,
version = version ,
user = _render ( chunk , doc , marks ) ,
)
return [ ( chunk , data ) ]
except ai . OutputIncomplete :
if len ( chunk ) < 2 or depth > = SPLIT_DEPTH :
stats [ " output_incomplete " ] + = 1
return [ ( chunk , { } ) ]
stats [ " chunks_split " ] + = 1
mid = len ( chunk ) / / 2
while mid < len ( chunk ) - 1 and chunk [ mid - 1 ] . key in doc . continuations :
mid + = 1 # 跨页接续的两句不拆开
a , b = await asyncio . gather (
ask ( task , version , chunk [ : mid ] , marks , depth + 1 ) , ask ( task , version , chunk [ mid : ] , marks , depth + 1 )
)
return a + b
async def call ( task : str , version : int , chunk : list [ Sentence ] , marks : bool = False ) - > Parts :
nonlocal done
got = await ask ( task , version , chunk , marks )
done + = 1
if progress :
await progress ( done / total )
return data
return got
jobs = [ ( t , c , call ( t . name , t . prompt_version , c , t . marks ) ) for t in TASKS for c in parts ]
info_jobs = [ ( c , call ( * INFO_TASK , c ) ) for c in parts [ : INFO_CHUNKS ] ]
results = await asyncio . gather ( * ( j for _ , _ , j in jobs ) , * ( j for _ , j in info_jobs ) )
jobs = [ ( t , call ( t . name , t . prompt_version , c , t . marks ) ) for t in TASKS for c in parts ]
info_jobs = [ call ( * INFO_TASK , c ) for c in parts [ : INFO_CHUNKS ] ]
results = await asyncio . gather ( * ( j for _ , j in jobs ) , * info_jobs )
out : list [ Extracted ] = [ ]
for ( task , chunk , _ ) , data in zip ( jobs , results [ : len ( jobs ) ] , strict = True ) :
by_id = { _sid ( s ) : s for s in chunk }
nxt = _next_map ( chunk , doc )
for it in data . get ( " items " ) or [ ] :
if not isinstance ( it , dict ) :
continue
sents , passed , ok = _evidence ( it , by_id , nxt )
sents = without_headings ( sents , doc )
if sents and all ( s . key in doc . note_keys or is_marker_talk ( s ) for s in sents ) :
continue # 只是「标注规则说明」或表头，不是条款（提示词要求不输出，这里再兜一道）
if ( e := task . build ( it , sents , passed , ok , chunk , doc ) ) is not None :
out + = split_listed ( e , doc )
info = _project_info ( [ ( c , d ) for ( c , _ ) , d in zip ( info_jobs , results [ len ( jobs ) : ] , strict = True ) ] )
for ( task , _ ) , got in zip ( jobs , results [ : len ( jobs ) ] , strict = True ) :
for chunk , data in got :
by_id = { _sid ( s ) : s for s in chunk }
nxt = _next_map ( chunk , doc )
for it in data . get ( " items " ) or [ ] :
if not isinstance ( it , dict ) :
continue
sents , passed , ok , fails = _evidence ( it , by_id , nxt )
sents = without_headings ( sents , doc )
if sents and all ( s . key in doc . note_keys or is_marker_talk ( s ) for s in sents ) :
continue # 只是「标注规则说明」或表头，不是条款（提示词要求不输出，这里再兜一道）
if ( e := task . build ( it , sents , passed , ok , chunk , doc ) ) is not None :
e . failures = [ * dict . fromkeys ( [ * fails , * e . failures ] ) ] if e . zone == " pending " else [ ]
out + = split_listed ( e , doc )
info = _project_info ( [ pair for got in results [ len ( jobs ) : ] for pair in got ] )
items = merge_items ( out )
items + = fallback ( items , doc )
retur nitems , info
for e i nitems :
if e . failures :
e . ai_values = { * * e . ai_values , " failures " : e . failures }
stats . update ( f " fail: { f } " for f in e . failures )
for f in info . status . values ( ) :
stats [ f " info: { f [ ' status ' ] } " ] + = 1
return items , info , dict ( stats )
def without_headings ( sents : list [ Sentence ] , doc : Doc ) - > list [ Sentence ] :
@@ -657,6 +760,7 @@ def split_listed(e: Extracted, doc: Doc) -> list[Extracted]:
sentences = sents ,
zone = " formal " if ok else " pending " ,
pending_reason = None if ok else " verify_failed " ,
failures = [ ] if ok else ( e . failures or [ " basis_unmatched " ] ) ,
checks_passed = e . checks_passed ,
importance = " substantive " ,
importance_basis = " marker " if basis != " explicit_invalid " else " tender_text " ,
@@ -693,13 +797,22 @@ def merge_items(items: list[Extracted]) -> list[Extracted]:
head . ai_values = { * * e . ai_values , * * head . ai_values }
head . evidence_required = head . evidence_required or e . evidence_required
head . basis = head . basis or e . basis
if head . zone == " pending " :
head . failures = [ * dict . fromkeys ( [ * head . failures , * e . failures ] ) ]
head . checks_passed = [ * dict . fromkeys ( [ * head . checks_passed , * e . checks_passed ] ) ]
keys = { s . key for s in head . sentences }
head . sentences = sorted (
[ * head . sentences , * ( s for s in e . sentences if s . key not in keys ) ] , key = lambda s : ( s . page_no , s . seq )
)
merged . append ( head )
return merged + loose
# 没有出处的结果没法按首句对齐：同一任务、同一概括的只留一条（相邻分块重叠时会抽到两次）
seen : set [ tuple [ str , str ] ] = set ( )
unique = [ ]
for e in loose :
if ( e . task , e . summary ) not in seen :
seen . add ( ( e . task , e . summary ) )
unique . append ( e )
return merged + unique
def fallback ( items : list [ Extracted ] , doc : Doc ) - > list [ Extracted ] :
@@ -737,6 +850,7 @@ def fallback(items: list[Extracted], doc: Doc) -> list[Extracted]:
sentences = sents ,
zone = " pending " ,
pending_reason = " marker_uncovered " ,
failures = [ " marker_uncovered " ] ,
checks_passed = [ " id " , " quote " , " basis " ] ,
importance = " substantive " ,
importance_basis = " marker " if marker != " explicit_invalid " else " tender_text " ,
@@ -747,33 +861,69 @@ def fallback(items: list[Extracted], doc: Doc) -> list[Extracted]:
return out
def _project_info ( results : list [ tuple [ list [ Sentence ] , dict [ str , Any ] ] ] ) - > ProjectInfo :
""" 项目基本信息：每个字段取第一个通过校验的值（文件开头的公告最权威）。 """
def _project_info ( results : Parts ) - > ProjectInfo :
""" 项目基本信息（参考 BidEngine 的固定字段车道）：每个字段在每个分块里都要表态
found / ambiguous / not_found； （ ）
最后没有 found 的字段分两种：看过的分块都干净地说 not_found → absent（ ， ） ；
有歧义、漏答、说没找到却带证据、证据或取值没通过 → needs_review（ ， ） ， """
info = ProjectInfo ( )
codes : dict [ str , list [ str ] ] = { k : [ ] for k in INFO_KEYS }
for chunk , data in results :
by_id = { _sid ( s ) : s for s in chunk }
fields = data . get ( " fields " ) or { }
fields = data . get ( " fields " )
if not isinstance ( fields , dict ) :
for key in INFO_KEYS :
codes [ key ] . append ( " missing_answer " )
continue
for key in INFO_KEYS :
= fields . get ( key )
if key in info . fields or not isinstance ( f , dict ) or f . get ( " value " ) in ( None , " " ) :
i fkey in info . fields :
continue
sents , _ , ok = _evidence ( { " evidence " : [ { " sid " : f . get ( " sid " ) , " quote " : f . get ( " quote " ) } ] } , by_id )
value = f [ " value " ]
if key not in fields :
codes [ key ] . append ( " missing_answer " )
continue
f = fields [ key ]
status = f . get ( " status " ) if isinstance ( f , dict ) else None
if not isinstance ( f , dict ) or status == " not_found " or ( status is None and f . get ( " value " ) in ( None , " " ) ) :
if isinstance ( f , dict ) and ( f . get ( " sid " ) or f . get ( " quote " ) ) :
codes [ key ] . append ( " not_found_with_evidence " )
continue
if status == " ambiguous " :
codes [ key ] . append ( " ambiguous " )
continue
value = f . get ( " value " )
if value in ( None , " " ) :
codes [ key ] . append ( " missing_answer " )
continue
if is_placeholder ( value ) :
codes [ key ] . append ( " placeholder_value " )
continue
sents , _ , ok , fails = _evidence ( { " evidence " : [ { " sid " : f . get ( " sid " ) , " quote " : f . get ( " quote " ) } ] } , by_id )
if not ok :
codes [ key ] + = fails
continue
if isinstance ( value , bool ) :
if not check_bool ( key , value , sents ) :
codes [ key ] . append ( " value_unverified " )
continue
elif key != " legal_regime " and not check_value ( value , sents ) :
codes [ key ] . append ( " value_unverified " )
continue
info . fields [ key ] = InfoField ( value = value , unit = f . get ( " unit " ) , sentence = sents [ 0 ] )
if not info . lots :
for lot in data . get ( " lots " ) or [ ] :
if not isinstance ( lot , dict ) or not lot . get ( " lot_no " ) :
continue
sents , _ , ok = _evidence ( { " evidence " : [ { " sid " : lot . get ( " sid " ) , " quote " : lot . get ( " quote " ) } ] } , by_id )
ev = { " evidence " : [ { " sid " : lot . get ( " sid " ) , " quote " : lot . get ( " quote " ) } ] }
sents , _ , ok , _ = _evidence ( ev , by_id )
if ok :
info . lots . append ( { * * lot , " sentence " : sents [ 0 ] } )
for key in INFO_KEYS :
if key in info . fields :
info . status [ key ] = { " status " : " found " }
elif codes [ key ] :
info . status [ key ] = { " status " : " needs_review " , " codes " : [ * dict . fromkeys ( codes [ key ] ) ] }
else :
info . status [ key ] = { " status " : " absent " }
return info
YANG JIANKUAN