# -*- coding: utf-8 -*- """表格要素 / 候选人 / 表头识别提取器。 按 ARCHITECTURE.md Phase 5 拆分建议，从 ``interface/predictor.py`` 迁出以下 表格相关类与函数： - ``TableTag2List`` — soup table 转列表并补全 span（原 8033-8192 行） - ``is_head_line`` — 调用表头识别模型判断是否为表头行（原 8195-8211 行） - ``TablePremExtractor`` — 表格要素（标段/招标人/中标人/金额）提取（原 8213-8754 行） - ``CandidateExtractor`` — 表格候选人提取（原 8756-9183 行） - ``get_header_line`` — 判断列表内文本哪些是表头（原 9298-9321 行） 原 ``from common.Utils import *`` / ``from interface.modelFactory import *`` 已替换为显式 import；``os.path.dirname(__file__)`` 路径引用替换为 ``predictors._common.INTERFACE_DIR``。 ``interface/predictor.py`` 仍保留原定义，老 import 不受影响。 """ from __future__ import absolute_import import re import pickle import numpy as np import pandas as pd from bs4 import BeautifulSoup from BiddingKG.dl.common.logging import log from BiddingKG.dl.common.context_utils import ( find_package, uniform_package_name, money_process, cut_repeat_name, ) from BiddingKG.dl.common.Utils import del_tabel_achievement from BiddingKG.dl.common.nerUtils import getNers from BiddingKG.dl.foolnltk import selffool from BiddingKG.dl.time.re_servicetime import extract_servicetime from BiddingKG.dl.common.attr_utils import extract_serviceTime from BiddingKG.dl.interface.predictor import getPredictor from BiddingKG.dl.predictors._common import ( INTERFACE_DIR, header_set, get_td_companys, get_role, ) __all__ = [ "TableTag2List", "is_head_line", "TablePremExtractor", "CandidateExtractor", "get_header_line", ] class TableTag2List(): '''把soup table 转化为表格补全后的文本列表[[td, td, td], [td, td, td]]''' def table2list(self, table, text_process=None, return_html_table=False,return_kv=False): ''' 表格补全及把表格内容列表返回 :param table: :param text_process: 预处理方法，segment(),不为None 时把td内容做预处理，结果返回加标签，适配表头识别 [[[text, 0], [text, 0]] ], 否则只返回文本[[text, text], [text, text]] :param return_html_table: :param return_kv: :return: ''' self._output = [] row_ind = 0 col_ind = 0 html_table = [] for row in table.find_all('tr'): # record the smallest row_span, so that we know how many rows # we should skip smallest_row_span = 1 if len(row.find_all(['td', 'th'], recursive=False)) > 20: log('未补全前表格列数大于20的不做表格处理') if return_html_table: return [], [] return [] for cell in row.children: if cell.name in ('td', 'th'): # check multiple rows # pdb.set_trace() row_span = int(re.sub('[^0-9]', '', cell.get('rowspan'))) if cell.get('rowspan') and cell.get('rowspan').isdigit() else 1 if row_span == 0: # 20250806 修复 659303627 附件OCR重构表格span为0导致缺少问题 row_span = 1 # try updating smallest_row_span smallest_row_span = min(smallest_row_span, row_span) # check multiple columns col_span = int(re.sub('[^0-9]', '', cell.get('colspan'))) if cell.get('colspan') and cell.get('colspan').isdigit() else 1 if col_span > 20: # 修复 335590254 山东港口阳光智采e平台 数据源表格第一行colspan为200超过50列造成无法提取问题 col_span = 20 elif col_span == 0: # 20250806 修复 659303627 附件OCR重构表格span为0导致缺少问题 col_span = 1 # find the right index while True: if self._check_cell_validity(row_ind, col_ind): break col_ind += 1 # insert into self._output try: if 'title' in cell.attrs and cell.get_text().replace(' ', '').endswith( '...') and cell.attrs['title'].replace(' ', '').startswith(cell.get_text().replace(' ', '')[:-3]): cell.string = cell.attrs['title'] # 修复 类似 215597851 215597851 省略号隐藏内容 if text_process != None: # text = [re.sub('\xa0', '', text_process(cell, final=False)), 0] # td_text = re.sub('\xa0', '', text_process(cell, final=False)) td_text = re.sub('\s|\xa0', '', str(cell.get_text())) # 修复 370835008 td 内公司被p标签拆分为两半情况 if len(td_text)>30: if return_kv: td_text = cell.get_text().strip() else: td_text = re.sub('\xa0', '', text_process(cell, final=False)) if td_text == "": td_text = ' ' text = [td_text,0] else: if return_kv: td_text = cell.get_text().strip() else: td_text = str(cell.get_text()).strip().replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '').replace(' ', '') td_text = re.sub('\s+', ' ', td_text) text = td_text # text = str(cell.get_text()).strip().replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '') # # text = re.sub('\s', '', text)[:200] # 只需取前200字即可 # text = ' ' if text == "" else text self._insert(row_ind, col_ind, row_span, col_span, text) if return_html_table: html_table = self._insert_new(row_ind, col_ind, row_span, col_span, str(cell), html_table) except UnicodeEncodeError: raise Exception( 'Failed to decode text; you might want to specify kwargs transformer=unicode' ) # update col_ind col_ind += col_span if col_ind > 50 and text_process == None: # 表格要素提取及候选人提取的 表格列数大于50的去掉 if return_html_table: return [], [] return [] # update row_ind row_ind += smallest_row_span col_ind = 0 if return_html_table: temp_list = [] for row in self._output: if len(row) > 0: temp_list.append(row) self._output = temp_list temp_list = [] for row in html_table: if len(row) > 0: temp_list.append(row) html_table = temp_list return self._output, html_table else: return self._output def _check_validity(self, i, j, height, width): """ check if a rectangle (i, j, height, width) can be put into self.output """ return all(self._check_cell_validity(ii, jj) for ii in range(i, i+height) for jj in range(j, j+width)) def _check_cell_validity(self, i, j): """ check if a cell (i, j) can be put into self._output """ if i >= len(self._output): return True if j >= len(self._output[i]): return True if self._output[i][j] == "": return True return False def _insert(self, i, j, height, width, val): # pdb.set_trace() for ii in range(i, i+height): for jj in range(j, j+width): self._insert_cell(ii, jj, val) def _insert_cell(self, i, j, val): while i >= len(self._output): self._output.append([]) while j >= len(self._output[i]): self._output[i].append("") if self._output[i][j] == "": self._output[i][j] = val def _insert_new(self, i, j, height, width, val, cell_list): # pdb.set_trace() for ii in range(i, i+height): for jj in range(j, j+width): cell_list = self._insert_cell_new(ii, jj, val, cell_list) return cell_list def _insert_cell_new(self, i, j, val, cell_list): while i >= len(cell_list): cell_list.append([]) while j >= len(cell_list[i]): cell_list[i].append("") if cell_list[i][j] == "": cell_list[i][j] = val return cell_list def is_head_line(list_item): ''' 调用表头识别模型判断是否为表头行 :param list_item: 行内容 例： ['技术参数、要求', '变更项'] :return: ''' x = [] for item in list_item: x.append(getPredictor("form").encode(item)) predict_y = getPredictor("form").predict(np.array(x), type="item") count = 0 for item, values in zip(list_item, list(predict_y)): if values[1] > 0.6: count += 1 if count/len(list_item)>0.6: return True return False class TablePremExtractor(object): def __init__(self): '''各要素表头规则''' self.head_rule_dic = { 'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|标段（包）|分[包标])(编号|编码|代码)", 'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$|^品目$", "project_name": "(包[段组件]|标[段包的项]|标段（包）|分[包标]|采购|项目|工程|^包)(名称?|内容)", # |货物|商品|产品|设备|通用|主要标的 20250812 货物的不作为项目名称 "win_sort": "排名|排序|名次|推荐顺序", 'win_or_not': '是否(建议|推荐)?(中标|成交|中选)|是否入围|是否入库|入围结论|未(中标|成交)原因|中标情况|^中标结果$', "tenderer": "(中标|中选|中价|中拍|成交|供货|承包|承建|承租|竞得|受让)）?(候选)?(人(单位)?|单位|供应商|公司|企业|厂家|商家?|客户|供?方|银行|回收商)(名称|$)|^(拟定|单一来源|邀请|拟?推荐(入选|入围)?)?供应商(名称)?$", "tenderee": "(项目|采购|招标|遴选|寻源|竞价|议价|比选|委托|询比?价|比价|评选|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选)(人|公司|单位|组织|用户|业主|主体|方|部门)(名称|$)", "budget": "最高(投标)?限价|总价限价|控制(价格?|金额|总价)|(总价|采购)限价|上限价|拦标价|(采购|招标|项目)?预算|(预算|招标|采购|计划)金额|挂牌价", "bid_amount": "投标[报总]?价|报价(总?金额|总价|总额)|总报价|^\w{,5}报价(（[\w、/]{1,15}）)?$|(中标|中选|中价|中拍|成交|合同)）?总?(金?额|[报均总]价|价[格款]?)|承包价|含税价|经评审的价格|中标存款金?额|中标资金|中标存款|存放金额|分配额度", "serviceTime": '合同期限|工期/交货期/服务期|工期\(交货期\)|合格工期|服务期限|工期' \ '|工期要求|项目周期|工期\(交货期\)|计划工期\(服务期限\)|服务时限|履行期限|服务周期|供货期限' \ '|合格工期|计划工期\(服务期\)|服务期|服务，期|交货\(完工\)时间|交付\(服务、完工\)时间' \ '|交货时间|保洁期限|维保期|管理年限|工期承诺|(服务|合同|施工|实施|工程|设计)的?(年限|期限|周期|期：)' \ '|计划工期|工期要求|服务期限?' \ '|投标工期|设计工期|合格服务周期|总工期|服务时间(范围)?|流转期限|维护期限|服务时限|交货期' \ '|完成时间|中标工期|项目周期|期限要求|周期|供货期|合同的?履行日期|计划周期' \ '|履约期限|合同的?约定完成时限|合同的?完成日期|承诺完成日期' \ '|合同起始日起|合同的?履约期|履约截止日期|承包期限|合同的?完成日期|特许经营期限' \ '|服务期间|服务履行期|委托(管理)?期限|经营期限|数量' \ '|(工期|服务期限?|交货期限?|服务履行期|合同期限?|履[行约]期限?)说明|存款期限?|(存款|存放|定存)(期|年)限' \ '|服务(有效期|年限)|本?合同有效期|协议有效期|项目期限' } with open(INTERFACE_DIR+'/header_set.pkl', 'rb') as f: self.headerset = pickle.load(f) self.tb = TableTag2List() def find_header(self, td_list, all_winner=False, first_line=False): fix_td_list = [re.sub('[:：]$|^[一二三四五六七八九十0-9]{1,3}、|(（[\w、×*/]{1,20}）)$|（不?含税）|/万?元|拟|\s', '', it) for it in td_list] # 去除表头无关信息，方便匹配判断是否为表头 header_dic = dict() flag = False contain_header = False not_sure_winner = False # 是否 不确定中标的中标人表达方式 for text in set(fix_td_list) - self.headerset: if len(text)<10 and re.search(self.head_rule_dic['bid_amount'], text): self.headerset.add(text) if len(set(fix_td_list))>0 and (first_line or len(set(fix_td_list) & self.headerset)>=2) and (len(set(fix_td_list) & self.headerset)/len(set(fix_td_list))>=0.6 or is_head_line(fix_td_list)): other_tenderer = "" other_tenderer2 = "" flag = True for i in range(len(td_list)) : text = td_list[i] text = re.sub('\s|[（(]排名不分先后[)）]', '', text) text = re.sub('排名价', '', text) # 20241225 修复 252208201 排名价（元）错误为排名 text = re.sub('^人选', '入选', text) if text == '备选中标人': text = '第二候选人' if len(re.sub('(（[\w、×*/]{1,20}）)$', '', text)) > 15: # 长度大于15 不进行表头匹配 continue if re.search('未(中标|成交|中选|入围|通过)原因', text): # 不提取此种表格 673143737 return flag, contain_header, dict(), not_sure_winner num = 0 for k, v in self.head_rule_dic.items(): if re.search('评分|得分|分数|分值', text): continue if re.search(v, text): if k in ['tenderer'] and re.search('是否|未', text): # 修复103367444 未中标单位名称 作为中标 continue if k == 'budget' and re.search('量', text): # 预算工作量 预算采购量 等不作为预算 continue elif k == 'bid_amount' and re.search('分配方案|基准利率|BP值', text): # 517987084 中标资金分配方案 continue elif k in header_dic: if k in ['budget', 'bid_amount'] and re.search('总(价|金?额)', text): # 总价替换单价 header_dic[k] = (i, text) num += 1 elif k == 'project_code' and text != header_dic[k][1] and 'package_code' not in header_dic\ and re.search(self.head_rule_dic['package_code'], re.sub('\s', '', '，'.join(td_list)))==None: # 如果出现两次项目编号且没有包号，把第二次出现的作为包号 例：472537470 header_dic['package_code'] = (i, text) continue header_dic[k] = (i, text) num += 1 elif re.search('^((中标|成交|中选|入围|入选)(候选)?)(人|单位|机构|中介(服务)?机构|供应商|客户|方|公司|厂商|商家?|社会资本方?|银行)(名称)?$', text) and re.search('未', text)==None: other_tenderer = (i, text) elif re.search('^((投标|应答|响应|候选)(人|单位|机构|中介(服务)?机构|供应商|客户|方|公司|厂商|商家?|社会资本方?|银行)|(存款|投标)?银行|供应商)(名称)?$|^机构名称$|^单位(名称)?$', text) and re.search('未', text)==None: other_tenderer2 = (i, text) if num>1: if re.search(self.head_rule_dic['project_code'], text) and re.search(self.head_rule_dic['package_code'], text): # 修复 528486798 分标编号-包号 continue # print('表头错误，一个td匹配到两个表头：', header_dic) return flag, contain_header, dict(), not_sure_winner if text == '单位': # 20241128 补充金额单位 header_dic['amount_unit'] = (i, text) if re.search('；金额(（万?元）)?；', '；'.join(td_list)): # 召回某些表格只写 金额 作为表头，不能识别为招标或中标金额 if 'tenderer' in header_dic and 'bid_amount' not in header_dic: for i in range(len(td_list)): text = td_list[i] if re.search('^金额(（万?元）)?$',text): header_dic['bid_amount'] = (i, text) break elif 'tenderee' in header_dic and 'budget' not in header_dic: for i in range(len(td_list)): text = td_list[i] if re.search('^金额(（万?元）)?$', text): header_dic['budget'] = (i, text) break if all_winner == 1 and 'tenderer' not in header_dic: # 标题有存款、入库、入围等公告补充其他表达做中标人 if other_tenderer!="": header_dic['tenderer'] = other_tenderer elif other_tenderer2!="": header_dic['tenderer'] = other_tenderer2 if 'win_sort' not in header_dic: not_sure_winner = True elif 'tenderer' not in header_dic and 'win_or_not' in header_dic: if other_tenderer!="": header_dic['tenderer'] = other_tenderer # elif other_tenderer2!="": # 20260115 注释 723429011 单位 是否中标 不是中标人 # header_dic['tenderer'] = other_tenderer2 if all_winner == 1 and 'win_sort' in header_dic: # 标题有存管类公告不分排名 header_dic.pop('win_sort') if 'tenderer' in header_dic and re.search('候选|入围|入选', header_dic['tenderer'][1]) and re.search('推荐的?((中标|成交|中选)候选人|(候选|入围|入选)供应商)', header_dic['tenderer'][1])==None and all_winner == False: header_dic.pop('tenderer') if ('project_code' in header_dic or 'package_code' in header_dic or 'project_name' in header_dic) and ( 'tenderer' in header_dic): # 包含标段及招标金额或中标人的进行提取 # or'budget' in header_dic 20250919 改为有中标人的才提取，只有预算的容易与非表格提取重复 例子：647543768 return flag, contain_header, header_dic, not_sure_winner elif ('tenderer' in header_dic) and ('bid_amount' in header_dic): # 包含中标人及中标金额的进行提取 if 'win_sort' in header_dic: # 有排名的 用候选人提取类 return flag, contain_header, dict(), not_sure_winner elif re.search('^(候选)?供应商(名称)?', header_dic['tenderer'][1]) and 'win_or_not' not in header_dic and re.search('(中标|成交|合同)）?总?(金?额|[报均总]价|价[格款]?)', header_dic['bid_amount'][1])==None: # 只有供应商名称 没排名和包号的去掉，预防错误包提取 334205629 # print('只有供应商名称 没排名和包号的去掉') return flag, contain_header, dict(), not_sure_winner return flag,contain_header, header_dic, not_sure_winner elif 'tenderer' in header_dic and (re.search('(中标|中选|中价|成交|竞得)(人|单位|供应商|公司|企业|厂家|商家?|客户|供?方|银行)',header_dic['tenderer'][1]) or all_winner): # 有中标人，且有明确中标关键词的进行提取 return flag, contain_header, header_dic, not_sure_winner # elif 'tenderer' in header_dic and 'serviceTime' in header_dic: # return flag, contain_header, header_dic, not_sure_winner elif len(set(fix_td_list) & self.headerset) >= 2 or (len(set(fix_td_list)) == 2 and len(set(td_list) & self.headerset) >= 1): # 如果包含两个表头以上或 只有两列且包含一个表头 contain_header = True return flag, contain_header, dict(), not_sure_winner def extract_from_df(self, df, headers, web_source_name, all_winner=False): prem_dic = {} previous_package = "" # 上一行包号 previous_project_name = "" # 上一行项目名称 multi_same_package = False # 非连续的重复包号 package_fix2raw = dict() # 处理后包号：处理前包号 字典 link_set = set() tenderer_list = [] # 保存所有中标人 serviceTime_list = [] not_package = True if 'project_name' in headers and re.search('(货物|商品|产品|通用|主要标的)(名称?|内容)', headers['project_name'][1]) and \ 'package_code' not in headers and 'budget' not in headers and "bid_amount" not in headers else False if set(['project_code', 'package_code', 'tenderee', 'tenderer']) & set(headers) == set() and ('project_name' not in headers # 补充没有项目名称或有项目名称且是货物的才过滤掉 or re.search('(货物|商品|产品|设备|通用|主要标的)(名称?|内容)', headers['project_name'][1])): # 20240131修复只有货物名称及最高限价的错误作为多包 396636683； 补充避免423647863采购意向被过滤 # print('没有包号及角色的不要') return {} have_bid_amount = False # 是否包含中标金额 if "bid_amount" in headers and re.search('[1-9]+', '#'.join([it.strip() for it in df[headers['bid_amount'][0]]])): have_bid_amount = True for i in df.index: same_package = False # 连续重复包号，一般是 rowspan 造成；一包 多个采购 project_code = df.loc[i, headers['project_code'][0]].strip().replace(' ', '') if "project_code" in headers else "" package_code_raw = df.loc[i, headers['package_code'][0]].strip().replace(' ', '') if "package_code" in headers else "" project_name = df.loc[i, headers['project_name'][0]].strip().replace(' ', '') if "project_name" in headers else "" tenderee = df.loc[i, headers['tenderee'][0]].strip().replace(' ', '') if "tenderee" in headers else "" tenderer = df.loc[i, headers['tenderer'][0]].strip().replace(' ', '') if "tenderer" in headers else "" budget_ = df.loc[i, headers['budget'][0]].strip().replace(' ', '') if "budget" in headers else "" bid_amount_ = df.loc[i, headers['bid_amount'][0]].strip().replace(' ', '') if "bid_amount" in headers else "" win_sort = df.loc[i, headers['win_sort'][0]].strip().replace(' ', '') if "win_sort" in headers else "" win_or_not = df.loc[i, headers['win_or_not'][0]].strip().replace(' ', '') if "win_or_not" in headers else "" serviceTime = df.loc[i, headers['serviceTime'][0]].strip().replace(' ', '') if "serviceTime" in headers else "" amount_unit = df.loc[i, headers['amount_unit'][0]].strip().replace(' ', '') if "amount_unit" in headers else "" joint = '' if set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_]) & self.headerset != set(): # 只要有一项为表头 停止匹配 # print('只要有一项为表头 停止匹配', set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_,win_sort]) & self.headerset) break if len(set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_,win_sort])- set(['', ' '])) < 2 and tenderer=='': # 内容为空或全部一样 停止匹配 # print('内容为空或全部一样 停止匹配') break if re.search('详见', project_name): # 去除某些表达： 详见招标文件 project_name = "" if package_code_raw == "" and re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))$|^(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}$', project_name): package_code_raw = project_name project_name = "" package_code = package_code_raw if re.search('合计|总计', package_code+project_code+project_name): continue if package_code + project_code == previous_package: # 处理 208162730 一个包采购多种东西情况 if project_name == previous_project_name: same_package = True if previous_package!="": # 有包号或项目编号且跟上一行相同时，去除项目名称 project_name = '' if win_sort != "" and re.search('排名|排序|名次|推荐顺序', headers['win_sort'][1]): # 此类型表由 CandidateExtractor类提取 防止类似 328485591 作为多包 break if win_or_not != "" and (re.search('(建议|推荐)(中标|成交|中选)|是|^(中标|成交|中选)', win_or_not)==None or re.search('\w', win_or_not)==None): # 2024/04/2 修复 252208201 为空的不中标 continue elif 'win_or_not' in headers and win_or_not == '': # 2024/12/25 修复 334753545 中标情况为空的不中标 continue if "win_sort" in headers and win_sort == "": # '表头有是否中标，内容却空白的，过滤掉' continue if win_sort == "" and "tenderer" in headers and re.search('候选|入围|入选', headers['tenderer'][1]) and re.search('推荐的?((中标|成交|中选)候选人|(候选|入围|入选)供应商)', headers['tenderer'][1])==None and all_winner == False: tenderer = "" if tenderer in ['采购失败', '废标']: # 避免类似 353867205 这篇只提取到一个 continue # tenderee = tenderee if self.is_role(tenderee) else "" # tenderer = tenderer if self.is_role(tenderer) else "" package = uniform_package_name(package_code) if package_code else '自增1' # 没有包号的自动编号的修改为提取到多少个包，某些行未必中标 if package == '自增1' and project_name != '' and 'project_name' in headers and re.search('包[段组件]|标[段包的项]|标段（包）|分[包标]', headers['project_name'][1]): # 避免 266252226 采购项目名称 做多包多招标人 修复 661973750 标段名称为多包 package = "自增%s"%str(len(prem_dic) + 1) if previous_project_name != project_name else "自增%s"%str(len(prem_dic)) if project_name != "" and package.startswith('自增'): pk_l = find_package(project_name) if len(pk_l)==1: package = uniform_package_name(pk_l[0].group(0)) elif re.search('[一二三四五六七八九十]+标段：|标段[一二三四五六七八九十]+：', tenderer) and package.startswith('自增'): pk_l = find_package(tenderer) if len(pk_l) == 1: package = uniform_package_name(pk_l[0].group(0)) tenderee = get_role(tenderee, self.nlp_enterprise) if tenderee!="" else tenderee # tenderer = get_role(tenderer, self.nlp_enterprise) if tenderer!='' else tenderer if tenderer!='': leader, joint = get_td_companys(tenderer, self.nlp_enterprise) if leader == '': log('表格角色内容不在公司列表：%s，docid：%s' % (tenderer, self.docid)) tenderer = leader tenderee = cut_repeat_name(tenderee) tenderer = cut_repeat_name(tenderer) if len(set([project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_])) < 2: break if not_package: if (project_code, package_code, tenderee, tenderer, budget_, bid_amount_) in link_set: continue link_set.add((project_code, package_code, tenderee, tenderer, budget_, bid_amount_)) else: if (project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_) in link_set: continue link_set.add((project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_)) if project_code != "": uni_project_code= uniform_package_name(project_code) if uni_project_code != "" and uni_project_code!=package: if package.startswith('自增'): # 没有包号有项目编号的，直接用项目编号 package = uni_project_code else: # print('重组包号：', '%s_%s'%(uni_project_code, package)) package = '%s_%s'%(uni_project_code, package) # 同时包号项目编号及包号的，组合起来做包号 if package_code_raw!='': if multi_same_package == False and package not in package_fix2raw: # 如果处理后的标段号 已经在列表里面，采用原始标段号文本 package_fix2raw[package] = package_code_raw elif same_package == False: multi_same_package = True if multi_same_package: package = package_code_raw if package not in prem_dic or not same_package: prem_dic[package] = { 'code': '', 'name': '', 'roleList': [], 'tendereeMoney': 0, 'tendereeMoneyUnit': "" } prem_dic[package]['code'] = project_code prem_dic[package]['name'] = project_name if budget_ != "": if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税|(六个月|一年|\w{2,3})期加点\d+BP', '', budget_)) > 5: # 金额字段出现超过5个非金额字符，中断匹配 prem_dic.pop(package) break budget_header = headers['budget'][1] if 'budget' in headers else '' if amount_unit!='' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元', budget_+budget_header)==None : # 20241128 补充某些表格价格单位分开两列， 例：557953660 budget_ += amount_unit budget, money_unit = money_process(budget_, budget_header) if re.search('元[/每]', amount_unit) or re.search('单价', budget_header): unit_tendereeMoney = budget budget = 0 else: unit_tendereeMoney = 0 if (re.search('费率|[上下]浮率|[%％‰折]|优惠率|期加点\d+BP', budget_header + budget_) and budget < 100) or budget > 50000000000: # 如果是费率或大于500亿的金额改为0 budget = 0 if budget > 0: if same_package and prem_dic[package]['tendereeMoney'] != budget: # 处理 类似 136839070 一包多物品多预算 prem_dic[package]['tendereeMoney'] += budget else: prem_dic[package]['tendereeMoney'] = budget prem_dic[package]['tendereeMoneyUnit'] = money_unit if unit_tendereeMoney > 0: if 'unit_tendereeMoney' not in prem_dic[package]: prem_dic[package]['unit_tendereeMoney'] = 0 if same_package and prem_dic[package]['unit_tendereeMoney'] != unit_tendereeMoney: # 处理 类似 136839070 一包多物品多预算 prem_dic[package]['unit_tendereeMoney'] += unit_tendereeMoney else: prem_dic[package]['unit_tendereeMoney'] = unit_tendereeMoney if tenderee and not same_package: prem_dic[package]['roleList'].append({ "address": "", "linklist": [], "role_money": { "discount_ratio": "", "downward_floating_ratio": "", "floating_ratio": "", "money": 0, "money_unit": "" }, "role_name": "tenderee", "role_text": tenderee, "serviceTime": "" }) if tenderer: if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税|(六个月|一年|\w{2,3})期加点\d+BP', '', bid_amount_)) > 5: # 金额字段出现超过5个非金额字符，中断匹配 prem_dic.pop(package) break bid_amount_header = headers['bid_amount'][1] if bid_amount_ != "" else '' if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and bid_amount_!='' and re.search('元', bid_amount_ + bid_amount_header) == None: bid_amount_ += amount_unit bid_amount, money_unit = money_process(bid_amount_, headers['bid_amount'][1]) if bid_amount_ != "" and 'bid_amount' in headers else (0, '') if re.search('[%％‰折]|浮率|期加点\d+BP', bid_amount_) and 0 < bid_amount < 100: bid_amount = 0 if re.search('元[/每]', amount_unit) or re.search('单价', bid_amount_header): unit_price = bid_amount bid_amount = 0 else: unit_price = 0 if web_source_name == '河钢供应链管理平台' and 'bid_amount' in headers and re.search('[%％‰折]|浮率', bid_amount_) == None and bid_amount == 0: # 有中标金额字段却金额为0的过滤掉，防止类似 河钢供应链管理平台 站源错误，金额不为0的才算中标 if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0: # 只有项目编号和名称的包 丢弃 prem_dic.pop(package) continue elif 'bid_amount' in headers and re.search('[%％‰折]|浮率', bid_amount_) == None and have_bid_amount and bid_amount_ in ['/','','0','0.0']: # 如果不是所有行中标金额都为0，则把为0的做非中标 if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0: # 只有项目编号和名称的包 丢弃 prem_dic.pop(package) continue if (re.search('费率|下浮率|[%％‰折]|优惠率', bid_amount_header + bid_amount_) and bid_amount < 100) or bid_amount > 50000000000: # 如果是费率或大于500亿的金额改为0 bid_amount = 0 if serviceTime: serviceTime_text = headers['serviceTime'][1] + serviceTime if headers['serviceTime'][1][-1] in [':','：'] else headers['serviceTime'][1] + ':' + serviceTime # print('serviceTime_text',serviceTime_text) serviceTime = extract_servicetime(serviceTime_text) serviceTime.sort(key=lambda x:x.get('begin_index',0)) serviceTime = extract_serviceTime(serviceTime[0]['body'],"") if serviceTime else "" # print(serviceTime) if not same_package or len(prem_dic[package]['roleList'])==0: role_dic = { "address": "", "linklist": [], "role_money": { "discount_ratio": "", "downward_floating_ratio": "", "floating_ratio": "", "money": bid_amount, "money_unit": money_unit }, "role_name": "win_tenderer", "role_text": tenderer, "serviceTime": serviceTime } if joint != '': role_dic['win_tenderer_joint'] = joint if unit_price > 0: role_dic['role_money']['unit_price'] = unit_price prem_dic[package]['roleList'].append(role_dic) elif prem_dic[package]['roleList'] and prem_dic[package]['roleList'][-1].get('role_name', '')=='win_tenderer': if 'multi_winner' not in prem_dic[package]['roleList'][-1]: prem_dic[package]['roleList'][-1]['multi_winner'] = prem_dic[package]['roleList'][-1]['role_text'] prem_dic[package]['roleList'][-1]['multi_winner'] += ','+ tenderer elif tenderer not in prem_dic[package]['roleList'][-1]['multi_winner']: prem_dic[package]['roleList'][-1]['multi_winner'] += ','+ tenderer if bid_amount != 0 or unit_price > 0: # 有中标金额的才放进去 if 'other_winner_dic' not in prem_dic[package]['roleList'][-1]: prem_dic[package]['roleList'][-1]['other_winner_dic'] = [] prem_dic[package]['roleList'][-1]['other_winner_dic'].append({'role_text': tenderer, "money": bid_amount, "money_unit": money_unit, "serviceTime": serviceTime}) tenderer_list.append(tenderer) serviceTime_list.append(serviceTime) if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0: # 只有项目编号和名称的 丢弃 并不再继续往下匹配 prem_dic.pop(package) # break # 注释掉避免 400084571 某些包废标 中断匹配 if multi_same_package: # 预处理后包号重复的，使用原始包号 for k, v in package_fix2raw.items(): if k in prem_dic: prem_dic[v] = prem_dic.pop(k) previous_package = package_code + project_code previous_project_name = project_name if len(tenderer_list)>2 and len(set(tenderer_list))==1 and "package_code" not in headers: # 没提取到包号且中标人一样应该是错误多包，需去掉多包 例 244355092 281854766 total_money = 0 for v in prem_dic.values(): for d in v['roleList']: if d['role_name'] == "win_tenderer": total_money += d['role_money']['money'] if 'other_winner_dic' in d: for other in d['other_winner_dic']: total_money += other.get('money', 0) return {'自增1': { 'code': '', 'name': '', 'roleList': [{ "address": "", "linklist": [], "role_money": { "discount_ratio": "", "downward_floating_ratio": "", "floating_ratio": "", "money": total_money, "money_unit": '' }, "role_name": "win_tenderer", "role_text": tenderer_list[0], "serviceTime": serviceTime_list[0] }], 'tendereeMoney': 0, 'tendereeMoneyUnit': "" }} return prem_dic def update_prem(self, rs_dic, tmp_dic): ''' 合并更新 prem :param rs_dic: 返回结果 :param tmp_dic: 待合并结果 :return: ''' if '自增1' in tmp_dic and '自增1' not in rs_dic and len(tmp_dic)==len(rs_dic): pass else: for pack in tmp_dic: if pack in rs_dic: for k in tmp_dic[pack]: if rs_dic[pack][k] in ['', 0]: rs_dic[pack][k] = tmp_dic[pack][k] elif rs_dic[pack][k] == []: rs_dic[pack][k] = tmp_dic[pack][k] elif k == 'roleList' and len(rs_dic[pack][k])>0 and rs_dic[pack][k][0].get('role_money', {}).get('money', 0) == 0: rs_dic[pack][k] = tmp_dic[pack][k] else: rs_dic[pack] = tmp_dic[pack] def get_prem(self, soup, web_source_name='', all_winner=False): tables = soup.find_all('table') tables.reverse() rs_dic = {} for table in tables: text = table.text.strip() pre_text = "" previous = None if table.findPreviousSibling() != None: previous = table.findPreviousSibling() pre_text = previous.text.strip() if pre_text == "" and table.findPreviousSibling().findPreviousSibling() != None: # 修复表格前一标签没内容，再前一个才有内容情况 previous = table.findPreviousSibling().findPreviousSibling() pre_text = previous.text.strip() if re.search('项目业主|业\s*主', text) and re.search('业\s*绩', text+pre_text): # 包含业绩的表格过滤掉，不进行处理 tb_ex = table.extract() if previous: sib = previous.extract() continue trs = self.tb.table2list(table) # table.extract() i = 0 headers = "" table_prem = {} while i < len(trs) - 1: flag_, contain_header_, headers_, not_sure_winner = self.find_header(trs[i], all_winner, first_line=i==0) if flag_ and 'tenderer' in headers_ and not_sure_winner and re.search('中标|成交|中选|入围|入选', pre_text)==None: # print('过滤：',headers_) flag_ = False headers_ = {} if flag_ and headers_ != dict(): table_items = [] headers = headers_ for j in range(i + 1, len(trs)): if len(trs[j]) == len(trs[i]): flag_2, contain_header_2, headers_2, not_sure_winner = self.find_header(trs[j], all_winner) if flag_2 or contain_header_2: if j == i+1 and flag_2: if len(headers_)<=len(headers_2): headers = headers_2 continue elif trs[i] == trs[j]: # 修复表格重复表头多次出现情况 例：514890585 continue break elif ''.join(trs[j]).strip() == '': # 修复整行为空的 例：514890585 continue else: table_items.append(trs[j]) else: # print('表头，内容 列数不一致', len(trs[i]), len(trs[j])) break if len(table_items) > 0: df = pd.DataFrame(table_items) prem_ = self.extract_from_df(df, headers, web_source_name, all_winner) # rs_dic.update(prem_) # table_prem.update(prem_) self.update_prem(table_prem, prem_) i = j - 1 i += 1 if table_prem and 'project_code' not in headers and 'package_code' not in headers and '自增1' in table_prem and table.find_previous_sibling(): # 表格内没有标段的，从上一个兄弟标签找标段 sib = table.find_previous_sibling() sib_text = sib.get_text() ser_sib = re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))|(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}|包名：[0-9一二三四五六七八九十]{1,4}', sib_text) if sib.name in ['p','div','dl','ol','ul','h1','h2','h3','h4','h5','h6'] and len(sib_text)<100 and ser_sib: package_sib = ser_sib.group(0) package_sib = uniform_package_name(package_sib) table_prem[package_sib] = table_prem.pop('自增1') if table_prem: # rs_dic.update(table_prem) self.update_prem(rs_dic, table_prem) table.extract() return rs_dic def predict(self, html, nlp_enterprise, web_source_name="", all_winner=False, docid=''): html = re.sub("|||","",html) html = re.sub("##attachment##","",html) soup = BeautifulSoup(html, 'lxml') richText = soup.find(name='div', attrs={'class': 'richTextFetch'}) self.nlp_enterprise = sorted(nlp_enterprise, key=lambda x:len(x), reverse=True) self.docid = docid in_attachment = False if richText: richText = richText.extract() # 过滤掉附件 del_tabel_achievement(soup) # 20240819 过滤掉业绩表格 prem = self.get_prem(soup, web_source_name, all_winner) if prem == {} and richText: del_tabel_achievement(richText) # 20240819 过滤掉业绩表格 prem = self.get_prem(richText, web_source_name, all_winner) in_attachment = True if len(prem) == 1: # 只有一个包且包号为1 或 长度大于2 的大概率为自动增加编号包，改为Project k = list(prem)[0] if k.startswith('自增'): prem['Project'] = prem.pop(k) return prem, in_attachment class CandidateExtractor(object): def __init__(self): '''各要素表头规则''' self.head_rule_dic = { 'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$", 'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|标段（包）|分[包标])(编号|编码)", "project_name": "(包[段组件]|标[段包的项]|标段（包）|分[包标]|采购|项目|工程|货物|商品|产品|设备|通用|主要标的|^包)(名称?|内容)|^标的$", "win_sort": "排名|排序|名次|中标及备选结果|(中标|成交|中选|推荐)(次序|顺序)", 'win_or_not': '是否(建议|推荐)?(中标|成交)|是否入围|是否入库|入围结论|^选择设备$', # 补充站源特别表达：例：577351909 选择设备 1 为中标 0 非中标 "candidate": "((候选|入围|入选|投标|应答|响应|参选|比选|参与)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|申请人|(中标)?公司|(中标)?企业|银行)|(通过)?名单|中标候选人)(名称|名单|全称|\d)?$|^(推荐的?)?供应商(名称|信息)?$|投标个人/单位|^(公司|单位)名称$|供应商单位名称$", #补充 368295593 投标个人/单位 提取 "bid_amount": "投标[报总]?价|报价(总?金额|总价|总额)|总报价|^\w{,5}报价(（[\w、/]{1,15}）)?$|(中标|成交|合同)(不?含税|\w{,3}?)）?([金总]额|[报均总]价|价[格款]?)|承包价|含税价|经评审的价格", "win_tenderer": "第[一1]名|第[一1]预?(中标|成交)?候选人", "second_tenderer": "第[二2]名|第[二2]预?(中标|成交)?候选人", "third_tenderer": "第[三3]名|第[三3]预?(中标|成交)?候选人", } '''非表格候选人正则''' # self.p = '((候选|入围|入选|投标)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|(中标)?公司|(中标)?企业|应答人)|(通过)?名单)(名称|名单|全称|\d)?：$' self.p = '((候选|入围|入选|投标|报价|成交|中标|中选|供[货应]|应答|响应)(人|方|人?单位|机构|厂?商|商家|服务商|公司|企业)|(通过|入围)名单)(名称|名单|全称|\d)?[是为：]?$' self.tb = TableTag2List() with open(INTERFACE_DIR+'/header_set.pkl', 'rb') as f: self.headerset = pickle.load(f) # self.headerset.update(set(['异议受理部门', '异议受理部门联系人', '招标投标监督部门'])) # with open(os.path.dirname(__file__)+'/header_set.pkl', 'wb') as f: # pickle.dump(self.headerset, f) def find_header(self, td_list): fix_td_list = [re.sub('[:：]$|^[一二三四五六七八九十0-9]{1,3}、|(（[\w、×*/]{1,20}）)$|（不?含税）|/万?元|拟|\s', '', it) for it in td_list] # 去除表头无关信息，方便匹配判断是否为表头 header_dic = dict() flag = False contain_header = False if len(set(fix_td_list) & self.headerset)>=2 and (len(set(fix_td_list) & self.headerset)/len(set(fix_td_list))>=0.6 or is_head_line(fix_td_list)): flag = True for i in range(len(td_list)) : text = td_list[i] text = re.sub('\s|[（(]排名不分先后[)）]', '', text) if len(text) > 15: # 长度大于15 不进行表头匹配 continue if re.search('未(中标|成交)原因', text): # 不提取此种表格 return flag, contain_header, dict() num = 0 for k, v in self.head_rule_dic.items(): if k == 'candidate' and re.search('第[一二三]名|第[一二三]预?(中标|成交)?候选人', text): continue if re.search('评分|得分|分数|分值', text) and re.search('排名|排序', text)==None: # 692295071 综合得分排序 continue if re.search(v, text): if k in ['candidate', 'win_tenderer', 'second_tenderer', 'third_tenderer'] and re.search('是否', text): continue elif k == 'win_or_not' and re.search('是否(中标|成交)候选人', text): # 修复 584112560 把第二作第一错误 continue elif k in header_dic: if k in ['bid_amount'] and re.search('总(价|金?额)', text): # 总价替换单价 header_dic[k] = (i, text) num += 1 elif k == 'project_code' and text != header_dic[k][1] and 'package_code' not in header_dic\ and re.search(self.head_rule_dic['package_code'], re.sub('\s', '', '，'.join(td_list)))==None: # 如果出现两次项目编号且没有包号，把第二次出现的作为包号 例：472537470 header_dic['package_code'] = (i, text) elif text == header_dic[k][1]: # 表头相同取后一个 header_dic[k] = (i, text) continue header_dic[k] = (i, text) # if k != 'candidate': # candidate 可与前三候选重复 num += 1 if 'win_tenderer'in header_dic and 'second_tenderer' in header_dic and 'candidate' in header_dic: header_dic.pop('candidate') if num>1: # print('表头错误，一个td匹配到两个表头：', header_dic) return flag, contain_header, dict() if text == '单位': # 20241128 补充金额单位 header_dic['amount_unit'] = (i, text) if ('candidate' in header_dic and 'win_sort' in header_dic) or ('win_tenderer' in header_dic and 'second_tenderer' in header_dic): # 有排名才返回表头进行提取 return flag, contain_header, header_dic elif len(set(fix_td_list) & self.headerset) >= 2 or (len(set(fix_td_list)) == 2 and len(set(fix_td_list[:-1]) & self.headerset) >= 1): # 如果包含两个表头以上或 只有两列且包含一个表头 # 最后一个不算 563507195 单位名称 中标及备选结果 contain_header = True if len(set(fix_td_list) & self.headerset) == 1: td_text = (set(fix_td_list) & self.headerset).pop() if re.match('第[一二三\d](名|(中标|成交)?候选人)$', td_text): contain_header = False return flag, contain_header, dict() def is_role(self, text): if len(text) > 25 or len(text) < 4: return False elif len(re.findall('有限责?任?公司', text)) > 1: return False elif re.search('[\w（）]{4,}(有限责?任?公司|学校|学院|大学|中学|小学|医院|管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园|厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处)$', text): return True else: ners = selffool.ner(text) if len(ners[0]) == 1 and ('company' in ners[0][0] or 'org' in ners[0][0]): return True return False def extract_from_df(self, df, headers): prem_dic = {} link_set = set() candidate_set = set() role_dic = dict() # 保存一二三候选人并排的情况 findtop3 = False findmoney = False line_num = 0 line_package = None package_flag = 0 if "package_code" in headers: package_flag = 1 if len(df)!=len(set(df[headers["package_code"][0]])): # 如果有包号但重复，进行下列判断是否和跟其他字段组合包号 if "project_code" in headers and df[headers["project_code"][0]][0] != df[headers["package_code"][0]][0]: package_flag = 2 elif "project_name" in headers and find_package(df[headers["package_code"][0]][0]): package_flag = 3 for i in df.index: package_code_raw = df.loc[i, headers['package_code'][0]].strip().replace(' ', '') if "package_code" in headers else "" project_code = df.loc[i, headers['project_code'][0]].strip().replace(' ', '') if "project_code" in headers else "" project_name = df.loc[i, headers['project_name'][0]].strip().replace(' ', '') if "project_name" in headers else "" candidate_ = df.loc[i, headers['candidate'][0]].strip().replace(' ', '') if "candidate" in headers else "" win_or_not = df.loc[i, headers['win_or_not'][0]].strip().replace(' ', '') if "win_or_not" in headers else "" # budget_ = df.loc[i, headers['budget'][0]] if "budget" in headers else "" bid_amount_ = df.loc[i, headers['bid_amount'][0]].strip().replace(' ', '') if "bid_amount" in headers else "" win_sort = df.loc[i, headers['win_sort'][0]].strip().replace(' ', '') if "win_sort" in headers else "" win_tenderer = df.loc[i, headers['win_tenderer'][0]].strip().replace(' ', '') if "win_tenderer" in headers else "" second_tenderer = df.loc[i, headers['second_tenderer'][0]].strip().replace(' ', '') if "second_tenderer" in headers else "" third_tenderer = df.loc[i, headers['third_tenderer'][0]].strip().replace(' ', '') if "third_tenderer" in headers else "" amount_unit = df.loc[i, headers['amount_unit'][0]].strip().replace(' ', '') if "amount_unit" in headers else "" joint = '' if set([package_code_raw, candidate_, win_or_not, bid_amount_, win_tenderer, second_tenderer, third_tenderer]) & self.headerset != set(): # 包含表头， 停止匹配 # 排除 ,win_sort 避免367940050漏提取 # print('包含表头， 停止匹配') break if len(set([package_code_raw, candidate_,win_sort, win_or_not, bid_amount_, win_tenderer, second_tenderer, third_tenderer]) - set(['', ' '])) < 2: # 全部为空或内容一样 停止匹配 # print('全部为空或内容一样 停止匹配') if len(set(df.loc[i,:]))==1 and re.search('^第?([一二三四五六七八九十]{1,3}|[a-zA-Z0-9-]{,9})?[分子]?(标[段包项]?|包[组件标]?|合同[包段])([一二三四五六七八九十]{1,3}|[a-zA-Z0-9-]{,9})?$', win_sort): line_package = win_sort continue elif win_sort == '' and candidate_ != '': # 修复 696375822 前几行为空 continue else: break if candidate_ != "" and win_sort == "" and headers['candidate'][0] > 0: # 修复某些表头不说 排名，直接用候选人代替 col_indx = headers['candidate'][0] -1 pre_col = df.loc[i, col_indx] if col_indx > 0 and pre_col == candidate_: pre_col = df.loc[i, col_indx - 1] if re.search('第[一二三]名|第[一二三](中标)?候选人', pre_col): win_sort = pre_col package_code = package_code_raw if package_code == '' and line_package: package_code = line_package # candidate = candidate_ if self.is_role(candidate_) else "" # tenderer = tenderer if self.is_role(tenderer) else "" # candidate = get_role(candidate_, self.nlp_enterprise) if candidate_!='': leader, joint = get_td_companys(candidate_, self.nlp_enterprise) if leader == '': log('表格角色内容不在公司列表：%s，docid：%s' % (candidate_, self.docid)) candidate = leader else: candidate = candidate_ # if len(set([project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_])) < 2: # break if(candidate_,win_tenderer, second_tenderer,third_tenderer, bid_amount_,package_code,project_code,win_sort) in link_set: continue link_set.add((candidate_, win_tenderer, second_tenderer, third_tenderer, bid_amount_,package_code,project_code,win_sort)) package = package_code if package == "" and project_code != "": # 修复 395747178 多项目 只提取到一个 package = project_code package = uniform_package_name(package) if package !="" else "Project" if package_flag == 2 and project_code != "": project_code_pk = uniform_package_name(project_code) package = "%s_%s"%(project_code_pk, package) elif package_flag == 3 and project_name != "": for iter in find_package(project_name): project_name_pk = uniform_package_name(iter.group(0)) package = "%s_%s"%(project_name_pk, package) break if candidate: if win_or_not and re.search('否|未入围', win_or_not): candidate_set.add(candidate) # elif re.search('^((建议|推荐)(中标|成交)|是)$', win_or_not) and win_sort in ['', '参与投标单位及排名'] and win_tenderer=='': # win_sort = '第一名' # candidate_set.add(candidate) else: candidate_set.add(candidate) if win_tenderer and second_tenderer: # and third_tenderer 128778062 这篇只有 第一二候选人 if re.search("(候选人|投标人|单位|公司)名?称?$", df.loc[i, 0]) or re.search("(候选人|投标人|单位|公司)名?称?", df.loc[i, 1]): findtop3 = True for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'], [win_tenderer, second_tenderer, third_tenderer]): # text = get_role(text, self.nlp_enterprise) leader, joint = get_td_companys(text, self.nlp_enterprise) if leader == '': log('表格角色内容不在公司列表：%s，docid：%s' % (text, self.docid)) text = leader if text: # if self.is_role(text): if type not in role_dic: role_dic[type] = dict() role_dic[type]['role_text'] = text candidate_set.add(text) if joint: role_dic[type]['win_tenderer_joint'] = joint elif re.search('投标报价|报价$', df.loc[i, 0]) or re.search('投标报价|报价$', df.loc[i, 1]): findmoney = True header = df.loc[i, 0] if re.search('投标报价|报价$', df.loc[i, 0]) else df.loc[i, 1] for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'], [win_tenderer, second_tenderer, third_tenderer]): if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税', '', text)) > 5: # 金额字段出现超过5个非金额字符，中断匹配 break if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元', text+header)==None: # 补充另外在一列的金额单位 text += amount_unit money, money_unit = money_process(text, header) if re.search('元[/每]', amount_unit) or re.search('单价', header): unit_price = money money = 0 else: unit_price = 0 if (re.search('费率|下浮率|[%％‰折]|优惠率', header+text) and money < 100) or money > 50000000000: # 如果是费率或大于500亿的金额改为0 money = 0 if money > 0: if type not in role_dic: role_dic[type] = dict() role_dic[type]['money'] = money role_dic[type]['money_unit'] = money_unit if unit_price > 0: if type not in role_dic: role_dic[type] = dict() role_dic[type]['unit_price'] = unit_price role_dic[type]['money_unit'] = money_unit else: line_num += 1 if findtop3 and findmoney: break if line_num > 3: break elif candidate and win_sort: role_type = "" if re.search('第[一1]|^([一1]|01|中标人)$', win_sort) and re.search('第一备选人', win_sort)==None: role_type = "win_tenderer" if win_or_not in ['否', '未中标', '0']: # 修复特别站源表达 577351909 选择设备：0 不是中标 role_type = '' elif re.search('第[二2]|^([二2]|02|第一备选人)$', win_sort) and re.search('第二备选人', win_sort)==None: role_type = "second_tenderer" if win_or_not in ['是', '1']: role_type = "win_tenderer" elif re.search('第[三3]|^([三3]|03|第二备选人)$', win_sort): role_type = "third_tenderer" if role_type != "": if package not in prem_dic: prem_dic[package] = { 'code': '', 'name': '', 'roleList': [], 'tendereeMoney': 0, 'tendereeMoneyUnit': "" } prem_dic[package]['code'] = project_code prem_dic[package]['name'] = project_name if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税', '', bid_amount_))> 5: # 金额字段出现超过5个非金额字符，中断匹配 break header = headers['bid_amount'][1] if "bid_amount" in headers else '' if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元', bid_amount_ + header) == None: # 补充另外在一列的金额单位 bid_amount_ += amount_unit bid_amount, money_unit = money_process(bid_amount_, headers['bid_amount'][1]) if "bid_amount" in headers else (0, "") if re.search('元[/每]', amount_unit) or re.search('单价', header): unit_price = bid_amount bid_amount = 0 else: unit_price = 0 if (re.search('费率|下浮率|[%％‰折]|优惠率', header + bid_amount_) and bid_amount < 100) or bid_amount > 50000000000: # 如果是费率或大于500亿的金额改为0 bid_amount = 0 tmp_role_dic = { "address": "", "linklist": [], "role_money": { "discount_ratio": "", "downward_floating_ratio": "", "floating_ratio": "", "money": bid_amount, "money_unit": money_unit }, "role_name": role_type, "role_text": candidate, "serviceTime": "" } if joint != '': tmp_role_dic['win_tenderer_joint'] = joint if unit_price > 0: tmp_role_dic['role_money']['unit_price'] = unit_price if role_type in str(prem_dic[package]['roleList']): # 一种角色只保留一个，解决 682633825 有多个第二名情况 continue prem_dic[package]['roleList'].append(tmp_role_dic) if len(prem_dic[package]['roleList']) == 0: # 只有项目编号和名称的 丢弃 prem_dic.pop(package) if role_dic and prem_dic == dict(): if package not in prem_dic: prem_dic[package] = { 'code': '', 'name': '', 'roleList': [], 'tendereeMoney': 0, 'tendereeMoneyUnit': "" } for role_type, v in role_dic.items(): role_text = v.get('role_text', '') if role_text == "": continue money = v.get('money', 0) money_unit = v.get('money_unit', '') tmp_d = { "address": "", "linklist": [], "role_money": { "discount_ratio": "", "downward_floating_ratio": "", "floating_ratio": "", "money": money, "money_unit": money_unit }, "role_name": role_type, "role_text": role_text, "serviceTime": "" } if 'win_tenderer_joint' in v: tmp_d['win_tenderer_joint'] = v['win_tenderer_joint'] prem_dic[package]['roleList'].append(tmp_d) if len(prem_dic[package]['roleList']) == 0: # 只有项目编号和名称的 丢弃 prem_dic.pop(package) return prem_dic, candidate_set def get_prem(self, soup): tables = soup.find_all('table') tables.reverse() rs_dic = {} candidate_set = set() for table in tables: trs = self.tb.table2list(table) i = 0 headers = "" while i < len(trs) - 1: flag_, contain_header_, headers_ = self.find_header(trs[i]) if flag_ and headers_ != dict(): table_items = [] headers = headers_ for j in range(i + 1, len(trs)): if len(trs[j]) == len(trs[i]): flag_, contain_header_, headers_ = self.find_header(trs[j]) if flag_ or contain_header_: break else: table_items.append(trs[j]) else: # print('表头，内容 列数不一致', len(trs[i]), len(trs[j])) break # print('内容：', len(table_items), table_items) if len(table_items) >= 1: df = pd.DataFrame(table_items) # print('候选人表格行数：', len(df)) prem_, candidate_set_ = self.extract_from_df(df, headers) # print('prem_: ', prem_) rs_dic.update(prem_) candidate_set.update(candidate_set_) i = j - 1 i += 1 if rs_dic and 'package_code' not in headers and 'Project' in rs_dic and (table.find('caption') or table.find_previous_sibling()): # 一个表格只有两行且没有标段的，从上一个兄弟标签找标段 sib = table.find('caption') or table.find_previous_sibling() # 693211745 table下是caption标签写标段信息 sib_text = sib.get_text().strip() ser_sib = re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))|(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}|包名：[0-9一二三四五六七八九十]{1,4}', sib_text) if sib.name in ['p', 'div', 'caption'] and len(sib_text)<100 and ser_sib: package_sib = ser_sib.group(0) package_sib = uniform_package_name(package_sib) rs_dic[package_sib] = rs_dic.pop('Project') table.extract() return rs_dic, candidate_set def get_candidates_from_text(self, list_sentences, list_entitys): candidates = set() tenderee_or_agency = set() sentences = sorted(list_sentences[0], key=lambda x: x.sentence_index) for ent in list_entitys[0]: if ent.entity_type in ['org', 'company']: sen_index = ent.sentence_index text = sentences[sen_index].sentence_text b = ent.wordOffset_begin e = ent.wordOffset_end if ent.label in [2,3,4]: # 直接加实体预测的候选人, 否则规则检查是否为候选人 candidates.add(ent.entity_text) elif isinstance(b, int) and isinstance(e, int) and ent.label in [5]: foreword = text[max(0, b - 10):b] if re.search(self.p, foreword): candidates.add(ent.entity_text) if ent.label in [0, 1] and ent.values[ent.label]>0.5: tenderee_or_agency.add(ent.entity_text) candidates -= tenderee_or_agency # 2024/05/10 463166661 把 四川省第二中医医院作为候选人 过滤掉为招标或代理角色 的候选人 return candidates def predict(self, html, list_sentences, list_entitys, nlp_enterprise, docid=''): self.nlp_enterprise = sorted(nlp_enterprise, key=lambda x:len(x), reverse=True) self.docid = docid html = html.replace('比选申请单位', '中标候选人') # 82347769 html = re.sub("|||","",html) html = re.sub("##attachment##","",html) soup = BeautifulSoup(html, 'lxml') richText = soup.find(name='div', attrs={'class': 'richTextFetch'}) in_attachment = False if richText: richText = richText.extract() # 过滤掉附件 del_tabel_achievement(soup) # 20240819 过滤掉业绩表格 例：500817166 prem, candidate_set = self.get_prem(soup) if prem == {} and richText: del_tabel_achievement(richText) # 20240819 过滤掉业绩表格 prem, candidate_set = self.get_prem(richText) in_attachment = True candidate_set2 = self.get_candidates_from_text(list_sentences, list_entitys) candidate_set.update(candidate_set2) return prem, {'candidate': '，'.join(candidate_set)}, in_attachment def get_header_line(list_item): ''' 判断列表内文本哪些是表头，哪些不是 :param list_item: [ '批复结果', '许可/同意', '批复文号',] :return: ''' rs = [] x = [] for item in list_item: if len(item.strip()) > 30: # 大于30字不可能是表头，直接替换为无意义字符 item = 'xxx' x.append(getPredictor("form").encode(item)) predict_y = getPredictor("form").predict(np.array(x), type="item") for item, values in zip(list_item, list(predict_y)): item = str(item).replace(' ', '') lb = 1 if values[1] > 0.5 else 0 if item in ['许可/同意', '办结（通过）', '办结（准予许可）','批准', '合格', '民间投资', '备案','综合排序第一','满足采购文件要求','未否决']: lb = 0 elif item in ['环境影响评价机构', '建设单位或地方政府作出的相关环保承诺', '环境影响评价技术服务机构', '报告全本'] or re.search('^比例\d{1,2}%$', item): lb = 1 elif lb == 0 and item in header_set: lb = 1 rs.append(lb) return rs