BIDI_EXTRACTION_AI/BiddingKG/dl/predictors/table_prem.py

# -*- coding: utf-8 -*-
"""表格要素 / 候选人 / 表头识别提取器。

按 ARCHITECTURE.md Phase 5 拆分建议，从 ``interface/predictor.py`` 迁出以下
表格相关类与函数：
    - ``TableTag2List``       — soup table 转列表并补全 span（原 8033-8192 行）
    - ``is_head_line``        — 调用表头识别模型判断是否为表头行（原 8195-8211 行）
    - ``TablePremExtractor``  — 表格要素（标段/招标人/中标人/金额）提取（原 8213-8754 行）
    - ``CandidateExtractor``  — 表格候选人提取（原 8756-9183 行）
    - ``get_header_line``     — 判断列表内文本哪些是表头（原 9298-9321 行）

原 ``from common.Utils import *`` / ``from interface.modelFactory import *``
已替换为显式 import；``os.path.dirname(__file__)`` 路径引用替换为
``predictors._common.INTERFACE_DIR``。

``interface/predictor.py`` 仍保留原定义，老 import 不受影响。
"""
from __future__ import absolute_import

import re
import pickle

import numpy as np
import pandas as pd
from bs4 import BeautifulSoup

from BiddingKG.dl.common.logging import log
from BiddingKG.dl.common.context_utils import (
    find_package,
    uniform_package_name,
    money_process,
    cut_repeat_name,
)
from BiddingKG.dl.common.Utils import del_tabel_achievement
from BiddingKG.dl.common.nerUtils import getNers
from BiddingKG.dl.foolnltk import selffool
from BiddingKG.dl.time.re_servicetime import extract_servicetime
from BiddingKG.dl.common.attr_utils import extract_serviceTime
from BiddingKG.dl.interface.predictor import getPredictor
from BiddingKG.dl.predictors._common import (
    INTERFACE_DIR,
    header_set,
    get_td_companys,
    get_role,
)

__all__ = [
    "TableTag2List",
    "is_head_line",
    "TablePremExtractor",
    "CandidateExtractor",
    "get_header_line",
]


class TableTag2List():
    '''把soup table 转化为表格补全后的文本列表[[td, td, td], [td, td, td]]'''
    def table2list(self, table, text_process=None, return_html_table=False,return_kv=False):
        '''
        表格补全及把表格内容列表返回
        :param table:
        :param text_process: 预处理方法，segment(),不为None 时把td内容做预处理，结果返回加标签，适配表头识别 [[[text, 0], [text, 0]] ], 否则只返回文本[[text, text], [text, text]]
        :param return_html_table:
        :param return_kv:
        :return:
        '''
        self._output = []
        row_ind = 0
        col_ind = 0
        html_table = []
        for row in table.find_all('tr'):
            # record the smallest row_span, so that we know how many rows
            # we should skip
            smallest_row_span = 1

            if len(row.find_all(['td', 'th'], recursive=False)) > 20:
                log('未补全前表格列数大于20的不做表格处理')
                if return_html_table:
                    return [], []
                return []

            for cell in row.children:
                if cell.name in ('td', 'th'):
                    # check multiple rows
                    # pdb.set_trace()
                    row_span = int(re.sub('[^0-9]', '', cell.get('rowspan'))) if cell.get('rowspan') and cell.get('rowspan').isdigit() else 1
                    if row_span == 0: # 20250806 修复 659303627 附件OCR重构表格span为0导致缺少问题
                        row_span = 1

                    # try updating smallest_row_span
                    smallest_row_span = min(smallest_row_span, row_span)

                    # check multiple columns
                    col_span = int(re.sub('[^0-9]', '', cell.get('colspan'))) if cell.get('colspan') and cell.get('colspan').isdigit() else 1
                    if col_span > 20: # 修复 335590254 山东港口阳光智采e平台 数据源表格第一行colspan为200超过50列造成无法提取问题
                        col_span = 20
                    elif col_span == 0: # 20250806 修复 659303627 附件OCR重构表格span为0导致缺少问题
                        col_span = 1

                    # find the right index
                    while True:
                        if self._check_cell_validity(row_ind, col_ind):
                            break
                        col_ind += 1

                    # insert into self._output
                    try:
                        if 'title' in cell.attrs and cell.get_text().replace(' ', '').endswith(
                                '...') and cell.attrs['title'].replace(' ', '').startswith(cell.get_text().replace(' ', '')[:-3]):
                            cell.string = cell.attrs['title']  # 修复 类似 215597851 215597851 省略号隐藏内容
                        if text_process != None:
                            # text = [re.sub('\xa0', '', text_process(cell, final=False)), 0]
                            # td_text = re.sub('\xa0', '', text_process(cell, final=False))
                            td_text = re.sub('\s|\xa0', '', str(cell.get_text()))  # 修复 370835008 td 内公司被p标签拆分为两半情况

                            if len(td_text)>30:
                                if return_kv:
                                    td_text = cell.get_text().strip()
                                else:
                                    td_text = re.sub('\xa0', '', text_process(cell, final=False))
                            if td_text == "":
                                td_text = ' '
                            text = [td_text,0]
                        else:
                            if return_kv:
                                td_text = cell.get_text().strip()
                            else:
                                td_text = str(cell.get_text()).strip().replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '').replace('&nbsp', '')
                                td_text = re.sub('\s+', ' ', td_text)
                            text = td_text

                            # text = str(cell.get_text()).strip().replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '')
                            # # text = re.sub('\s', '', text)[:200] # 只需取前200字即可
                            # text = ' ' if text == "" else text

                        self._insert(row_ind, col_ind, row_span, col_span, text)
                        if return_html_table:
                            html_table = self._insert_new(row_ind, col_ind, row_span, col_span, str(cell), html_table)
                    except UnicodeEncodeError:
                        raise Exception( 'Failed to decode text; you might want to specify kwargs transformer=unicode' )

                    # update col_ind
                    col_ind += col_span
                    if col_ind > 50 and text_process == None: # 表格要素提取及候选人提取的 表格列数大于50的去掉
                        if return_html_table:
                            return [], []
                        return []

            # update row_ind
            row_ind += smallest_row_span
            col_ind = 0
        if return_html_table:
            temp_list = []
            for row in self._output:
                if len(row) > 0:
                    temp_list.append(row)
            self._output =  temp_list
            temp_list = []
            for row in html_table:
                if len(row) > 0:
                    temp_list.append(row)
            html_table = temp_list
            return self._output, html_table
        else:
            return self._output

    def _check_validity(self, i, j, height, width):
        """
        check if a rectangle (i, j, height, width) can be put into self.output
        """
        return all(self._check_cell_validity(ii, jj) for ii in range(i, i+height) for jj in range(j, j+width))

    def _check_cell_validity(self, i, j):
        """
        check if a cell (i, j) can be put into self._output
        """
        if i >= len(self._output):
            return True
        if j >= len(self._output[i]):
            return True
        if self._output[i][j] == "":
            return True
        return False

    def _insert(self, i, j, height, width, val):
        # pdb.set_trace()
        for ii in range(i, i+height):
            for jj in range(j, j+width):
                self._insert_cell(ii, jj, val)

    def _insert_cell(self, i, j, val):
        while i >= len(self._output):
            self._output.append([])
        while j >= len(self._output[i]):
            self._output[i].append("")

        if self._output[i][j] == "":
            self._output[i][j] = val

    def _insert_new(self, i, j, height, width, val, cell_list):
        # pdb.set_trace()
        for ii in range(i, i+height):
            for jj in range(j, j+width):
                cell_list = self._insert_cell_new(ii, jj, val, cell_list)
        return cell_list

    def _insert_cell_new(self, i, j, val, cell_list):
        while i >= len(cell_list):
            cell_list.append([])
        while j >= len(cell_list[i]):
            cell_list[i].append("")

        if cell_list[i][j] == "":
            cell_list[i][j] = val
        return cell_list


def is_head_line(list_item):
    '''
    调用表头识别模型判断是否为表头行
    :param list_item: 行内容 例： ['技术参数、要求', '变更项']
    :return:
    '''
    x = []
    for item in list_item:
        x.append(getPredictor("form").encode(item))
    predict_y = getPredictor("form").predict(np.array(x), type="item")
    count = 0
    for item, values in zip(list_item, list(predict_y)):
        if values[1] > 0.6:
            count += 1
    if count/len(list_item)>0.6:
        return True
    return False

class TablePremExtractor(object):
    def __init__(self):
        '''各要素表头规则'''
        self.head_rule_dic = {
            'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|标段（包）|分[包标])(编号|编码|代码)",
            'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$|^品目$",
            "project_name": "(包[段组件]|标[段包的项]|标段（包）|分[包标]|采购|项目|工程|^包)(名称?|内容)", # |货物|商品|产品|设备|通用|主要标的 20250812 货物的不作为项目名称
            "win_sort": "排名|排序|名次|推荐顺序",
            'win_or_not': '是否(建议|推荐)?(中标|成交|中选)|是否入围|是否入库|入围结论|未(中标|成交)原因|中标情况|^中标结果$',
            "tenderer": "(中标|中选|中价|中拍|成交|供货|承包|承建|承租|竞得|受让)）?(候选)?(人(单位)?|单位|供应商|公司|企业|厂家|商家?|客户|供?方|银行|回收商)(名称|$)|^(拟定|单一来源|邀请|拟?推荐(入选|入围)?)?供应商(名称)?$",
            "tenderee": "(项目|采购|招标|遴选|寻源|竞价|议价|比选|委托|询比?价|比价|评选|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选)(人|公司|单位|组织|用户|业主|主体|方|部门)(名称|$)",
            "budget": "最高(投标)?限价|总价限价|控制(价格?|金额|总价)|(总价|采购)限价|上限价|拦标价|(采购|招标|项目)?预算|(预算|招标|采购|计划)金额|挂牌价",
            "bid_amount": "投标[报总]?价|报价(总?金额|总价|总额)|总报价|^\w{,5}报价(（[\w、/]{1,15}）)?$|(中标|中选|中价|中拍|成交|合同)）?总?(金?额|[报均总]价|价[格款]?)|承包价|含税价|经评审的价格|中标存款金?额|中标资金|中标存款|存放金额|分配额度",
            "serviceTime": '合同期限|工期/交货期/服务期|工期\(交货期\)|合格工期|服务期限|工期' \
                 '|工期要求|项目周期|工期\(交货期\)|计划工期\(服务期限\)|服务时限|履行期限|服务周期|供货期限' \
                 '|合格工期|计划工期\(服务期\)|服务期|服务，期|交货\(完工\)时间|交付\(服务、完工\)时间' \
                 '|交货时间|保洁期限|维保期|管理年限|工期承诺|(服务|合同|施工|实施|工程|设计)的?(年限|期限|周期|期：)' \
                 '|计划工期|工期要求|服务期限?' \
                 '|投标工期|设计工期|合格服务周期|总工期|服务时间(范围)?|流转期限|维护期限|服务时限|交货期' \
                 '|完成时间|中标工期|项目周期|期限要求|周期|供货期|合同的?履行日期|计划周期' \
                 '|履约期限|合同的?约定完成时限|合同的?完成日期|承诺完成日期' \
                 '|合同起始日起|合同的?履约期|履约截止日期|承包期限|合同的?完成日期|特许经营期限' \
                 '|服务期间|服务履行期|委托(管理)?期限|经营期限|数量' \
                 '|(工期|服务期限?|交货期限?|服务履行期|合同期限?|履[行约]期限?)说明|存款期限?|(存款|存放|定存)(期|年)限' \
                 '|服务(有效期|年限)|本?合同有效期|协议有效期|项目期限'
        }

        with open(INTERFACE_DIR+'/header_set.pkl', 'rb') as f:
            self.headerset = pickle.load(f)

        self.tb = TableTag2List()


    def find_header(self, td_list, all_winner=False, first_line=False):
        fix_td_list = [re.sub('[:：]$|^[一二三四五六七八九十0-9]{1,3}、|(（[\w、×*/]{1,20}）)$|（不?含税）|/万?元|拟|\s', '', it) for it in td_list]  # 去除表头无关信息，方便匹配判断是否为表头
        header_dic = dict()
        flag = False
        contain_header = False
        not_sure_winner = False  # 是否 不确定中标的中标人表达方式
        for text in set(fix_td_list) - self.headerset:
            if len(text)<10 and re.search(self.head_rule_dic['bid_amount'], text):
                self.headerset.add(text)
        if len(set(fix_td_list))>0 and (first_line or len(set(fix_td_list) & self.headerset)>=2) and (len(set(fix_td_list) & self.headerset)/len(set(fix_td_list))>=0.6 or is_head_line(fix_td_list)):
            other_tenderer = ""
            other_tenderer2 = ""
            flag = True
            for i in range(len(td_list)) :
                text = td_list[i]
                text = re.sub('\s|[（(]排名不分先后[)）]', '', text)
                text = re.sub('排名价', '', text) # 20241225 修复 252208201 排名价（元）错误为排名
                text = re.sub('^人选', '入选', text)
                if text == '备选中标人':
                    text = '第二候选人'
                if len(re.sub('(（[\w、×*/]{1,20}）)$', '', text)) > 15: # 长度大于15 不进行表头匹配
                    continue
                if re.search('未(中标|成交|中选|入围|通过)原因', text):  # 不提取此种表格 673143737
                    return flag, contain_header, dict(), not_sure_winner
                num = 0
                for k, v in self.head_rule_dic.items():
                    if re.search('评分|得分|分数|分值', text):
                        continue
                    if re.search(v, text):
                        if k in ['tenderer'] and re.search('是否|未', text): # 修复103367444 未中标单位名称 作为中标
                            continue
                        if k == 'budget' and re.search('量', text): # 预算工作量 预算采购量 等不作为预算
                            continue
                        elif k == 'bid_amount' and re.search('分配方案|基准利率|BP值', text): # 517987084 中标资金分配方案
                            continue
                        elif k in header_dic:
                            if k in ['budget', 'bid_amount'] and re.search('总(价|金?额)', text):  # 总价替换单价
                                header_dic[k] = (i, text)
                                num += 1
                            elif k == 'project_code' and text != header_dic[k][1] and 'package_code' not in header_dic\
                                    and re.search(self.head_rule_dic['package_code'], re.sub('\s', '', '，'.join(td_list)))==None: # 如果出现两次项目编号且没有包号，把第二次出现的作为包号 例：472537470
                                header_dic['package_code'] = (i, text)
                            continue
                        header_dic[k] = (i, text)
                        num += 1
                    elif re.search('^((中标|成交|中选|入围|入选)(候选)?)(人|单位|机构|中介(服务)?机构|供应商|客户|方|公司|厂商|商家?|社会资本方?|银行)(名称)?$', text) and re.search('未', text)==None:
                        other_tenderer = (i, text)
                    elif re.search('^((投标|应答|响应|候选)(人|单位|机构|中介(服务)?机构|供应商|客户|方|公司|厂商|商家?|社会资本方?|银行)|(存款|投标)?银行|供应商)(名称)?$|^机构名称$|^单位(名称)?$', text) and re.search('未', text)==None:
                        other_tenderer2 = (i, text)
                if num>1:
                    if re.search(self.head_rule_dic['project_code'], text) and re.search(self.head_rule_dic['package_code'], text): # 修复 528486798 分标编号-包号
                        continue
                    # print('表头错误，一个td匹配到两个表头：', header_dic)
                    return flag, contain_header, dict(), not_sure_winner
                if text == '单位': # 20241128 补充金额单位
                    header_dic['amount_unit'] = (i, text)
            if re.search('；金额(（万?元）)?；', '；'.join(td_list)):  # 召回某些表格只写 金额 作为表头，不能识别为招标或中标金额
                if 'tenderer' in header_dic and 'bid_amount' not in header_dic:
                    for i in range(len(td_list)):
                        text = td_list[i]
                        if  re.search('^金额(（万?元）)?$',text):
                            header_dic['bid_amount'] = (i, text)
                            break
                elif 'tenderee' in header_dic and 'budget' not in header_dic:
                    for i in range(len(td_list)):
                        text = td_list[i]
                        if re.search('^金额(（万?元）)?$', text):
                            header_dic['budget'] = (i, text)
                            break
            if all_winner == 1 and 'tenderer' not in header_dic: # 标题有存款、入库、入围等公告补充其他表达做中标人
                if other_tenderer!="":
                    header_dic['tenderer'] = other_tenderer
                elif other_tenderer2!="":
                    header_dic['tenderer'] = other_tenderer2
                    if 'win_sort' not in header_dic:
                        not_sure_winner = True
            elif 'tenderer' not in header_dic and 'win_or_not' in header_dic:
                if other_tenderer!="":
                    header_dic['tenderer'] = other_tenderer
                # elif other_tenderer2!="":  # 20260115 注释 723429011 单位 是否中标 不是中标人
                #     header_dic['tenderer'] = other_tenderer2
            if all_winner == 1 and 'win_sort' in header_dic: # 标题有存管类公告不分排名
                header_dic.pop('win_sort')
            if 'tenderer' in header_dic and re.search('候选|入围|入选', header_dic['tenderer'][1]) and re.search('推荐的?((中标|成交|中选)候选人|(候选|入围|入选)供应商)', header_dic['tenderer'][1])==None and all_winner == False:
                header_dic.pop('tenderer')
            if ('project_code' in header_dic or 'package_code' in header_dic or 'project_name' in header_dic) and (
                     'tenderer' in header_dic): # 包含标段及招标金额或中标人的进行提取  #  or'budget' in header_dic 20250919 改为有中标人的才提取，只有预算的容易与非表格提取重复 例子：647543768
                return flag, contain_header, header_dic, not_sure_winner
            elif ('tenderer' in header_dic) and ('bid_amount' in header_dic): # 包含中标人及中标金额的进行提取
                if 'win_sort' in header_dic: # 有排名的 用候选人提取类
                    return flag, contain_header, dict(), not_sure_winner
                elif re.search('^(候选)?供应商(名称)?', header_dic['tenderer'][1]) and 'win_or_not' not in header_dic and re.search('(中标|成交|合同)）?总?(金?额|[报均总]价|价[格款]?)', header_dic['bid_amount'][1])==None:  # 只有供应商名称 没排名和包号的去掉，预防错误包提取 334205629
                    # print('只有供应商名称 没排名和包号的去掉')
                    return flag, contain_header, dict(), not_sure_winner
                return flag,contain_header, header_dic, not_sure_winner
            elif 'tenderer' in header_dic and (re.search('(中标|中选|中价|成交|竞得)(人|单位|供应商|公司|企业|厂家|商家?|客户|供?方|银行)',header_dic['tenderer'][1]) or all_winner): # 有中标人，且有明确中标关键词的进行提取
                return flag, contain_header, header_dic, not_sure_winner
            # elif 'tenderer' in header_dic and 'serviceTime' in header_dic:
            #     return flag, contain_header, header_dic, not_sure_winner
        elif len(set(fix_td_list) & self.headerset) >= 2 or (len(set(fix_td_list)) == 2 and len(set(td_list) & self.headerset) >= 1): # 如果包含两个表头以上或 只有两列且包含一个表头
            contain_header = True
        return flag, contain_header, dict(), not_sure_winner

    def extract_from_df(self, df, headers, web_source_name, all_winner=False):
        prem_dic = {}
        previous_package = ""  # 上一行包号
        previous_project_name = ""  # 上一行项目名称
        multi_same_package = False # 非连续的重复包号
        package_fix2raw = dict()  # 处理后包号：处理前包号 字典
        link_set = set()
        tenderer_list = [] # 保存所有中标人
        serviceTime_list = []
        not_package = True if 'project_name' in headers and re.search('(货物|商品|产品|通用|主要标的)(名称?|内容)', headers['project_name'][1]) and \
                          'package_code' not in headers and 'budget' not in headers and "bid_amount" not in headers else False

        if set(['project_code', 'package_code', 'tenderee', 'tenderer']) & set(headers) == set() and ('project_name' not in headers # 补充没有项目名称或有项目名称且是货物的才过滤掉
            or re.search('(货物|商品|产品|设备|通用|主要标的)(名称?|内容)', headers['project_name'][1])): # 20240131修复只有货物名称及最高限价的错误作为多包 396636683；  补充避免423647863采购意向被过滤
            # print('没有包号及角色的不要')
            return {}
        have_bid_amount = False # 是否包含中标金额
        if "bid_amount" in headers and re.search('[1-9]+', '#'.join([it.strip() for it in df[headers['bid_amount'][0]]])):
            have_bid_amount = True
        for i in df.index:
            same_package = False  # 连续重复包号，一般是 rowspan 造成；一包 多个采购
            project_code = df.loc[i, headers['project_code'][0]].strip().replace(' ', '') if "project_code" in headers else ""
            package_code_raw = df.loc[i, headers['package_code'][0]].strip().replace(' ', '') if "package_code" in headers else ""
            project_name = df.loc[i, headers['project_name'][0]].strip().replace(' ', '') if "project_name" in headers else ""
            tenderee = df.loc[i, headers['tenderee'][0]].strip().replace(' ', '') if "tenderee" in headers else ""
            tenderer = df.loc[i, headers['tenderer'][0]].strip().replace(' ', '') if "tenderer" in headers else ""
            budget_ = df.loc[i, headers['budget'][0]].strip().replace(' ', '') if "budget" in headers else ""
            bid_amount_ = df.loc[i, headers['bid_amount'][0]].strip().replace(' ', '') if "bid_amount" in headers else ""
            win_sort = df.loc[i, headers['win_sort'][0]].strip().replace(' ', '') if "win_sort" in headers else ""
            win_or_not = df.loc[i, headers['win_or_not'][0]].strip().replace(' ', '') if "win_or_not" in headers else ""
            serviceTime = df.loc[i, headers['serviceTime'][0]].strip().replace(' ', '') if "serviceTime" in headers else ""
            amount_unit = df.loc[i, headers['amount_unit'][0]].strip().replace(' ', '') if "amount_unit" in headers else ""
            joint = ''

            if set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_]) & self.headerset != set(): # 只要有一项为表头 停止匹配
                # print('只要有一项为表头 停止匹配', set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_,win_sort]) & self.headerset)
                break
            if len(set([project_code, package_code_raw, project_name,tenderee,tenderer,budget_,bid_amount_,win_sort])- set(['', ' '])) < 2 and tenderer=='':  # 内容为空或全部一样 停止匹配
                # print('内容为空或全部一样 停止匹配')
                break
            if re.search('详见', project_name):  # 去除某些表达： 详见招标文件
                project_name = ""
            if package_code_raw == "" and re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))$|^(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}$', project_name):
                package_code_raw = project_name
                project_name = ""

            package_code = package_code_raw
            if re.search('合计|总计', package_code+project_code+project_name):
                continue
            if package_code + project_code == previous_package:  # 处理 208162730 一个包采购多种东西情况
                if project_name == previous_project_name:
                    same_package = True
                if previous_package!="": # 有包号或项目编号且跟上一行相同时，去除项目名称
                    project_name = ''
            if win_sort != "" and re.search('排名|排序|名次|推荐顺序', headers['win_sort'][1]): # 此类型表由 CandidateExtractor类提取  防止类似 328485591 作为多包
                break
            if win_or_not != "" and (re.search('(建议|推荐)(中标|成交|中选)|是|^(中标|成交|中选)', win_or_not)==None or re.search('\w', win_or_not)==None): # 2024/04/2 修复 252208201 为空的不中标
                continue
            elif 'win_or_not' in headers and win_or_not == '': # 2024/12/25 修复 334753545 中标情况为空的不中标
                continue
            if "win_sort" in headers and win_sort == "": # '表头有是否中标，内容却空白的，过滤掉'
                continue
            if win_sort == "" and "tenderer" in headers and re.search('候选|入围|入选', headers['tenderer'][1]) and re.search('推荐的?((中标|成交|中选)候选人|(候选|入围|入选)供应商)', headers['tenderer'][1])==None and all_winner == False:
                tenderer = ""

            if tenderer in ['采购失败', '废标']: # 避免类似 353867205 这篇只提取到一个
                continue
            # tenderee = tenderee if self.is_role(tenderee) else ""
            # tenderer = tenderer if self.is_role(tenderer) else ""

            package = uniform_package_name(package_code) if package_code else '自增1' # 没有包号的自动编号的修改为提取到多少个包，某些行未必中标
            if package == '自增1' and project_name != '' and 'project_name' in headers and re.search('包[段组件]|标[段包的项]|标段（包）|分[包标]', headers['project_name'][1]): # 避免 266252226 采购项目名称 做多包多招标人 修复 661973750 标段名称为多包
                package = "自增%s"%str(len(prem_dic) + 1) if previous_project_name != project_name else "自增%s"%str(len(prem_dic))
            if project_name != "" and package.startswith('自增'):
                pk_l = find_package(project_name)
                if len(pk_l)==1:
                    package = uniform_package_name(pk_l[0].group(0))
            elif re.search('[一二三四五六七八九十]+标段：|标段[一二三四五六七八九十]+：', tenderer) and package.startswith('自增'):
                pk_l = find_package(tenderer)
                if len(pk_l) == 1:
                    package = uniform_package_name(pk_l[0].group(0))

            tenderee = get_role(tenderee, self.nlp_enterprise) if tenderee!="" else tenderee
            # tenderer = get_role(tenderer, self.nlp_enterprise) if tenderer!='' else tenderer
            if tenderer!='':
                leader, joint = get_td_companys(tenderer, self.nlp_enterprise)
                if leader == '':
                    log('表格角色内容不在公司列表：%s，docid：%s' % (tenderer, self.docid))
                tenderer = leader

            tenderee = cut_repeat_name(tenderee)
            tenderer = cut_repeat_name(tenderer)

            if len(set([project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_])) < 2:
                break
            if not_package:
                if (project_code, package_code, tenderee, tenderer, budget_, bid_amount_) in link_set:
                    continue
                link_set.add((project_code, package_code, tenderee, tenderer, budget_, bid_amount_))
            else:
                if (project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_) in link_set:
                    continue
                link_set.add((project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_))

            if project_code != "":
                uni_project_code= uniform_package_name(project_code)
                if uni_project_code != "" and uni_project_code!=package:
                    if package.startswith('自增'): # 没有包号有项目编号的，直接用项目编号
                        package = uni_project_code
                    else:
                        # print('重组包号：', '%s_%s'%(uni_project_code, package))
                        package = '%s_%s'%(uni_project_code, package) # 同时包号项目编号及包号的，组合起来做包号
            if package_code_raw!='':
                if multi_same_package == False and package not in package_fix2raw: # 如果处理后的标段号 已经在列表里面，采用原始标段号文本
                    package_fix2raw[package] = package_code_raw
                elif same_package == False:
                    multi_same_package = True
                if multi_same_package:
                    package = package_code_raw
            if package not in prem_dic or not same_package:
                prem_dic[package] = {
                    'code': '',
                    'name': '',
                    'roleList': [],
                    'tendereeMoney': 0,
                    'tendereeMoneyUnit': ""
                }

            prem_dic[package]['code'] = project_code
            prem_dic[package]['name'] = project_name

            if budget_ != "":
                if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税|(六个月|一年|\w{2,3})期加点\d+BP', '', budget_)) > 5:  # 金额字段出现超过5个非金额字符，中断匹配
                    prem_dic.pop(package)
                    break
                budget_header = headers['budget'][1] if 'budget' in headers else ''
                if amount_unit!='' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元', budget_+budget_header)==None : # 20241128 补充某些表格价格单位分开两列， 例：557953660
                    budget_ += amount_unit
                budget, money_unit = money_process(budget_, budget_header)
                if re.search('元[/每]', amount_unit) or re.search('单价', budget_header):
                    unit_tendereeMoney = budget
                    budget = 0
                else:
                    unit_tendereeMoney = 0

                if (re.search('费率|[上下]浮率|[%％‰折]|优惠率|期加点\d+BP',
                              budget_header + budget_) and budget < 100) or budget > 50000000000:  # 如果是费率或大于500亿的金额改为0
                    budget = 0
                if budget > 0:
                    if same_package and prem_dic[package]['tendereeMoney'] != budget: # 处理 类似 136839070 一包多物品多预算
                        prem_dic[package]['tendereeMoney'] += budget
                    else:
                        prem_dic[package]['tendereeMoney'] = budget
                    prem_dic[package]['tendereeMoneyUnit'] = money_unit
                if unit_tendereeMoney > 0:
                    if 'unit_tendereeMoney' not in prem_dic[package]:
                        prem_dic[package]['unit_tendereeMoney'] = 0
                    if same_package and prem_dic[package]['unit_tendereeMoney'] != unit_tendereeMoney:  # 处理 类似 136839070 一包多物品多预算
                        prem_dic[package]['unit_tendereeMoney'] += unit_tendereeMoney
                    else:
                        prem_dic[package]['unit_tendereeMoney'] = unit_tendereeMoney
            if tenderee and not same_package:
                prem_dic[package]['roleList'].append({
                        "address": "",
                        "linklist": [],
                        "role_money": {
                            "discount_ratio": "",
                            "downward_floating_ratio": "",
                            "floating_ratio": "",
                            "money": 0,
                            "money_unit": ""
                        },
                        "role_name": "tenderee",
                        "role_text": tenderee,
                        "serviceTime": ""
                })
            if tenderer:
                if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税|(六个月|一年|\w{2,3})期加点\d+BP', '',
                              bid_amount_)) > 5:  # 金额字段出现超过5个非金额字符，中断匹配
                    prem_dic.pop(package)
                    break
                bid_amount_header = headers['bid_amount'][1] if bid_amount_ != "" else ''
                if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and bid_amount_!='' and re.search('元',
                                                                                                       bid_amount_ + bid_amount_header) == None:
                    bid_amount_ += amount_unit
                bid_amount, money_unit = money_process(bid_amount_, headers['bid_amount'][1]) if bid_amount_ != "" and 'bid_amount' in headers else (0, '')
                if re.search('[%％‰折]|浮率|期加点\d+BP', bid_amount_) and 0 < bid_amount < 100:
                    bid_amount = 0

                if re.search('元[/每]', amount_unit) or re.search('单价', bid_amount_header):
                    unit_price = bid_amount
                    bid_amount = 0
                else:
                    unit_price = 0
                if web_source_name == '河钢供应链管理平台' and 'bid_amount' in headers and re.search('[%％‰折]|浮率', bid_amount_) == None and bid_amount == 0: # 有中标金额字段却金额为0的过滤掉，防止类似 河钢供应链管理平台 站源错误，金额不为0的才算中标
                    if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0:  # 只有项目编号和名称的包 丢弃
                        prem_dic.pop(package)
                    continue
                elif 'bid_amount' in headers and re.search('[%％‰折]|浮率', bid_amount_) == None and have_bid_amount and bid_amount_ in ['/','','0','0.0']: # 如果不是所有行中标金额都为0，则把为0的做非中标
                    if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0:  # 只有项目编号和名称的包 丢弃
                        prem_dic.pop(package)
                    continue

                if (re.search('费率|下浮率|[%％‰折]|优惠率',
                              bid_amount_header + bid_amount_) and bid_amount < 100) or bid_amount > 50000000000:  # 如果是费率或大于500亿的金额改为0
                    bid_amount = 0
                if serviceTime:
                    serviceTime_text = headers['serviceTime'][1] + serviceTime if headers['serviceTime'][1][-1] in [':','：'] else headers['serviceTime'][1] + ':' + serviceTime
                    # print('serviceTime_text',serviceTime_text)
                    serviceTime = extract_servicetime(serviceTime_text)
                    serviceTime.sort(key=lambda x:x.get('begin_index',0))
                    serviceTime = extract_serviceTime(serviceTime[0]['body'],"") if serviceTime else ""
                    # print(serviceTime)
                if not same_package or len(prem_dic[package]['roleList'])==0:
                    role_dic = {
                            "address": "",
                            "linklist": [],
                            "role_money": {
                                "discount_ratio": "",
                                "downward_floating_ratio": "",
                                "floating_ratio": "",
                                "money": bid_amount,
                                "money_unit": money_unit
                            },
                            "role_name": "win_tenderer",
                            "role_text": tenderer,
                            "serviceTime": serviceTime
                    }
                    if joint != '':
                        role_dic['win_tenderer_joint'] = joint
                    if unit_price > 0:
                        role_dic['role_money']['unit_price'] = unit_price
                    prem_dic[package]['roleList'].append(role_dic)
                elif prem_dic[package]['roleList'] and prem_dic[package]['roleList'][-1].get('role_name', '')=='win_tenderer':
                    if 'multi_winner' not in prem_dic[package]['roleList'][-1]:
                        prem_dic[package]['roleList'][-1]['multi_winner'] = prem_dic[package]['roleList'][-1]['role_text']
                        prem_dic[package]['roleList'][-1]['multi_winner'] += ','+ tenderer
                    elif tenderer not in prem_dic[package]['roleList'][-1]['multi_winner']:
                        prem_dic[package]['roleList'][-1]['multi_winner'] += ','+ tenderer
                    if bid_amount != 0 or unit_price > 0: # 有中标金额的才放进去
                        if 'other_winner_dic' not in prem_dic[package]['roleList'][-1]:
                            prem_dic[package]['roleList'][-1]['other_winner_dic'] = []
                        prem_dic[package]['roleList'][-1]['other_winner_dic'].append({'role_text': tenderer, "money": bid_amount, "money_unit": money_unit, "serviceTime": serviceTime})
                tenderer_list.append(tenderer)
                serviceTime_list.append(serviceTime)
            if len(prem_dic[package]['roleList']) == 0 and prem_dic[package]['tendereeMoney'] == 0:  # 只有项目编号和名称的 丢弃 并不再继续往下匹配
                prem_dic.pop(package)
                # break # 注释掉避免 400084571 某些包废标 中断匹配
            if multi_same_package: # 预处理后包号重复的，使用原始包号
                for k, v in package_fix2raw.items():
                    if k in prem_dic:
                        prem_dic[v] = prem_dic.pop(k)
            previous_package = package_code + project_code
            previous_project_name = project_name
        if len(tenderer_list)>2 and len(set(tenderer_list))==1 and "package_code" not in headers: # 没提取到包号且中标人一样应该是错误多包，需去掉多包 例 244355092  281854766
            total_money = 0
            for v in prem_dic.values():
                for d in v['roleList']:
                    if d['role_name'] == "win_tenderer":
                        total_money += d['role_money']['money']
                        if 'other_winner_dic' in d:
                            for other in d['other_winner_dic']:
                                total_money += other.get('money', 0)
            return {'自增1': {
                'code': '',
                'name': '',
                'roleList': [{
                        "address": "",
                        "linklist": [],
                        "role_money": {
                            "discount_ratio": "",
                            "downward_floating_ratio": "",
                            "floating_ratio": "",
                            "money": total_money,
                            "money_unit": ''
                        },
                        "role_name": "win_tenderer",
                        "role_text": tenderer_list[0],
                        "serviceTime": serviceTime_list[0]
                }],
                'tendereeMoney': 0,
                'tendereeMoneyUnit': ""
            }}
        return prem_dic

    def update_prem(self, rs_dic, tmp_dic):
        '''
        合并更新 prem
        :param rs_dic: 返回结果
        :param tmp_dic: 待合并结果
        :return:
        '''
        if '自增1' in tmp_dic and '自增1' not in rs_dic and len(tmp_dic)==len(rs_dic):
            pass
        else:
            for pack in tmp_dic:
                if pack in rs_dic:
                    for k in tmp_dic[pack]:
                        if rs_dic[pack][k] in ['', 0]:
                            rs_dic[pack][k] = tmp_dic[pack][k]
                        elif rs_dic[pack][k] == []:
                            rs_dic[pack][k]  = tmp_dic[pack][k]
                        elif k == 'roleList' and len(rs_dic[pack][k])>0 and rs_dic[pack][k][0].get('role_money', {}).get('money', 0) == 0:
                            rs_dic[pack][k] = tmp_dic[pack][k]
                else:
                    rs_dic[pack] = tmp_dic[pack]

    def get_prem(self, soup, web_source_name='', all_winner=False):
        tables = soup.find_all('table')
        tables.reverse()

        rs_dic = {}
        for table in tables:

            text = table.text.strip()
            pre_text = ""
            previous = None
            if table.findPreviousSibling() != None:
                previous = table.findPreviousSibling()
                pre_text = previous.text.strip()
                if pre_text == "" and table.findPreviousSibling().findPreviousSibling() != None:  # 修复表格前一标签没内容，再前一个才有内容情况
                    previous = table.findPreviousSibling().findPreviousSibling()
                    pre_text = previous.text.strip()
            if re.search('项目业主|业\s*主', text) and re.search('业\s*绩', text+pre_text): # 包含业绩的表格过滤掉，不进行处理
                tb_ex = table.extract()
                if previous:
                    sib = previous.extract()
                continue

            trs = self.tb.table2list(table)
            # table.extract()
            i = 0
            headers = ""
            table_prem = {}
            while i < len(trs) - 1:
                flag_, contain_header_, headers_, not_sure_winner = self.find_header(trs[i], all_winner, first_line=i==0)

                if flag_ and 'tenderer' in headers_ and not_sure_winner and re.search('中标|成交|中选|入围|入选', pre_text)==None:
                    # print('过滤：',headers_)
                    flag_ = False
                    headers_ = {}

                if flag_ and headers_ != dict():
                    table_items = []
                    headers = headers_
                    for j in range(i + 1, len(trs)):
                        if len(trs[j]) == len(trs[i]):
                            flag_2, contain_header_2, headers_2, not_sure_winner = self.find_header(trs[j], all_winner)
                            if flag_2 or contain_header_2:
                                if j == i+1 and flag_2:
                                    if len(headers_)<=len(headers_2):
                                        headers = headers_2
                                    continue
                                elif trs[i] == trs[j]: # 修复表格重复表头多次出现情况 例：514890585
                                    continue
                                break
                            elif ''.join(trs[j]).strip() == '': # 修复整行为空的 例：514890585
                                continue
                            else:
                                table_items.append(trs[j])
                        else:
                            # print('表头，内容 列数不一致', len(trs[i]), len(trs[j]))
                            break
                    if len(table_items) > 0:
                        df = pd.DataFrame(table_items)
                        prem_ = self.extract_from_df(df, headers, web_source_name, all_winner)
                        # rs_dic.update(prem_)
                        # table_prem.update(prem_)
                        self.update_prem(table_prem, prem_)
                    i = j - 1
                i += 1
            if table_prem and 'project_code' not in headers and 'package_code' not in headers and '自增1' in table_prem and table.find_previous_sibling(): # 表格内没有标段的，从上一个兄弟标签找标段
                sib = table.find_previous_sibling()
                sib_text = sib.get_text()
                ser_sib = re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))|(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}|包名：[0-9一二三四五六七八九十]{1,4}', sib_text)
                if sib.name in ['p','div','dl','ol','ul','h1','h2','h3','h4','h5','h6'] and len(sib_text)<100 and ser_sib:
                    package_sib = ser_sib.group(0)
                    package_sib = uniform_package_name(package_sib)
                    table_prem[package_sib] = table_prem.pop('自增1')
            if table_prem:
                # rs_dic.update(table_prem)
                self.update_prem(rs_dic, table_prem)
            table.extract()
        return rs_dic

    def predict(self, html, nlp_enterprise, web_source_name="", all_winner=False, docid=''):
        html = re.sub("<html>|</html>|<body>|</body>","",html)
        html = re.sub("##attachment##","",html)
        soup = BeautifulSoup(html, 'lxml')
        richText = soup.find(name='div', attrs={'class': 'richTextFetch'})
        self.nlp_enterprise = sorted(nlp_enterprise, key=lambda x:len(x), reverse=True)
        self.docid = docid
        in_attachment = False
        if richText:
            richText = richText.extract()  # 过滤掉附件
        del_tabel_achievement(soup) # 20240819 过滤掉业绩表格
        prem = self.get_prem(soup, web_source_name, all_winner)
        if prem == {} and richText:
            del_tabel_achievement(richText) # 20240819 过滤掉业绩表格
            prem = self.get_prem(richText, web_source_name, all_winner)
            in_attachment = True
        if len(prem) == 1:  # 只有一个包且包号为1 或 长度大于2 的大概率为自动增加编号包，改为Project
            k = list(prem)[0]
            if k.startswith('自增'):
                prem['Project'] = prem.pop(k)
        return prem, in_attachment

class CandidateExtractor(object):
    def __init__(self):
        '''各要素表头规则'''
        self.head_rule_dic = {
            'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$",
            'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|标段（包）|分[包标])(编号|编码)",
            "project_name": "(包[段组件]|标[段包的项]|标段（包）|分[包标]|采购|项目|工程|货物|商品|产品|设备|通用|主要标的|^包)(名称?|内容)|^标的$",
            "win_sort": "排名|排序|名次|中标及备选结果|(中标|成交|中选|推荐)(次序|顺序)",
            'win_or_not': '是否(建议|推荐)?(中标|成交)|是否入围|是否入库|入围结论|^选择设备$', # 补充站源特别表达：例：577351909 选择设备 1 为中标 0 非中标
            "candidate": "((候选|入围|入选|投标|应答|响应|参选|比选|参与)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|申请人|(中标)?公司|(中标)?企业|银行)|(通过)?名单|中标候选人)(名称|名单|全称|\d)?$|^(推荐的?)?供应商(名称|信息)?$|投标个人/单位|^(公司|单位)名称$|供应商单位名称$", #补充 368295593 投标个人/单位 提取
            "bid_amount": "投标[报总]?价|报价(总?金额|总价|总额)|总报价|^\w{,5}报价(（[\w、/]{1,15}）)?$|(中标|成交|合同)(不?含税|\w{,3}?)）?([金总]额|[报均总]价|价[格款]?)|承包价|含税价|经评审的价格",
            "win_tenderer": "第[一1]名|第[一1]预?(中标|成交)?候选人",
            "second_tenderer": "第[二2]名|第[二2]预?(中标|成交)?候选人",
            "third_tenderer": "第[三3]名|第[三3]预?(中标|成交)?候选人",
        }
        '''非表格候选人正则'''
        # self.p = '((候选|入围|入选|投标)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|(中标)?公司|(中标)?企业|应答人)|(通过)?名单)(名称|名单|全称|\d)?：$'
        self.p = '((候选|入围|入选|投标|报价|成交|中标|中选|供[货应]|应答|响应)(人|方|人?单位|机构|厂?商|商家|服务商|公司|企业)|(通过|入围)名单)(名称|名单|全称|\d)?[是为：]?$'
        self.tb = TableTag2List()
        with open(INTERFACE_DIR+'/header_set.pkl', 'rb') as f:
            self.headerset = pickle.load(f)
        # self.headerset.update(set(['异议受理部门', '异议受理部门联系人', '招标投标监督部门']))
        # with open(os.path.dirname(__file__)+'/header_set.pkl', 'wb') as f:
        #     pickle.dump(self.headerset, f)

    def find_header(self, td_list):
        fix_td_list = [re.sub('[:：]$|^[一二三四五六七八九十0-9]{1,3}、|(（[\w、×*/]{1,20}）)$|（不?含税）|/万?元|拟|\s', '', it) for it in td_list] # 去除表头无关信息，方便匹配判断是否为表头
        header_dic = dict()
        flag = False
        contain_header = False
        if len(set(fix_td_list) & self.headerset)>=2 and (len(set(fix_td_list) & self.headerset)/len(set(fix_td_list))>=0.6 or is_head_line(fix_td_list)):
            flag = True
            for i in range(len(td_list)) :
                text = td_list[i]
                text = re.sub('\s|[（(]排名不分先后[)）]', '', text)
                if len(text) > 15: # 长度大于15 不进行表头匹配
                    continue
                if re.search('未(中标|成交)原因', text):  # 不提取此种表格
                    return flag, contain_header, dict()
                num = 0
                for k, v in self.head_rule_dic.items():
                    if k == 'candidate' and re.search('第[一二三]名|第[一二三]预?(中标|成交)?候选人', text):
                        continue
                    if re.search('评分|得分|分数|分值', text) and re.search('排名|排序', text)==None: # 692295071 综合得分排序
                        continue
                    if re.search(v, text):
                        if k in ['candidate', 'win_tenderer', 'second_tenderer', 'third_tenderer']  and re.search('是否', text):
                            continue
                        elif k == 'win_or_not' and re.search('是否(中标|成交)候选人', text): # 修复 584112560 把第二作第一错误
                            continue
                        elif k in header_dic:
                            if k in ['bid_amount'] and re.search('总(价|金?额)', text):  # 总价替换单价
                                header_dic[k] = (i, text)
                                num += 1
                            elif k == 'project_code' and text != header_dic[k][1] and 'package_code' not in header_dic\
                                    and re.search(self.head_rule_dic['package_code'], re.sub('\s', '', '，'.join(td_list)))==None: # 如果出现两次项目编号且没有包号，把第二次出现的作为包号 例：472537470
                                header_dic['package_code'] = (i, text)
                            elif text == header_dic[k][1]: # 表头相同取后一个
                                header_dic[k] = (i, text)
                            continue
                        header_dic[k] = (i, text)
                        # if k != 'candidate': # candidate 可与前三候选重复
                        num += 1
                if 'win_tenderer'in header_dic and 'second_tenderer' in header_dic and 'candidate' in header_dic:
                    header_dic.pop('candidate')
                if num>1:
                    # print('表头错误，一个td匹配到两个表头：', header_dic)
                    return flag, contain_header, dict()
                if text == '单位': # 20241128 补充金额单位
                    header_dic['amount_unit'] = (i, text)
            if ('candidate' in header_dic and 'win_sort' in header_dic) or ('win_tenderer' in header_dic and 'second_tenderer' in header_dic): # 有排名才返回表头进行提取
                return flag, contain_header, header_dic
        elif len(set(fix_td_list) & self.headerset) >= 2  or (len(set(fix_td_list)) == 2 and len(set(fix_td_list[:-1]) & self.headerset) >= 1):  # 如果包含两个表头以上或 只有两列且包含一个表头 # 最后一个不算 563507195 单位名称 中标及备选结果
            contain_header = True
            if len(set(fix_td_list) & self.headerset) == 1:
                td_text = (set(fix_td_list) & self.headerset).pop()
                if re.match('第[一二三\d](名|(中标|成交)?候选人)$', td_text):
                    contain_header = False
        return flag, contain_header, dict()

    def is_role(self, text):
        if len(text) > 25 or len(text) < 4:
            return False
        elif len(re.findall('有限责?任?公司', text)) > 1:
            return False
        elif re.search('[\w（）]{4,}(有限责?任?公司|学校|学院|大学|中学|小学|医院|管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园|厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处)$', text):
            return True
        else:
            ners = selffool.ner(text)
            if len(ners[0]) == 1 and ('company' in ners[0][0] or 'org' in ners[0][0]):
                return True
        return False

    def extract_from_df(self, df, headers):
        prem_dic = {}
        link_set = set()
        candidate_set = set()
        role_dic = dict()  # 保存一二三候选人并排的情况
        findtop3 = False
        findmoney = False
        line_num = 0
        line_package = None
        package_flag = 0
        if "package_code" in headers:
            package_flag = 1
            if len(df)!=len(set(df[headers["package_code"][0]])): # 如果有包号但重复，进行下列判断是否和跟其他字段组合包号
                if "project_code" in headers and df[headers["project_code"][0]][0] != df[headers["package_code"][0]][0]:
                    package_flag = 2
                elif "project_name" in headers and find_package(df[headers["package_code"][0]][0]):
                    package_flag = 3
        for i in df.index:
            package_code_raw = df.loc[i, headers['package_code'][0]].strip().replace(' ', '') if "package_code" in headers else ""
            project_code = df.loc[i, headers['project_code'][0]].strip().replace(' ', '') if "project_code" in headers else ""
            project_name = df.loc[i, headers['project_name'][0]].strip().replace(' ', '') if "project_name" in headers else ""
            candidate_ = df.loc[i, headers['candidate'][0]].strip().replace(' ', '') if "candidate" in headers else ""
            win_or_not = df.loc[i, headers['win_or_not'][0]].strip().replace(' ', '') if "win_or_not" in headers else ""
            # budget_ = df.loc[i, headers['budget'][0]] if "budget" in headers else ""
            bid_amount_ = df.loc[i, headers['bid_amount'][0]].strip().replace(' ', '') if "bid_amount" in headers else ""
            win_sort = df.loc[i, headers['win_sort'][0]].strip().replace(' ', '') if "win_sort" in headers else ""
            win_tenderer = df.loc[i, headers['win_tenderer'][0]].strip().replace(' ', '') if "win_tenderer" in headers else ""
            second_tenderer = df.loc[i, headers['second_tenderer'][0]].strip().replace(' ', '') if "second_tenderer" in headers else ""
            third_tenderer = df.loc[i, headers['third_tenderer'][0]].strip().replace(' ', '') if "third_tenderer" in headers else ""
            amount_unit = df.loc[i, headers['amount_unit'][0]].strip().replace(' ', '') if "amount_unit" in headers else ""
            joint = ''

            if set([package_code_raw, candidate_, win_or_not, bid_amount_, win_tenderer, second_tenderer, third_tenderer]) & self.headerset != set(): # 包含表头， 停止匹配 # 排除 ,win_sort 避免367940050漏提取
                # print('包含表头， 停止匹配')
                break
            if len(set([package_code_raw, candidate_,win_sort, win_or_not, bid_amount_, win_tenderer, second_tenderer, third_tenderer]) - set(['', ' '])) < 2:  # 全部为空或内容一样 停止匹配
                # print('全部为空或内容一样 停止匹配')
                if len(set(df.loc[i,:]))==1 and re.search('^第?([一二三四五六七八九十]{1,3}|[a-zA-Z0-9-]{,9})?[分子]?(标[段包项]?|包[组件标]?|合同[包段])([一二三四五六七八九十]{1,3}|[a-zA-Z0-9-]{,9})?$', win_sort):
                    line_package = win_sort
                    continue
                elif win_sort == '' and candidate_ != '': # 修复 696375822 前几行为空
                    continue
                else:
                    break

            if candidate_ != "" and win_sort == "" and headers['candidate'][0] > 0: # 修复某些表头不说 排名，直接用候选人代替
                col_indx = headers['candidate'][0] -1
                pre_col = df.loc[i, col_indx]
                if col_indx > 0 and pre_col == candidate_:
                    pre_col = df.loc[i, col_indx - 1]
                if re.search('第[一二三]名|第[一二三](中标)?候选人', pre_col):
                    win_sort = pre_col

            package_code = package_code_raw
            if package_code == '' and line_package:
                package_code = line_package

            # candidate = candidate_ if self.is_role(candidate_) else ""
            # tenderer = tenderer if self.is_role(tenderer) else ""
            # candidate = get_role(candidate_, self.nlp_enterprise)
            if candidate_!='':
                leader, joint = get_td_companys(candidate_, self.nlp_enterprise)
                if leader == '':
                    log('表格角色内容不在公司列表：%s，docid：%s' % (candidate_, self.docid))
                candidate = leader
            else:
                candidate = candidate_

            # if len(set([project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_])) < 2:
            #     break
            if(candidate_,win_tenderer, second_tenderer,third_tenderer, bid_amount_,package_code,project_code,win_sort) in link_set:
                continue
            link_set.add((candidate_, win_tenderer, second_tenderer, third_tenderer, bid_amount_,package_code,project_code,win_sort))
            package = package_code
            if package == "" and project_code != "":  # 修复 395747178 多项目 只提取到一个
                package = project_code
            package = uniform_package_name(package) if package !="" else "Project"
            if package_flag == 2 and project_code != "":
                project_code_pk = uniform_package_name(project_code)
                package = "%s_%s"%(project_code_pk, package)
            elif package_flag == 3 and project_name != "":
                for iter in find_package(project_name):
                    project_name_pk = uniform_package_name(iter.group(0))
                    package = "%s_%s"%(project_name_pk, package)
                    break

            if candidate:
                if win_or_not and re.search('否|未入围', win_or_not):
                    candidate_set.add(candidate)
                # elif re.search('^((建议|推荐)(中标|成交)|是)$', win_or_not) and win_sort in ['', '参与投标单位及排名'] and win_tenderer=='':
                #     win_sort = '第一名'
                #     candidate_set.add(candidate)
                else:
                    candidate_set.add(candidate)

            if win_tenderer and second_tenderer:  #  and third_tenderer  128778062 这篇只有 第一二候选人
                if re.search("(候选人|投标人|单位|公司)名?称?$", df.loc[i, 0]) or re.search("(候选人|投标人|单位|公司)名?称?", df.loc[i, 1]):
                    findtop3 = True
                    for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'],
                                           [win_tenderer, second_tenderer, third_tenderer]):
                        # text = get_role(text, self.nlp_enterprise)
                        leader, joint = get_td_companys(text, self.nlp_enterprise)
                        if leader == '':
                            log('表格角色内容不在公司列表：%s，docid：%s' % (text, self.docid))
                        text = leader
                        if text:
                        # if self.is_role(text):
                            if type not in role_dic:
                                role_dic[type] = dict()
                            role_dic[type]['role_text'] = text
                            candidate_set.add(text)
                            if joint:
                                role_dic[type]['win_tenderer_joint'] = joint

                elif re.search('投标报价|报价$', df.loc[i, 0]) or re.search('投标报价|报价$', df.loc[i, 1]):
                    findmoney = True
                    header = df.loc[i, 0] if re.search('投标报价|报价$', df.loc[i, 0]) else df.loc[i, 1]
                    for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'],
                                           [win_tenderer, second_tenderer, third_tenderer]):
                        if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税', '',
                                      text)) > 5:  # 金额字段出现超过5个非金额字符，中断匹配
                            break
                        if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元', text+header)==None: # 补充另外在一列的金额单位
                            text += amount_unit
                        money, money_unit = money_process(text, header)
                        if re.search('元[/每]', amount_unit) or re.search('单价', header):
                            unit_price = money
                            money = 0
                        else:
                            unit_price = 0

                        if (re.search('费率|下浮率|[%％‰折]|优惠率', header+text) and money < 100) or money > 50000000000: # 如果是费率或大于500亿的金额改为0
                            money = 0
                        if money > 0:
                            if type not in role_dic:
                                role_dic[type] = dict()
                            role_dic[type]['money'] = money
                            role_dic[type]['money_unit'] = money_unit
                        if unit_price > 0:
                            if type not in role_dic:
                                role_dic[type] = dict()
                            role_dic[type]['unit_price'] = unit_price
                            role_dic[type]['money_unit'] = money_unit
                else:
                    line_num += 1
                    if findtop3 and findmoney:
                        break
                    if line_num > 3:
                        break
            elif candidate and win_sort:
                role_type = ""
                if re.search('第[一1]|^([一1]|01|中标人)$', win_sort) and re.search('第一备选人', win_sort)==None:
                    role_type = "win_tenderer"
                    if win_or_not in ['否', '未中标', '0']: # 修复特别站源表达 577351909 选择设备：0 不是中标
                        role_type = ''
                elif re.search('第[二2]|^([二2]|02|第一备选人)$', win_sort) and re.search('第二备选人', win_sort)==None:
                    role_type = "second_tenderer"
                    if win_or_not in ['是', '1']:
                        role_type = "win_tenderer"
                elif re.search('第[三3]|^([三3]|03|第二备选人)$', win_sort):
                    role_type = "third_tenderer"
                if role_type != "":
                    if package not in prem_dic:
                        prem_dic[package] = {
                            'code': '',
                            'name': '',
                            'roleList': [],
                            'tendereeMoney': 0,
                            'tendereeMoneyUnit': ""
                        }
                        prem_dic[package]['code'] = project_code
                        prem_dic[package]['name'] = project_name
                    if len(re.sub('[金额万元（）():：零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分￥整\s\d,.]|人民币|不?含税', '', bid_amount_))> 5:  # 金额字段出现超过5个非金额字符，中断匹配
                        break
                    header = headers['bid_amount'][1] if "bid_amount" in headers else ''
                    if amount_unit != '' and re.search('^[万亿]?元|%|折[\w/]{,6}$', amount_unit) and re.search('元',
                                                                                                           bid_amount_ + header) == None:  # 补充另外在一列的金额单位
                        bid_amount_ += amount_unit
                    bid_amount, money_unit  = money_process(bid_amount_, headers['bid_amount'][1])  if "bid_amount" in headers else (0, "")
                    if re.search('元[/每]', amount_unit) or re.search('单价', header):
                        unit_price = bid_amount
                        bid_amount = 0
                    else:
                        unit_price = 0

                    if (re.search('费率|下浮率|[%％‰折]|优惠率',
                                  header + bid_amount_) and bid_amount < 100) or bid_amount > 50000000000:  # 如果是费率或大于500亿的金额改为0
                        bid_amount = 0
                    tmp_role_dic = {
                            "address": "",
                            "linklist": [],
                            "role_money": {
                                "discount_ratio": "",
                                "downward_floating_ratio": "",
                                "floating_ratio": "",
                                "money": bid_amount,
                                "money_unit": money_unit
                            },
                            "role_name": role_type,
                            "role_text": candidate,
                            "serviceTime": ""
                    }
                    if joint != '':
                        tmp_role_dic['win_tenderer_joint'] = joint
                    if unit_price > 0:
                        tmp_role_dic['role_money']['unit_price'] = unit_price
                    if role_type in str(prem_dic[package]['roleList']):  # 一种角色只保留一个，解决 682633825 有多个第二名情况
                        continue
                    prem_dic[package]['roleList'].append(tmp_role_dic)
                    if len(prem_dic[package]['roleList']) == 0:  # 只有项目编号和名称的 丢弃
                        prem_dic.pop(package)
        if role_dic and prem_dic == dict():
            if package not in prem_dic:
                prem_dic[package] = {
                    'code': '',
                    'name': '',
                    'roleList': [],
                    'tendereeMoney': 0,
                    'tendereeMoneyUnit': ""
                }
            for role_type, v in role_dic.items():
                role_text = v.get('role_text', '')
                if role_text == "":
                    continue
                money = v.get('money', 0)
                money_unit = v.get('money_unit', '')
                tmp_d = {
                    "address": "",
                    "linklist": [],
                    "role_money": {
                        "discount_ratio": "",
                        "downward_floating_ratio": "",
                        "floating_ratio": "",
                     "money": money,
                        "money_unit": money_unit
                    },
                    "role_name": role_type,
                    "role_text": role_text,
                    "serviceTime": ""
                    }
                if 'win_tenderer_joint' in v:
                    tmp_d['win_tenderer_joint'] = v['win_tenderer_joint']
                prem_dic[package]['roleList'].append(tmp_d)

            if len(prem_dic[package]['roleList']) == 0:  # 只有项目编号和名称的 丢弃
                prem_dic.pop(package)
        return prem_dic, candidate_set

    def get_prem(self, soup):
        tables = soup.find_all('table')
        tables.reverse()
        rs_dic = {}
        candidate_set = set()
        for table in tables:
            trs = self.tb.table2list(table)
            i = 0
            headers = ""
            while i < len(trs) - 1:
                flag_, contain_header_, headers_ = self.find_header(trs[i])
                if flag_ and headers_ != dict():
                    table_items = []
                    headers = headers_
                    for j in range(i + 1, len(trs)):
                        if len(trs[j]) == len(trs[i]):
                            flag_, contain_header_, headers_ = self.find_header(trs[j])
                            if flag_ or contain_header_:
                                break
                            else:
                                table_items.append(trs[j])
                        else:
                            # print('表头，内容 列数不一致', len(trs[i]), len(trs[j]))
                            break
                    # print('内容：', len(table_items), table_items)
                    if len(table_items) >= 1:
                        df = pd.DataFrame(table_items)
                        # print('候选人表格行数：', len(df))
                        prem_, candidate_set_ = self.extract_from_df(df, headers)
                        # print('prem_: ', prem_)
                        rs_dic.update(prem_)
                        candidate_set.update(candidate_set_)
                    i = j - 1
                i += 1
            if rs_dic and 'package_code' not in headers and 'Project' in rs_dic and (table.find('caption') or table.find_previous_sibling()): # 一个表格只有两行且没有标段的，从上一个兄弟标签找标段
                sib = table.find('caption') or table.find_previous_sibling() # 693211745 table下是caption标签写标段信息
                sib_text = sib.get_text().strip()
                ser_sib = re.search('第?[0-9一二三四五六七八九十a-zA-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))|(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zA-Z]{1,4}|包名：[0-9一二三四五六七八九十]{1,4}', sib_text)
                if sib.name in ['p', 'div', 'caption'] and len(sib_text)<100 and ser_sib:
                    package_sib = ser_sib.group(0)
                    package_sib = uniform_package_name(package_sib)
                    rs_dic[package_sib] = rs_dic.pop('Project')
            table.extract()
        return rs_dic, candidate_set

    def get_candidates_from_text(self, list_sentences, list_entitys):
        candidates = set()
        tenderee_or_agency = set()
        sentences = sorted(list_sentences[0], key=lambda x: x.sentence_index)
        for ent in list_entitys[0]:
            if ent.entity_type in ['org', 'company']:
                sen_index = ent.sentence_index

                text = sentences[sen_index].sentence_text
                b = ent.wordOffset_begin
                e = ent.wordOffset_end
                if ent.label in [2,3,4]: # 直接加实体预测的候选人, 否则规则检查是否为候选人
                    candidates.add(ent.entity_text)
                elif isinstance(b, int) and isinstance(e, int) and ent.label in [5]:
                    foreword = text[max(0, b - 10):b]
                    if re.search(self.p, foreword):
                        candidates.add(ent.entity_text)
                if ent.label in [0, 1] and ent.values[ent.label]>0.5:
                    tenderee_or_agency.add(ent.entity_text)
        candidates -= tenderee_or_agency # 2024/05/10 463166661 把 四川省第二中医医院作为候选人 过滤掉为招标或代理角色 的候选人
        return candidates

    def predict(self, html, list_sentences, list_entitys, nlp_enterprise, docid=''):
        self.nlp_enterprise = sorted(nlp_enterprise, key=lambda x:len(x), reverse=True)
        self.docid = docid
        html = html.replace('比选申请单位', '中标候选人')  # 82347769
        html = re.sub("<html>|</html>|<body>|</body>","",html)
        html = re.sub("##attachment##","",html)
        soup = BeautifulSoup(html, 'lxml')
        richText = soup.find(name='div', attrs={'class': 'richTextFetch'})
        in_attachment = False
        if richText:
            richText = richText.extract()  # 过滤掉附件
        del_tabel_achievement(soup) # 20240819 过滤掉业绩表格 例：500817166
        prem, candidate_set = self.get_prem(soup)
        if prem == {} and richText:
            del_tabel_achievement(richText) # 20240819 过滤掉业绩表格
            prem, candidate_set = self.get_prem(richText)
            in_attachment = True
        candidate_set2 = self.get_candidates_from_text(list_sentences, list_entitys)
        candidate_set.update(candidate_set2)
        return prem, {'candidate': '，'.join(candidate_set)}, in_attachment


def get_header_line(list_item):
    '''
    判断列表内文本哪些是表头，哪些不是
    :param list_item: [ '批复结果', '许可/同意', '批复文号',]
    :return:
    '''
    rs = []
    x = []
    for item in list_item:
        if len(item.strip()) > 30: # 大于30字不可能是表头，直接替换为无意义字符
            item = 'xxx'
        x.append(getPredictor("form").encode(item))
    predict_y = getPredictor("form").predict(np.array(x), type="item")
    for item, values in zip(list_item, list(predict_y)):
        item = str(item).replace(' ', '')
        lb = 1 if values[1] > 0.5 else 0
        if item in ['许可/同意', '办结（通过）', '办结（准予许可）','批准', '合格', '民间投资', '备案','综合排序第一','满足采购文件要求','未否决']:
            lb = 0
        elif item in ['环境影响评价机构', '建设单位或地方政府作出的相关环保承诺', '环境影响评价技术服务机构', '报告全本'] or re.search('^比例\d{1,2}%$', item):
            lb = 1
        elif lb == 0 and item in header_set:
            lb = 1
        rs.append(lb)
    return rs