BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/Preprocessing.py

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup, Comment
import copy
import sys
import os
import time
import codecs

from BiddingKG.dl.ratio.re_ratio import extract_ratio
from BiddingKG.dl.table_head.predict import predict

sys.setrecursionlimit(1000000)
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath(".."))
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Entitys import *
from BiddingKG.dl.interface.predictor import getPredictor, TableTag2List
from BiddingKG.dl.common.nerUtils import *
from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
from BiddingKG.dl.time.re_servicetime import extract_servicetime
from BiddingKG.dl.relation_extraction.re_email import extract_email
from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
from BiddingKG.dl.entityLink.entityLink import *

def tableToText(soup):
    '''
    @param:
        soup:网页html的soup
    @return:处理完表格信息的网页text
    '''
    
    def getTrs(tbody):
    #获取所有的tr
        trs = []
        objs = tbody.find_all(recursive=False)
        for obj in objs:
            if obj.name=="tr":
                trs.append(obj)
            if obj.name=="tbody":
                for tr in obj.find_all("tr",recursive=False):
                    trs.append(tr)
        return trs

    def fixSpan(tbody):
        # 处理colspan, rowspan信息补全问题
        #trs = tbody.findChildren('tr', recursive=False)

        trs = getTrs(tbody)
        ths_len = 0
        ths = list()
        trs_set = set()
        #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱
        # 遍历每一个tr

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有colspan 则补全同一行下一个位置
                if 'colspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
                        col = int(re.sub("[^0-9]","",str(td['colspan'])))
                        if col<100 and len(td.get_text())<1000:
                            td['colspan'] = 1
                            for i in range(1, col, 1):
                                td.insert_after(copy.copy(td))

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有rowspan 则补全下一行同样位置
                if 'rowspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
                        row = int(re.sub("[^0-9]","",str(td['rowspan'])))
                        td['rowspan'] = 1
                        for i in range(1, row, 1):
                            # 获取下一行的所有td， 在对应的位置插入
                            if indtr+i<len(trs):
                                tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
                                if len(tds1) >= (indtd) and len(tds1)>0:
                                    if indtd > 0:
                                        tds1[indtd - 1].insert_after(copy.copy(td))
                                    else:
                                        tds1[0].insert_before(copy.copy(td))
                                elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1:  # 修正某些表格最后一列没补全
                                    tds1[indtd-2].insert_after(copy.copy(td))
    def getTable(tbody):
        #trs = tbody.findChildren('tr', recursive=False)
        trs = getTrs(tbody)
        inner_table = []
        for tr in trs:
            tr_line = []
            tds = tr.findChildren(['td','th'], recursive=False)
            if len(tds)==0:
                tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
            for td in tds:
                tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
                #tr_line.append([td.get_text(),0])
            inner_table.append(tr_line)
        return inner_table                          
    
    #处理表格不对齐的问题
    def fixTable(inner_table,fix_value="~~"):
        maxWidth = 0
        for item in inner_table:
            if len(item)>maxWidth:
                maxWidth = len(item)
        if maxWidth > 100:
            # log('表格列数大于100，表格异常不做处理。')
            return []
        for i in range(len(inner_table)):
            if len(inner_table[i])<maxWidth:
                for j in range(maxWidth-len(inner_table[i])):
                    inner_table[i].append([fix_value,0])
        return inner_table
    
    def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            point = ""
            for j in range(width):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_row
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        for j in range(width):
            point = ""
            for i in range(height):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_col
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        
    def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if inner_table[i][j][0]==pad_row:
                    inner_table[i][j][0] = inner_table[i][j-1][0]
                    inner_table[i][j][1] = inner_table[i][j-1][1]
                if inner_table[i][j][0]==pad_col:
                    inner_table[i][j][0] = inner_table[i-1][j][0]
                    inner_table[i][j][1] = inner_table[i-1][j][1]

    def repairTable(inner_table, dye_set=set(), key_set=set(), fix_value="~~"):
        """
        @summary: 修复表头识别，将明显错误的进行修正
        """
        def repairNeeded(line):
            first_1 = -1
            last_1 = -1
            first_0 = -1
            last_0 = -1
            count_1 = 0
            count_0 = 0
            for i in range(len(line)):
                if line[i][0]==fix_value:
                    continue
                if line[i][1]==1:
                    if first_1==-1:
                        first_1 = i
                    last_1 = i
                    count_1 += 1
                if line[i][1]==0:
                    if first_0 == -1:
                        first_0 = i
                    last_0 = i
                    count_0 += 1
            if first_1 ==-1 or last_0 == -1:
                return False
            # 异常情况：第一个不是表头；最后一个是表头；表头个数远大于属性值个数
            if first_1-0 > 0 or last_0-len(line)+1 < 0 or last_1 == len(line)-1 or count_1-count_0 >= 3:
                return True
            return False

        def getsimilarity(line, line1):
            same_count = 0
            for item, item1 in zip(line,line1):
                if item[1] == item1[1]:
                    same_count += 1
            return same_count/len(line)

        def selfrepair(inner_table,index,dye_set,key_set):
            """
            @summary: 计算每个节点受到的挤压度来判断是否需要染色
            """
            #print("B",inner_table[index])
            min_presure = 3
            list_dye = []
            first = None
            count = 0
            # temp_set = set()
            temp_set = set(['~~'])  # 2023/10/10纠正236239652 受让单位识别不到表头； 受让单位，明细用途：用途名称：陵川县民政局，
            _index = 0
            for item in inner_table[index]:
                if first is None:
                    first = item[1]
                    if item[0] not in temp_set:
                        count += 1
                        temp_set.add(item[0])
                else:
                    if first == item[1]:
                        if item[0] not in temp_set:
                            temp_set.add(item[0])
                            count += 1
                    else:
                        list_dye.append([first,count,_index])
                        first = item[1]
                        temp_set.add(item[0])
                        count = 1
                _index += 1
            list_dye.append([first,count,_index])
            if len(list_dye)>1:
                begin = 0
                end = 0
                for i in range(len(list_dye)):
                    end = list_dye[i][2]
                    dye_flag = False
                    # 首尾要求压力减一
                    if i==0:
                        if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i+1][0]
                    elif i==len(list_dye)-1:
                        if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i-1][0]
                    else:
                        if list_dye[i][1]>1:
                            if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                        else:
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                    if dye_flag:
                        for h in range(begin,end):
                            inner_table[index][h][1] = dye_type
                            dye_set.add((inner_table[index][h][0],dye_type))
                            key_set.add(inner_table[index][h][0])
                    begin = end
                #print("E",inner_table[index])

        def otherrepair(inner_table,index,dye_set,key_set):
            list_provide_repair = []
            if index==0 and len(inner_table)>1:
                list_provide_repair.append(index+1)
            elif index==len(inner_table)-1:
                list_provide_repair.append(index-1)
            else:
                list_provide_repair.append(index+1)
                list_provide_repair.append(index-1)
            for provide_index in list_provide_repair:
                if not repairNeeded(inner_table[provide_index]):
                    same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
                    if same_prob>=0.8:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = inner_table[provide_index][i][1]
                    elif same_prob<=0.2:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]==inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1

        len_dye_set = len(dye_set)
        height = len(inner_table)
        for i in range(height):
            if repairNeeded(inner_table[i]):
                selfrepair(inner_table, i, dye_set, key_set)
                #otherrepair(inner_table,i,dye_set,key_set)
        for h in range(len(inner_table)):
            for w in range(len(inner_table[0])):
                if inner_table[h][w][0] in key_set:
                    for item in dye_set:
                        if inner_table[h][w][0] == item[0]:
                            inner_table[h][w][1] = item[1]
        # 如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代
        if len(dye_set) != len(key_set):
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    selfrepair(inner_table,i,dye_set,key_set)
                    #otherrepair(inner_table,i,dye_set,key_set)
            return
        if len(dye_set) == len_dye_set:
            '''
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    otherrepair(inner_table,i,dye_set,key_set)
            '''
            return
        repairTable(inner_table, dye_set, key_set)

    def repair_table2(inner_table):
        """
        @summary: 修复表头识别，将明显错误的进行修正
        """
        # 修复第一第二第三中标候选人作为列表头
        if len(inner_table) >= 2 and len(inner_table[0]) >= 3:
            for i in range(len(inner_table[:3])):
                for j in range(len(inner_table[i]) - 2):
                    if inner_table[i][j][0] == '第一中标候选人' \
                            and inner_table[i][j + 1][0] == '第二中标候选人' \
                            and inner_table[i][j + 2][0] == '第三中标候选人' \
                            and i + 1 < len(inner_table) \
                            and inner_table[i + 1][j][1] == 0 \
                            and inner_table[i + 1][j + 1][1] == 0 \
                            and inner_table[i + 1][j + 2][1] == 0:
                        inner_table[i][j][1] = 1
                        inner_table[i][j + 1][1] = 1
                        inner_table[i][j + 2][1] = 1
                        break

        # 修复连续的第一第二第三候选人行表头
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i])):
                only_chinese1 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j][0]))
                if only_chinese1 in ['第一候选人', '第一中标候选人'] and inner_table[i][j][1] == 0:
                    if j + 1 < len(inner_table[i]) and ''.join(
                            re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 1][0])) in ['第二候选人', '第二中标候选人']:
                        inner_table[i][j][1] = 1
                        inner_table[i][j + 1][1] = 1

                        if j + 2 < len(inner_table[i]) and ''.join(
                                re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 2][0])) in ['第三候选人', '第三中标候选人']:
                            inner_table[i][j + 2][1] = 1

        # 修复多个重复的单元格表头不一致
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i]) - 1):
                only_chinese1 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j][0]))
                only_chinese2 = ''.join(re.findall('[\u4e00-\u9fa5]+', inner_table[i][j + 1][0]))
                if only_chinese1 == only_chinese2 and inner_table[i][j][1] != inner_table[i][j + 1][1]:
                    inner_table[i][j][1] = 1
                    inner_table[i][j + 1][1] = 1

        # 修复一行几乎都是表头，个别不是；或者一行几乎都是非表头，个别是
        for i in range(len(inner_table)):
            head_dict = {}
            not_head_dict = {}
            for j in range(len(inner_table[i])):
                if inner_table[i][j][1] == 1:
                    if inner_table[i][j][0] not in head_dict:
                        head_dict[inner_table[i][j][0]] = 1
                else:
                    if inner_table[i][j][0] not in not_head_dict:
                        not_head_dict[inner_table[i][j][0]] = 1

            # 非表头:表头 <= 1:3
            if len(head_dict.keys()) > 0 and len(not_head_dict.keys()) / len(head_dict.keys()) <= 1 / 3 and len(
                    head_dict.keys()) >= 3:
                for j in range(len(inner_table[i])):
                    if len(re.sub(' ', '', inner_table[i][j][0])) > 0:
                        inner_table[i][j][1] = 1

            # 表头数一个且非表头数大于2且上一行都是表头
            if i > 0 and len(head_dict.keys()) == 1 and len(not_head_dict.keys()) >= 2 and inner_table[i][0][1] == 0:
                last_row = inner_table[i - 1]
                col_list = []
                for j in range(len(last_row)):
                    if len(re.sub(' ', '', last_row[j][0])) > 0:
                        if last_row[j][1] == 0:
                            col_list = []
                            break
                        col_list.append(last_row[j][0])
                if col_list:
                    col_list = list(set(col_list))
                    if len(col_list) > 2:
                        for j in range(len(inner_table[i])):
                            if inner_table[i][j][1] == 1:
                                inner_table[i][j][1] = 0

        # 修复冒号在文本中间的，不能作为表头
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i])):
                _text = inner_table[i][j][0]
                if len(_text) >= 3 and inner_table[i][j][1] == 1:
                    match = re.search('[:：]', _text)
                    if match:
                        start_index, end_index = match.span()
                        if start_index == 0 or end_index == len(_text):
                            continue
                        if re.search('[\u4e00-\u9fa50-9a-zA-Z]', _text[:start_index]) and re.search(
                                '[\u4e00-\u9fa50-9a-zA-Z]', _text[end_index:]):
                            inner_table[i][j][1] = 0

        # 修复表头关键词未作为表头
        head_keyword = ['供应商']
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i])):
                match = re.search('[\u4e00-\u9fa50-9a-zA-Z:：]+', inner_table[i][j][0])
                if inner_table[i][j][1] == 0 and match and match.group() in head_keyword:
                    inner_table[i][j][1] = 1

        # 修复姓名被作为表头 # 2023-02-10 取消修复，避免项目名称、编号，单位、单价等作为了非表头
        # surname = [
        #     "赵", "钱", "孙", "李", "周", "吴", "郑", "王", "冯", "陈", "褚", "卫", "蒋", "沈", "韩", "杨", "朱", "秦", "尤", "许", "何", "吕", "施", "张", "孔", "曹", "严", "华", "金", "魏", "陶", "姜", "戚", "谢", "邹", "喻", "柏", "水", "窦", "章", "云", "苏", "潘", "葛", "奚", "范", "彭", "郎", "鲁", "韦", "昌", "马", "苗", "凤", "花", "方", "俞", "任", "袁", "柳", "酆", "鲍", "史", "唐", "费", "廉", "岑", "薛", "雷", "贺", "倪", "汤", "滕", "殷", "罗", "毕", "郝", "邬", "安", "常", "乐", "于", "时", "傅", "皮", "卞", "齐", "康", "伍", "余", "元", "卜", "顾", "孟", "平", "黄", "和", "穆", "萧", "尹", "姚", "邵", "湛", "汪", "祁", "毛", "禹", "狄", "米", "贝", "明", "臧", "计", "伏", "成", "戴", "谈", "宋", "茅", "庞", "熊", "纪", "舒", "屈", "项", "祝", "董", "梁", "杜", "阮", "蓝", "闵", "席", "季", "麻", "强", "贾", "路", "娄", "危", "江", "童", "颜", "郭", "梅", "盛", "林", "刁", "钟", "徐", "邱", "骆", "高", "夏", "蔡", "田", "樊", "胡", "凌", "霍", "虞", "万", "支", "柯", "昝", "管", "卢", "莫", "经", "房", "裘", "缪", "干", "解", "应", "宗", "丁", "宣", "贲", "邓", "郁", "单", "杭", "洪", "包", "诸", "左", "石", "崔", "吉", "钮", "龚", "程", "嵇", "邢", "滑", "裴", "陆", "荣", "翁", "荀", "羊", "於", "惠", "甄", "麴", "家", "封", "芮", "羿", "储", "靳", "汲", "邴", "糜", "松", "井", "段", "富", "巫", "乌", "焦", "巴", "弓", "牧", "隗", "山", "谷", "车", "侯", "宓", "蓬", "全", "郗", "班", "仰", "秋", "仲", "伊", "宫", "宁", "仇", "栾", "暴", "甘", "钭", "厉", "戎", "祖", "武", "符", "刘", "景", "詹", "束", "龙", "叶", "幸", "司", "韶", "郜", "黎", "蓟", "薄", "印", "宿", "白", "怀", "蒲", "邰", "从", "鄂", "索", "咸", "籍", "赖", "卓", "蔺", "屠", "蒙", "池", "乔", "阴", "欎", "胥", "能", "苍", "双", "闻", "莘", "党", "翟", "谭", "贡", "劳", "逄", "姬", "申", "扶", "堵", "冉", "宰", "郦", "雍", "舄", "璩", "桑", "桂", "濮", "牛", "寿", "通", "边", "扈", "燕", "冀", "郏", "浦", "尚", "农", "温", "别", "庄", "晏", "柴", "瞿", "阎", "充", "慕", "连", "茹", "习", "宦", "艾", "鱼", "容", "向", "古", "易", "慎", "戈", "廖", "庾", "终", "暨", "居", "衡", "步", "都", "耿", "满", "弘", "匡", "国", "文", "寇", "广", "禄", "阙", "东", "殴", "殳", "沃", "利", "蔚", "越", "夔", "隆", "师", "巩", "厍", "聂", "晁", "勾", "敖", "融", "冷", "訾", "辛", "阚", "那", "简", "饶", "空", "曾", "毋", "沙", "乜", "养", "鞠", "须", "丰", "巢", "关", "蒯", "相", "查", "後", "荆", "红", "游", "竺", "权", "逯", "盖", "益", "桓", "公", "万俟", "司马", "上官", "欧阳", "夏侯", "诸葛", "闻人", "东方", "赫连", "皇甫", "尉迟", "公羊", "澹台", "公冶", "宗政", "濮阳", "淳于", "单于", "太叔", "申屠", "公孙", "仲孙", "轩辕", "令狐", "钟离", "宇文", "长孙", "慕容", "鲜于", "闾丘", "司徒", "司空", "亓官", "司寇", "仉", "督", "子车", "颛孙", "端木", "巫马", "公西", "漆雕", "乐正", "壤驷", "公良", "拓跋", "夹谷", "宰父", "谷梁", "晋", "楚", "闫", "法", "汝", "鄢", "涂", "钦", "段干", "百里", "东郭", "南门", "呼延", "归", "海", "羊舌", "微生", "岳", "帅", "缑", "亢", "况", "后", "有", "琴", "梁丘", "左丘", "东门", "西门", "商", "牟", "佘", "佴", "伯", "赏", "南宫", "墨", "哈", "谯", "笪", "年", "爱", "阳", "佟", "第五", "言", "福",
        # ]
        # for i in range(len(inner_table)):
        #     for j in range(len(inner_table[i])):
        #         if inner_table[i][j][1] == 1 \
        #                 and 2 <= len(inner_table[i][j][0]) <= 4 \
        #                 and (inner_table[i][j][0][0] in surname or inner_table[i][j][0][:2] in surname) \
        #                 and re.search("[^\u4e00-\u9fa5]", inner_table[i][j][0]) is None:
        #             inner_table[i][j][1] = 0
        return inner_table

    def sliceTable(inner_table,fix_value="~~"):
        #进行分块
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        last_head = None
        last_is_same_value = False
        for h in range(height):
            is_all_key = True#是否是全表头行
            is_all_value = True#是否是全属性值
            is_same_with_lastHead = True#和上一行的结构是否相同
            is_same_value=True#一行的item都一样
            #is_same_first_item = True#与上一行的第一项是否相同
            same_value = inner_table[h][0][0]
            for w in range(width):
                if last_head is not None:
                    if inner_table[h-1][w][0] != fix_value and inner_table[h-1][w][0] != "" and inner_table[h-1][w][1] == 0:
                        is_all_key = False

                    if inner_table[h][w][0]==1:
                        is_all_value = False

                    if inner_table[h][w][1]!= inner_table[h-1][w][1]:
                        is_same_with_lastHead = False

                if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
                    is_same_value = False
                else:
                    if re.search("\d+",same_value) is not None:
                        is_same_value = False
            if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
                is_same_first_item = False

            last_head = h

            if last_is_same_value:
                last_is_same_value = is_same_value
                continue

            if is_same_value:
                # 该块只有表头一行不合法
                if h - head_list[-1] > 1:
                    head_list.append(h)
                    last_is_same_value = is_same_value
                    continue
            if not is_all_key:
                if not is_same_with_lastHead:
                    # 该块只有表头一行不合法
                    if h - head_list[-1] > 1:
                        head_list.append(h)

        head_list.append(height)
        return head_list
                    
    def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):


        set_item = set()
        height = len(inner_table)
        width = len(inner_table[0])
        empty_set = set()
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                if item.strip()=="":
                    empty_set.add(item)
                else:
                    set_item.add(item)
        list_item = list(set_item)
        if list_item:
            x = []
            for item in list_item:
                x.append(getPredictor("form").encode(item))
            predict_y = getPredictor("form").predict(np.array(x),type="item")
            _dict = dict()

            for item,values in zip(list_item,list(predict_y)):
                _dict[item] = values[1]
                # print("##",item,values)
            #print(_dict)
            for i in range(height):
                for j in range(width):
                    item = inner_table[i][j][0]
                    if item not in empty_set:
                        inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")
        repairTable(inner_table)
        head_list = sliceTable(inner_table)
        
        return inner_table,head_list

    def set_head_model(inner_table):
        origin_inner_table = copy.deepcopy(inner_table)
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i])):
                # 删掉单格前后符号，以免影响表头预测
                col = inner_table[i][j][0]
                col = re.sub("^[^\u4e00-\u9fa5a-zA-Z0-9]+", "", col)
                col = re.sub("[^\u4e00-\u9fa5a-zA-Z0-9]+$", "", col)
                inner_table[i][j] = col

        # 模型预测表头
        predict_list = predict(inner_table)

        # 组合结果
        for i in range(len(inner_table)):
            for j in range(len(inner_table[i])):
                inner_table[i][j] = [origin_inner_table[i][j][0], int(predict_list[i][j])]

        # print("table_head before repair", inner_table)

        # 表头修正
        # repairTable(inner_table)
        inner_table = repair_table2(inner_table)

        # 按表头分割表格
        head_list = sliceTable(inner_table)
        return inner_table, head_list

    def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):

        data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
        predict_y = getPredictor("form").getModel("context").predict(data_x)

        for _position,_y in zip(data_position,predict_y):
            _w = _position[0]
            _h = _position[1]
            if _y[1]>prob_min:
                inner_table[_h][_w][1] = 1
            else:
                inner_table[_h][_w][1] = 0
                _item = inner_table[_h][_w][0]
                if re.search(pat_head,_item) is not None and len(_item)<8:
                    inner_table[_h][_w][1] = 1

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if re.search("[:：]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
                    inner_table[i][j][1] = 1

        repairTable(inner_table)
        head_list = sliceTable(inner_table)
        # print("inner_table:",inner_table)

        return inner_table,head_list
                    
    #设置表头
    def setHead_inline(inner_table,prob_min=0.64):
        pad_row = "@@"
        pad_col = "##"
        removePadding(inner_table, pad_row, pad_col)
        pad_pattern = re.compile(pad_row+"|"+pad_col)
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            
            is_head = False
            is_long_value = False
            
            #判断是否是全padding值
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
                    is_same_value = False
                    break
                
            #predict is head or not with model
            temp_item = ""
            for j in range(width):
                temp_item += inner_table[i][j][0]+"|"
            temp_item = re.sub(pad_pattern,"",temp_item)
            form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
            if form_prob is not None:
                if form_prob[0][1]>prob_min:
                    is_head = True
                else:
                    is_head = False
            
            #print(temp_item,form_prob)
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                #不把连续表头分开
                if not is_head_last:
                    head_list.append(i)
                if is_long_value or is_same_value:
                    head_list.append(i+1)
            if is_head:
                for j in range(width):
                    inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                is_head = False
                
                #predict is head or not with model
                temp_item = ""
                for j in range(head_begin,head_end):
                    temp_item += inner_table[j][i][0]+"|"
                temp_item = re.sub(pad_pattern,"",temp_item)
                form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
                if form_prob is not None:
                    if form_prob[0][1]>prob_min:
                        is_head = True
                    else:
                        is_head = False
                
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        addPadding(inner_table, pad_row, pad_col)
        return inner_table,head_list
    
    #设置表头
    def setHead_withRule(inner_table,pattern,pat_value,count):
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            set_match = set()
            is_head = False
            is_long_value = False
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value:
                    is_same_value = False
                    break
            for j in range(width):
                if re.search(pat_value,inner_table[i][j][0]) is not None:
                    is_head = False
                    break
                str_find = re.findall(pattern,inner_table[i][j][0])
                if len(str_find)>0:
                    set_match.add(inner_table[i][j][0])
                if len(set_match)>=count:
                    is_head = True
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                if not is_head_last:
                    head_list.append(i)
                if is_head:
                    for j in range(width):
                        inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                set_match = set()
                is_head = False
                for j in range(head_begin,head_end):
                    if re.search(pat_value,inner_table[j][i][0]) is not None:
                        is_head = False
                        break
                    str_find = re.findall(pattern,inner_table[j][i][0])
                    if len(str_find)>0:
                        set_match.add(inner_table[j][i][0])
                    if len(set_match)>=count:
                        is_head = True
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        return inner_table,head_list
    
    #取得表格的处理方向
    def getDirect(inner_table,begin,end):
        '''
        column_head = set()
        row_head = set()
        widths = len(inner_table[0])
        for height in range(begin,end):
            for width in range(widths):
                if inner_table[height][width][1] ==1:
                    row_head.add(height)
                if inner_table[height][width][1] ==2:
                    column_head.add(width)
        company_pattern = re.compile("公司")
        if 0 in column_head and begin not in row_head:
            return "column"
        if 0 in column_head and begin in row_head:
            for height in range(begin,end):
                count = 0
                count_flag = True
                for width_index in range(width):
                    if inner_table[height][width_index][1]==0:
                        if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                            count += 1
                        else:
                            count_flag = False
                if count_flag and count>=2:
                    return "column"
        return "row"
        '''
        count_row_keys = 0
        count_column_keys = 0
        width = len(inner_table[0])
        if begin<end:
            for w in range(len(inner_table[begin])):
                if inner_table[begin][w][1]!=0:
                    count_row_keys += 1
        for h in range(begin,end):
            if inner_table[h][0][1]!=0:
                count_column_keys += 1
                
        company_pattern = re.compile("有限(责任)?公司")
        for height in range(begin,end):
            count_set = set()
            count_flag = True
            for width_index in range(width):
                if inner_table[height][width_index][1]==0:
                    if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                        count_set.add(inner_table[height][width_index][0])
                    else:
                        count_flag = False
            if count_flag and len(count_set)>=2:
                return "column"
        # if count_column_keys>count_row_keys:  #2022/2/15 此项不够严谨，造成很多错误，故取消
        #     return "column"
        return "row"
        
            
    #根据表格处理方向生成句子，        
    def getTableText(inner_table,head_list,key_direct=False):
        # packPattern = "(标包|[标包][号段名])"
        packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))"  # 2020/11/23 大网站规则，补充采购类包名
        rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见|评标情况|推荐顺序)"  # 2020/11/23 大网站规则，添加序号为排序
        entityPattern = "((候选|[中投]标|报价)(单位|公司|人|供应商))|供应商名称"
        moneyPattern = "([中投]标|报价)(金额|价)"
        height = len(inner_table)
        width = len(inner_table[0])
        text = ""
         
        for head_i in range(len(head_list)-1):
            
            head_begin = head_list[head_i]
            head_end = head_list[head_i+1]
                
            direct = getDirect(inner_table, head_begin, head_end)

            #若只有一行，则直接按行读取
            if head_end-head_begin==1:
                text_line = ""
                for i in range(head_begin,head_end):
                    for w in range(len(inner_table[i])):
                        if inner_table[i][w][1]==1:
                            _punctuation = "："
                        else:
                            _punctuation = "，"  #2021/12/15 统一为中文标点，避免 206893924 国际F座1108,1,009,197.49元
                        if w>0:
                            if inner_table[i][w][0]!= inner_table[i][w-1][0]:
                                text_line += inner_table[i][w][0]+_punctuation
                        else:
                            text_line += inner_table[i][w][0]+_punctuation
                    text_line = text_line+"。" if text_line!="" else text_line
                text += text_line
            else:
                #构建一个共现矩阵
                table_occurence = []
                for i in range(head_begin,head_end):
                    line_oc = []
                    for j in range(width):
                        cell = inner_table[i][j]
                        line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
                    table_occurence.append(line_oc)


                occu_height = len(table_occurence)
                occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
                #为每个属性值寻找表头
                for i in range(occu_height):
                    for j in range(occu_width):
                        cell = table_occurence[i][j]
                        #是属性值
                        if cell["type"]==0 and cell["text"]!="":
                            left_head = ""
                            top_head = ""

                            find_flag = False
                            temp_head = ""
                            for loop_i in range(1,i+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [1]
                                if table_occurence[i-loop_i][j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i-loop_i][j]["text"]!=temp_head:
                                            top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head
                                    else:
                                        top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head
                                    find_flag = True
                                    temp_head = table_occurence[i-loop_i][j]["text"]
                                    table_occurence[i-loop_i][j]["occu_count"] += 1
                                else:
                                    #找到表头后遇到属性值就返回
                                    if find_flag:
                                        break


                            cell["top_head"] += top_head
                            find_flag = False
                            temp_head = ""



                            for loop_j in range(1,j+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [2]
                                if table_occurence[i][j-loop_j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i][j-loop_j]["text"]!=temp_head:
                                            left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head
                                    else:
                                        left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head
                                    find_flag = True
                                    temp_head = table_occurence[i][j-loop_j]["text"]
                                    table_occurence[i][j-loop_j]["occu_count"] += 1
                                else:
                                    if find_flag:
                                        break
                            cell["left_head"] += left_head
                if direct=="row":
                    for i in range(occu_height):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        money_text = ""
                        #在同一句话中重复的可以去掉
                        text_set = set()
                        for j in range(width):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["top_head"]+"：") if len(cell["top_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["left_head"] + head
                                else:
                                    head += cell["left_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None:
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    if re.search(moneyPattern,head) is not None and entity_text!="":
                                        money_text += head+cell["text"]+","
                                    else:
                                        text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))

                        text += pack_text+rank_text+entity_text+money_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text

                else:
                    for j in range(occu_width):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        text_set = set()
                        for i in range(occu_height):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["top_head"] + head
                                else:
                                    head += cell["top_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None and \
                                        re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))
                        text += pack_text+rank_text+entity_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text


                # if direct=="row":
                #     for i in range(head_begin,head_end):
                #         pack_text = ""
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         #在同一句话中重复的可以去掉
                #         text_set = set()
                #         for j in range(width):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 head = ""
                #
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         #找到表头后遇到属性值就返回
                #                         if find_flag:
                #                             break
                #
                #                 find_flag = False
                #                 temp_head = ""
                #
                #
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(packPattern,head) is not None:
                #                     pack_text += head+inner_table[i][j][0]+"，"
                #                 elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                #                     #排名替换为同一种表达
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += pack_text+rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
                # else:
                #     for j in range(width):
                #
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         text_set = set()
                #         for i in range(head_begin,head_end):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 find_flag = False
                #                 head = ""
                #                 temp_head = ""
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(rankPattern,head) is not None:
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
        return text
    
    def removeFix(inner_table,fix_value="~~"):
        height = len(inner_table)
        width = len(inner_table[0])
        for h in range(height):
            for w in range(width):
                if inner_table[h][w][0]==fix_value:
                    inner_table[h][w][0] = ""
    
    def trunTable(tbody,in_attachment):
        # print(tbody.find('tbody'))
        # 附件中的表格，排除异常错乱的表格
        if in_attachment:
            if tbody.name=='table':
                _tbody = tbody.find('tbody')
                if _tbody is None:
                    _tbody = tbody
            else:
                _tbody = tbody
            _td_len_list = []
            for _tr in _tbody.find_all(recursive=False):
                len_td = len(_tr.find_all(recursive=False))
                _td_len_list.append(len_td)
            if _td_len_list:
                if len(list(set(_td_len_list))) >= 8 or max(_td_len_list) > 100:
                    string_list = [re.sub("\s+","",i)for i in tbody.strings if i and i!='\n']
                    tbody.string = "，".join(string_list)
                    table_max_len = 30000
                    tbody.string = tbody.string[:table_max_len]
                    tbody.name = "turntable"
                    return None
        # fixSpan(tbody)
        # inner_table = getTable(tbody)
        # inner_table = fixTable(inner_table)

        table2list = TableTag2List()
        inner_table = table2list.table2list(tbody, segment)
        inner_table = fixTable(inner_table)

        if inner_table == []:
            string_list = [re.sub("\s+", "", i) for i in tbody.strings if i and i != '\n']
            tbody.string = "，".join(string_list)
            table_max_len = 30000
            tbody.string = tbody.string[:table_max_len]
            # log('异常表格直接取全文')
            tbody.name = "turntable"
            return None

        if len(inner_table)>0 and len(inner_table[0])>0:

            for tr in inner_table:
                for td in tr:
                    if isinstance(td, str):
                        tbody.string = segment(tbody,final=False)
                        table_max_len = 30000
                        tbody.string = tbody.string[:table_max_len]
                        # log('异常表格，不做表格处理，直接取全文')
                        tbody.name = "turntable"
                        return None

            #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
            #inner_table,head_list = setHead_inline(inner_table)
            # inner_table, head_list = setHead_initem(inner_table,pat_head)
            inner_table, head_list = set_head_model(inner_table)
            # inner_table,head_list = setHead_incontext(inner_table,pat_head)
            # print("table_head", inner_table)
            # print("head_list", head_list)
            # for begin in range(len(head_list[:-1])):
            #     for item in inner_table[head_list[begin]:head_list[begin+1]]:
            #         print(item)
            #     print("====")

            removeFix(inner_table)
            
            # print("----")
            # print(head_list)
            # for item in inner_table:
            #     print(item)

            tbody.string = getTableText(inner_table,head_list)
            table_max_len = 30000
            tbody.string = tbody.string[:table_max_len]
            # print(tbody.string)
            tbody.name = "turntable"
            return inner_table
        return None


        
    pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$')
    #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
    pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")

    list_innerTable = []

    # 2022/2/9 删除干扰标签
    for tag in soup.find_all('option'): #例子： 216661412
        if 'selected' not in tag.attrs:
            tag.extract()
    for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题
        if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall(
                '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|资格评审',
                ul.get_text(), re.S)))>3:
            ul.extract()

    # tbodies = soup.find_all('table')
    # 遍历表格中的每个tbody
    tbodies = []
    in_attachment = False
    for _part in soup.find_all():
        if _part.name=='table':
            tbodies.append((_part,in_attachment))
        elif _part.name=='div':
            if 'class' in _part.attrs and "richTextFetch" in _part['class']:
                in_attachment = True
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody,_in_attachment)
        list_innerTable.append(inner_table)

    # tbodies = soup.find_all('tbody')
    # 遍历表格中的每个tbody
    tbodies = []
    in_attachment = False
    for _part in soup.find_all():
        if _part.name == 'tbody':
            tbodies.append((_part, in_attachment))
        elif _part.name == 'div':
            if 'class' in _part.attrs and "richTextFetch" in _part['class']:
                in_attachment = True
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody,_in_attachment = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody,_in_attachment)
        list_innerTable.append(inner_table)

    return soup
    # return list_innerTable

re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]")
num_dict = {
    "一": 1, "二": 2,
    "三": 3, "四": 4,
    "五": 5, "六": 6,
    "七": 7, "八": 8,
    "九": 9, "十": 10}
# 一百以内的中文大写转换为数字
def change2num(text):
    result_num = -1
    # text = text[:6]
    match = re_num.search(text)
    if match:
        _num = match.group()
        if num_dict.get(_num):
            return num_dict.get(_num)
        else:
            tenths = 1
            the_unit = 0
            num_split = _num.split("十")
            if num_dict.get(num_split[0]):
                tenths = num_dict.get(num_split[0])
            if num_dict.get(num_split[1]):
                the_unit = num_dict.get(num_split[1])
            result_num = tenths * 10 + the_unit
    elif re.search("\d{1,2}",text):
        _num = re.search("\d{1,2}",text).group()
        result_num = int(_num)
    return result_num
#大纲分段处理
def get_preprocessed_outline(soup):
    pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、．\.]")
    pattern_1 = re.compile("^[\(（]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\)）]")
    pattern_2 = re.compile("^\d{1,2}[、．\.](?=[^\d]{1,2}|$)")
    pattern_3 = re.compile("^[\(（]?\d{1,2}[\)）]")
    pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3]

    body = soup.find("body")
    if body == None:
        return soup  # 修复 无body的报错 例子：264419050
    body_child = body.find_all(recursive=False)
    deal_part = body
    # print(body_child[0]['id'])
    if 'id' in body_child[0].attrs:
        if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
            deal_part = body_child[0]
    if len(deal_part.find_all(recursive=False))>2:
        deal_part = deal_part.parent
    skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot']
    for part in deal_part.find_all(recursive=False):
        # 查找解析文本的主干部分
        is_main_text = False
        through_text_num = 0
        while (not is_main_text and part.find_all(recursive=False)):
            while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \
                    part.find_all(recursive=False)[0].get_text(strip=True):
                part = part.find_all(recursive=False)[0]
            max_len = len(part.get_text(strip=True))
            is_main_text = True
            for t_part in part.find_all(recursive=False):
                if t_part.name not in skip_tag and t_part.get_text(strip=True)!="":
                    through_text_num += 1
                if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65:
                    if t_part.name not in skip_tag:
                        is_main_text = False
                        part = t_part
                        break
                    else:
                        while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \
                                t_part.find_all(recursive=False)[0].get_text(strip=True):
                            t_part = t_part.find_all(recursive=False)[0]
                        if through_text_num>2:
                            is_table = True
                            for _t_part in t_part.find_all(recursive=False):
                                if _t_part.name not in skip_tag:
                                    is_table = False
                                    break
                            if not is_table:
                                is_main_text = False
                                part = t_part
                                break
                        else:
                            is_main_text = False
                            part = t_part
                            break
        is_find = False
        for _pattern in pattern_list:
            last_index = 0
            handle_list = []
            for _part in part.find_all(recursive=False):
                if _part.name not in skip_tag and _part.get_text(strip=True) != "":
                    # print('text:', _part.get_text(strip=True))
                    re_match = re.search(_pattern, _part.get_text(strip=True))
                    if re_match:
                        outline_index = change2num(re_match.group())
                        if last_index < outline_index:
                            # _part.insert_before("##split##")
                            handle_list.append(_part)
                            last_index = outline_index
            if len(handle_list)>1:
                is_find = True
                for _part in handle_list:
                    _part.insert_before("##split##")
            if is_find:
                break
    # print(soup)
    return soup

#数据清洗
def segment(soup,final=True):
    # print("==")
    # print(soup)
    # print("====")
    #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
    subspaceList = ["td",'a',"span","p"]
    if soup.name in subspaceList:
        #判断有值叶子节点数
        _count = 0
        for child in soup.find_all(recursive=True):
            if child.get_text().strip()!="" and len(child.find_all())==0:
                _count += 1
        if _count<=1:
            text = soup.get_text()
            # 2020/11/24 大网站规则添加
            if 'title' in soup.attrs:
                if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
                    text = soup.attrs['title']

            _list = []
            for x in re.split("\s+",text):
                if x.strip()!="":
                    _list.append(len(x))
            if len(_list)>0:
                _minLength = min(_list)
                if _minLength>2:
                    _substr = "，"
                else:
                    _substr = ""
            else:
                _substr = ""

            text = text.replace("\r\n","，").replace("\n","，")
            text = re.sub("\s+",_substr,text)
            # text = re.sub("\s+","##space##",text)
            return text
    segList = ["title"]
    commaList = ["div","br","td","p","li"]
    #commaList = []
    spaceList = ["span"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        # print(child.name,child.get_text())
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            child.insert_after("，")
        # if child.name == 'div' and 'class' in child.attrs:
        #     # 添加附件"attachment"标识
        #     if "richTextFetch" in child['class']:
        #         child.insert_before("##attachment##")
                # print(child.parent)
        # if child.name in subspaceList:
        #     child.insert_before("#subs"+str(child.name)+"#")
        #     child.insert_after("#sube"+str(child.name)+"#")
        # if child.name in spaceList:
        #     child.insert_after(" ")
    text = str(soup.get_text())
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
    # 感叹号替换为中文句号
    text = re.sub("(?<=[\u4e00-\u9fa5])[!！]|[!！](?=[\u4e00-\u9fa5])","。",text)
    #替换格式未识别的问号为" " ,update:2021/7/20
    text = re.sub("[？\?]{2,}|\n"," ",text)


    #替换"""为"“",否则导入deepdive出错
    # text = text.replace('"',"“").replace("\r","").replace("\n","，")
    text = text.replace('"',"“").replace("\r","").replace("\n","").replace("\\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
    text = re.sub("(&nbsp)+"," ",text) # 空白符替换
    # print('==1',text)
    # text = re.sub("\s{4,}","，",text)
    # 解决公告中的" "空格替换问题
    if re.search("\s{4,}",text):
        _text = ""
        for _sent in re.split("。+",text):
            for _sent2 in re.split('，+',_sent):
                for _sent3 in re.split("：+",_sent2):
                    for _t in re.split("\s{4,}",_sent3):
                        if len(_t)<3:
                            _text += _t
                        else:
                            _text += "，"+_t
                    _text += "："
                _text = _text[:-1]
                _text += "，"
            _text = _text[:-1]
            _text += "。"
        _text = _text[:-1]
        text = _text
    # print('==2',text)
    #替换标点

    #替换连续的标点

    if final:
        text = re.sub("##space##"," ",text)

    punc_pattern = "(?P<del>[。，；：:,\s]+)"

    list_punc = re.findall(punc_pattern,text)
    list_punc.sort(key=lambda x:len(x),reverse=True)
    for punc_del in list_punc:
        if len(punc_del)>1:
            if len(punc_del.strip())>0:
                if "：" in punc_del.strip():
                    if "。" in punc_del.strip():
                        text = re.sub(punc_del, "：。", text)
                    else:
                        text = re.sub(punc_del,"：",text)
                else:
                    text = re.sub(punc_del,punc_del.strip()[0],text)   #2021/12/09 修正由于某些标签后插入符号把原来符号替换
            else:
                text = re.sub(punc_del,"",text)

    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)

    # #删除标签中的所有空格
    # for subs in subspaceList:
    #     patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
    #     while(True):
    #         oneMatch = re.search(re.compile(patten),text)
    #         if oneMatch is not None:
    #             _match = oneMatch.group(1)
    #             text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
    #         else:
    #             break

    # text过大报错
    LOOP_LEN = 10000
    LOOP_BEGIN = 0
    _text = ""



    if len(text)<10000000:
        while(LOOP_BEGIN<len(text)):
            _text += re.sub("）",")",re.sub("（","(",re.sub("\s(?!\d{2}:\d{2})","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
            LOOP_BEGIN += LOOP_LEN
        text = _text
    # 附件标识前修改为句号，避免正文和附件内容混合在一起
    text = re.sub("[^。](?=##attachment##)","。",text)
    text = re.sub("[^。](?=##attachment_begin##)","。",text)
    text = re.sub("[^。](?=##attachment_end##)","。",text)
    text = re.sub("##attachment_begin##。","##attachment_begin##",text)
    text = re.sub("##attachment_end##。","##attachment_end##",text)

    return text

'''
#数据清洗
def segment(soup):
    segList = ["title"]
    commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]    
    spaceList = ["span"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        if child.name == 'br':
            child.insert_before('，')
        child_text = re.sub('\s', '', child.get_text())
        if child_text == '' or child_text[-1] in ['。','，','：','；']:
            continue
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            if len(child_text)>3 and len(child_text) <50:   # 先判断是否字数少于50，成立加逗号，否则加句号
                child.insert_after("，")           
            elif len(child_text) >=50:
                child.insert_after("。")
        #if child.name in spaceList:
            #child.insert_after(" ")  
    text = str(soup.get_text())
    text = re.sub("\s{5,}","，",text)
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“")    
    #text = text.replace('"',"“").replace("\r","").replace("\n","")
    
    #删除所有空格
    text = re.sub("\s+","#nbsp#",text)
    text_list = text.split('#nbsp#')
    new_text = ''
    for i in range(len(text_list)-1):
        if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']:
            new_text += text_list[i]  
        elif  re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '。' 
        elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '；' 
        elif text_list[i].isdigit() and text_list[i+1].isdigit():
            new_text += text_list[i] + ' '           
        elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']:
            new_text += text_list[i]       
        elif len(text_list[i]) >= 3 and  len(text_list[i+1]) >= 3:
            new_text += text_list[i] + '，'
        else:
            new_text += text_list[i]       
    new_text += text_list[-1]      
    text = new_text
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
            
    #替换标点
    while(True):
        #替换连续的标点
        punc = re.search("，(?P<punc>：|。|，|；)\s*",text)
        if punc is not None:
            text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text)
        
        punc = re.search("(?P<punc>：|。|，|；)\s*，",text)
        if punc is not None:
            text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text)
        else:
            #替换标点之后的空格
            punc = re.search("(?P<punc>：|。|，|；)\s+",text)
            if punc is not None:
                text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
            else:
                break
    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)
    #替换中文括号为英文括号
    text = re.sub("（","(",text)
    text = re.sub("）",")",text)
    return text
'''

#连续实体合并（弃用）
def union_ner(list_ner):
    result_list = []
    union_index = []
    union_index_set = set()
    for i in range(len(list_ner)-1):
        if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
            if list_ner[i][1]-list_ner[i+1][0]==1:
                union_index_set.add(i)
                union_index_set.add(i+1)
                union_index.append((i,i+1))
    for i in range(len(list_ner)):
        if i not in union_index_set:
            result_list.append(list_ner[i])
    for item in union_index:
        #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
        result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
    return result_list
                

# def get_preprocessed(articles,useselffool=False):
#     '''
#     @summary:预处理步骤，NLP处理、实体识别
#     @param:
#         articles:待处理的文章list [[id,source,jointime,doc_id,title]]
#     @return:list of articles,list of each article of sentences,list of each article of entitys
#     '''
#     list_articles = []
#     list_sentences = []
#     list_entitys = []
#     cost_time = dict()
#     for article in articles:
#         list_sentences_temp = []
#         list_entitys_temp = []
#         doc_id = article[0]
#         sourceContent = article[1]
#         _send_doc_id = article[3]
#         _title = article[4]
#         #表格处理
#         key_preprocess = "tableToText"
#         start_time = time.time()
#         article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
#
#         # log(article_processed)
#
#         if key_preprocess not in cost_time:
#             cost_time[key_preprocess] = 0
#         cost_time[key_preprocess] += time.time()-start_time
#
#         #article_processed = article[1]
#         list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
#         #nlp处理
#         if article_processed is not None and len(article_processed)!=0:
#             split_patten = "。"
#             sentences = []
#             _begin = 0
#             for _iter in re.finditer(split_patten,article_processed):
#                 sentences.append(article_processed[_begin:_iter.span()[1]])
#                 _begin = _iter.span()[1]
#             sentences.append(article_processed[_begin:])
#
#             lemmas = []
#             doc_offsets = []
#             dep_types = []
#             dep_tokens = []
#
#             time1 = time.time()
#
#             '''
#             tokens_all = fool.cut(sentences)
#             #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
#             #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
#             ner_entitys_all = fool.ner(sentences)
#             '''
#             #限流执行
#             key_nerToken = "nerToken"
#             start_time = time.time()
#             tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
#             if key_nerToken not in cost_time:
#                 cost_time[key_nerToken] = 0
#             cost_time[key_nerToken] += time.time()-start_time
#
#
#             for sentence_index in range(len(sentences)):
#
#
#
#                 list_sentence_entitys = []
#                 sentence_text = sentences[sentence_index]
#                 tokens = tokens_all[sentence_index]
#
#                 list_tokenbegin = []
#                 begin = 0
#                 for i in range(0,len(tokens)):
#                     list_tokenbegin.append(begin)
#                     begin += len(str(tokens[i]))
#                 list_tokenbegin.append(begin+1)
#                 #pos_tag = pos_all[sentence_index]
#                 pos_tag = ""
#
#                 ner_entitys = ner_entitys_all[sentence_index]
#
#                 list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
#
#                 #识别package
#
#
#                 #识别实体
#                 for ner_entity in ner_entitys:
#                     begin_index_temp = ner_entity[0]
#                     end_index_temp = ner_entity[1]
#                     entity_type = ner_entity[2]
#                     entity_text = ner_entity[3]
#
#                     for j in range(len(list_tokenbegin)):
#                         if list_tokenbegin[j]==begin_index_temp:
#                             begin_index = j
#                             break
#                         elif list_tokenbegin[j]>begin_index_temp:
#                             begin_index = j-1
#                             break
#                     begin_index_temp += len(str(entity_text))
#                     for j in range(begin_index,len(list_tokenbegin)):
#                         if list_tokenbegin[j]>=begin_index_temp:
#                             end_index = j-1
#                             break
#                     entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#                     #去掉标点符号
#                     entity_text = re.sub("[,，。：]","",entity_text)
#                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
#
#
#                 #使用正则识别金额
#                 entity_type = "money"
#
#                 #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"
#
#                 list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
#                                       "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"}
#
#                 set_begin = set()
#                 for pattern_key in list_money_pattern.keys():
#                     pattern = re.compile(list_money_pattern[pattern_key])
#                     all_match = re.findall(pattern, sentence_text)
#                     index = 0
#                     for i in range(len(all_match)):
#                         if len(all_match[i][0])>0:
#                             # print("===",all_match[i])
#                             #print(all_match[i][0])
#                             unit = ""
#                             entity_text = all_match[i][3]
#                             if pattern_key in ["key_word","front_m"]:
#                                 unit = all_match[i][1]
#                             else:
#                                 unit = all_match[i][4]
#                             if entity_text.find("元")>=0:
#                                 unit = ""
#
#                             index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
#
#                             begin_index_temp = index
#                             for j in range(len(list_tokenbegin)):
#                                 if list_tokenbegin[j]==index:
#                                     begin_index = j
#                                     break
#                                 elif list_tokenbegin[j]>index:
#                                     begin_index = j-1
#                                     break
#                             index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
#                             end_index_temp = index
#                             #index += len(str(all_match[i][0]))
#                             for j in range(begin_index,len(list_tokenbegin)):
#                                 if list_tokenbegin[j]>=index:
#                                     end_index = j-1
#                                     break
#                             entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#
#                             entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
#                             if len(unit)>0:
#                                 entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
#                             else:
#                                 entity_text = str(getUnifyMoney(entity_text))
#
#                             _exists = False
#                             for item in list_sentence_entitys:
#                                 if item.entity_id==entity_id and item.entity_type==entity_type:
#                                     _exists = True
#                             if not _exists:
#                                 if float(entity_text)>10:
#                                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
#
#                         else:
#                             index += 1
#
#                 list_sentence_entitys.sort(key=lambda x:x.begin_index)
#                 list_entitys_temp = list_entitys_temp+list_sentence_entitys
#         list_sentences.append(list_sentences_temp)
#         list_entitys.append(list_entitys_temp)
#     return list_articles,list_sentences,list_entitys,cost_time

def get_preprocessed(articles, useselffool=False):
    '''
    @summary:预处理步骤，NLP处理、实体识别
    @param:
        articles:待处理的文章list [[id,source,jointime,doc_id,title]]
    @return:list of articles,list of each article of sentences,list of each article of entitys
    '''
    cost_time = dict()
    list_articles = get_preprocessed_article(articles,cost_time)
    list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
    list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)

    calibrateEnterprise(list_articles,list_sentences,list_entitys)

    return list_articles,list_sentences,list_entitys,list_outlines,cost_time

def special_treatment(sourceContent, web_source_no):
    try:
        if web_source_no == 'DX000202-1':
             ser = re.search('中标供应商及中标金额：【(([\w（）]{5,20}-[\d,.]+,)+)】', sourceContent)
             if ser:
                 new = ""
                 l = ser.group(1).split(',')
                 for i in range(len(l)):
                     it = l[i]
                     if '-' in it:
                         role, money = it.split('-')
                         new += '标段%d, 中标供应商: ' % (i + 1) + role + '，中标金额：' + money + '。'
                 sourceContent = sourceContent.replace(ser.group(0), new, 1)
        elif web_source_no == '00753-14':
            body = sourceContent.find("body")
            body_child = body.find_all(recursive=False)
            pcontent = body
            if 'id' in body_child[0].attrs:
                if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
                    pcontent = body_child[0]
            # pcontent = sourceContent.find("div", id="pcontent")
            pcontent = pcontent.find_all(recursive=False)[0]
            first_table = None
            for idx in range(len(pcontent.find_all(recursive=False))):
                t_part = pcontent.find_all(recursive=False)[idx]
                if t_part.name != "table":
                    break
                if idx == 0:
                    first_table = t_part
                else:
                    for _tr in t_part.find("tbody").find_all(recursive=False):
                        first_table.find("tbody").append(_tr)
                    t_part.clear()
        elif web_source_no == 'DX008357-11':
            body = sourceContent.find("body")
            body_child = body.find_all(recursive=False)
            pcontent = body
            if 'id' in body_child[0].attrs:
                if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
                    pcontent = body_child[0]
            # pcontent = sourceContent.find("div", id="pcontent")
            pcontent = pcontent.find_all(recursive=False)[0]
            error_table = []
            is_error_table = False
            for part in pcontent.find_all(recursive=False):
                if is_error_table:
                    if part.name == "table":
                        error_table.append(part)
                    else:
                        break
                if part.name == "div" and part.get_text(strip=True) == "中标候选单位：":
                    is_error_table = True
            first_table = None
            for idx in range(len(error_table)):
                t_part = error_table[idx]
                # if t_part.name != "table":
                #     break
                if idx == 0:
                    for _tr in t_part.find("tbody").find_all(recursive=False):
                        if _tr.get_text(strip=True) == "":
                            _tr.decompose()
                    first_table = t_part
                else:
                    for _tr in t_part.find("tbody").find_all(recursive=False):
                        if _tr.get_text(strip=True) != "":
                            first_table.find("tbody").append(_tr)
                    t_part.clear()
        elif web_source_no == '18021-2':
            body = sourceContent.find("body")
            body_child = body.find_all(recursive=False)
            pcontent = body
            if 'id' in body_child[0].attrs:
                if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
                    pcontent = body_child[0]
            # pcontent = sourceContent.find("div", id="pcontent")
            td = pcontent.find_all("td")
            for _td in td:
                if str(_td.string).strip() == "报价金额":
                    _td.string = "单价"
        elif web_source_no == '13740-2':
            # “xxx成为成交供应商”
            re_match = re.search("[^，。]+成为[^，。]*成交供应商", sourceContent)
            if re_match:
                sourceContent = sourceContent.replace(re_match.group(), "成交人：" + re_match.group())
        elif web_source_no == '03786-10':
            ser1 = re.search('中标价：([\d,.]+)', sourceContent)
            ser2 = re.search('合同金额[(（]万元[)）]：([\d,.]+)', sourceContent)
            if ser1 and ser2:
                m1 = ser1.group(1).replace(',', '')
                m2 = ser2.group(1).replace(',', '')
                if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'):
                    new = '中标价(万元)：' + m1
                    sourceContent = sourceContent.replace(ser1.group(0), new, 1)
        elif web_source_no=='00076-4':
            ser = re.search('主要标的数量：([0-9一]+)\w{,3}，主要标的单价：([\d,.]+)元?，合同金额：(.00)，', sourceContent)
            if ser:
                num = ser.group(1).replace('一', '1')
                try:
                    num = 1 if num == '0' else num
                    unit_price = ser.group(2).replace(',', '')
                    total_price = str(int(num) * float(unit_price))
                    new = '合同金额：' + total_price
                    sourceContent = sourceContent.replace('合同金额：.00', new, 1)
                except Exception as e:
                    log('preprocessing.py special_treatment exception')
        elif web_source_no=='DX000105-2':
            if re.search("成交公示", sourceContent) and re.search('，投标人：', sourceContent) and re.search('，成交人：', sourceContent)==None:
                sourceContent = sourceContent.replace('，投标人：', '，成交人：')
        elif web_source_no in ['03795-1', '03795-2']:
            if re.search('中标单位如下', sourceContent) and re.search('，投标人：', sourceContent) and re.search('，中标人：', sourceContent)==None:
                sourceContent = sourceContent.replace('，投标人：', '，中标人：')
        elif web_source_no in ['04080-3', '04080-4']:
            ser = re.search('合同金额：([0-9,]+.[0-9]{3,})(.{,4})', sourceContent)
            if ser and '万' not in ser.group(2):
                sourceContent = sourceContent.replace('合同金额：', '合同金额(万元)：')
        elif web_source_no=='03761-3':
            ser = re.search('中标价，([0-9]+)[.0-9]*%', sourceContent)
            if ser and int(ser.group(1))>100:
                sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元')
        elif web_source_no=='00695-7':
            ser = re.search('支付金额：', sourceContent)
            if ser:
                sourceContent = sourceContent.replace('支付金额：', '合同金额：')
        elif web_source_no=='00811-8':
            if re.search('是否中标：是', sourceContent) and re.search('排名：\d，', sourceContent):
                sourceContent = re.sub('排名：\d，', '候选', sourceContent)
        elif web_source_no=='DX000726-6':
            sourceContent = re.sub('卖方[:：\s]+宝山钢铁股份有限公司', '招标单位：宝山钢铁股份有限公司', sourceContent)
        return sourceContent
    except Exception as e:
        log('特殊数据源: %s 预处理特别修改抛出异常: %s'%(web_source_no, e))
        return sourceContent

def article_limit(soup,limit_words=30000):
    sub_space = re.compile("\s+")
    def soup_limit(_soup,_count,max_count=30000,max_gap=500):
        """
        :param _soup: soup
        :param _count: 当前字数
        :param max_count: 字数最大限制
        :param max_gap: 超过限制后的最大误差
        :return:
        """
        _gap = _count - max_count
        _is_skip = False
        next_soup = None
        while len(_soup.find_all(recursive=False)) == 1 and \
                _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
            _soup = _soup.find_all(recursive=False)[0]
        if len(_soup.find_all(recursive=False)) == 0:
            _soup.string = str(_soup.get_text())[:max_count-_count]
            _count += len(re.sub(sub_space, "", _soup.string))
            _gap = _count - max_count
            next_soup = None
        else:
            for _soup_part in _soup.find_all(recursive=False):
                if not _is_skip:
                    _count += len(re.sub(sub_space, "", _soup_part.get_text()))
                    if _count >= max_count:
                        _gap = _count - max_count
                        if _gap <= max_gap:
                            _is_skip = True
                        else:
                            _is_skip = True
                            next_soup = _soup_part
                            _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
                            continue
                else:
                    _soup_part.decompose()
        return _count,_gap,next_soup

    text_count = 0
    have_attachment = False
    attachment_part = None
    for child in soup.find_all(recursive=True):
        if child.name == 'div' and 'class' in child.attrs:
            if "richTextFetch" in child['class']:
                child.insert_before("##attachment##。") # 句号分开，避免项目名称等提取
                attachment_part = child
                have_attachment = True
                break
    if not have_attachment:
        # 无附件
        if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
            text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)

    else:
        # 有附件
        _text = re.sub(sub_space, "", soup.get_text())
        _text_split = _text.split("##attachment##")
        if len(_text_split[0])>limit_words:
            main_soup = attachment_part.parent
            main_text = main_soup.find_all(recursive=False)[0]
            text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
        if len(_text_split[1])>limit_words:
            # attachment_html纯文本，无子结构
            if len(attachment_part.find_all(recursive=False))==0:
                attachment_part.string = str(attachment_part.get_text())[:limit_words]
            else:
                attachment_text_nums = 0
                attachment_skip = False
                for part in attachment_part.find_all(recursive=False):
                    if not attachment_skip:
                        last_attachment_text_nums = attachment_text_nums
                        attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text()))
                        if attachment_text_nums>=limit_words:
                            part.string = str(part.get_text())[:limit_words-last_attachment_text_nums]
                            attachment_skip = True
                    else:
                        part.decompose()
    return soup

def attachment_filelink(soup):
    have_attachment = False
    attachment_part = None
    for child in soup.find_all(recursive=True):
        if child.name == 'div' and 'class' in child.attrs:
            if "richTextFetch" in child['class']:
                attachment_part = child
                have_attachment = True
                break
    if not have_attachment:
        return soup
    else:
        # 附件类型：图片、表格
        attachment_type = re.compile("\.(?:png|jpg|jpeg|tif|bmp|xlsx|xls)$")
        attachment_dict = dict()
        for _attachment in attachment_part.find_all(recursive=False):
            if _attachment.name == 'div' and 'filemd5' in _attachment.attrs:
                # print('filemd5',_attachment['filemd5'])
                attachment_dict[_attachment['filemd5']] = _attachment
        # print(attachment_dict)
        for child in soup.find_all(recursive=True):
            if child.name == 'div' and 'class' in child.attrs:
                if "richTextFetch" in child['class']:
                    break
            if "filelink" in child.attrs and child['filelink'] in attachment_dict:
                if re.search(attachment_type,str(child.string).strip()) or \
                        ('original' in child.attrs and re.search(attachment_type,str(child['original']).strip())) or \
                        ('href' in child.attrs and re.search(attachment_type,str(child['href']).strip())):
                    # 附件插入正文标识
                    child.insert_before("。##attachment_begin##")
                    child.insert_after("。##attachment_end##")
                    child.replace_with(attachment_dict[child['filelink']])

        # print('格式化输出',soup.prettify())
        return soup

def del_achievement(text):
    if re.search('中标|成交|入围|结果|评标|开标|候选人', text[:500]) == None or re.search('业绩', text) == None:
        return text
    p0 = '[，。；](（\d{1,2}）|\d{1,2}、)[\w、]{,8}：|(（\d{1,2}）|\d{1,2}、)|。' # 例子 264392818
    p1 = '业绩[：，](\d、[-\w（）、]{6,30}(工程|项目|勘察|设计|施工|监理|总承包|采购|更新)[\w（）]{,10}[，；])+' # 例子 257717618
    p2 = '(类似业绩情况：|业绩：)(\w{,20}：)?((（\d）|\d、)项目名称：[-\w（），；、\d\s：]{5,100}[；。])+' # 例子 264345826
    p3 = '(投标|类似|(类似)?项目|合格|有效|企业|工程)?业绩(名称|信息|\d)?：(项目名称：)?[-\w（）、]{6,50}(项目|工程|勘察|设计|施工|监理|总承包|采购|更新)'

    l = []
    tmp = []
    for it in re.finditer(p0, text):
        if it.group(0)[-3:] in ['业绩：', '荣誉：']:
            if tmp != []:
                del_text = text[tmp[0]:it.start()]
                l.append(del_text)
                tmp = []
            tmp.append(it.start())
        elif tmp != []:
            del_text = text[tmp[0]:it.start()]
            l.append(del_text)
            tmp = []
    if tmp != []:
        del_text = text[tmp[0]:]
        l.append(del_text)
    for del_text in l:
        text = text.replace(del_text, '')
        # print('删除业绩信息：', del_text)

    for rs in re.finditer(p1, text):
        # print('删除业绩信息：', rs.group(0))
        text = text.replace(rs.group(0), '')

    for rs in re.finditer(p2, text):
        # print('删除业绩信息：', rs.group(0))
        text = text.replace(rs.group(0), '')

    for rs in re.finditer(p3, text):
        # print('删除业绩信息：', rs.group(0))
        text = text.replace(rs.group(0), '')
    return text

def del_tabel_achievement(soup):
    if re.search('中标|成交|入围|结果|评标|开标|候选人', soup.text[:800]) == None or re.search('业绩', soup.text)==None:
        return None
    p1 = '(中标|成交)(单位|候选人)的?(企业|项目|项目负责人|\w{,5})?业绩|类似(项目)?业绩|\w{,10}业绩$|业绩(公示|情况|荣誉)'
    '''删除前面标签 命中业绩规则；当前标签为表格且公布业绩相关信息的去除'''
    for tag in soup.find_all('table'):
        pre_text = tag.findPreviousSibling().text.strip() if tag.findPreviousSibling() != None else ""
        tr_text = tag.find('tr').text.strip() if tag.find('tr') != None else ""
        #     print(re.search(p1, pre_text),pre_text, len(pre_text), re.findall('序号|中标候选人名称|项目名称|工程名称|合同金额|建设单位|业主', tr_text))
        if re.search(p1, pre_text) and len(pre_text) < 20 and tag.find('tr') != None and len(tr_text)<100:
            _count = 0
            for td in tag.find('tr').find_all('td'):
                td_text = td.text.strip()
                if len(td_text) > 25:
                    break
                if len(td_text) < 25 and re.search('中标候选人|(项目|业绩|工程)名称|\w{,10}业绩$|合同金额|建设单位|采购单位|业主|甲方', td_text):
                    _count += 1
                if _count >=2:
                    pre_tag = tag.findPreviousSibling().extract()
                    del_tag = tag.extract()
                    # print('删除表格业绩内容', pre_tag.text + del_tag.text)
                    break
        elif re.search('业绩名称', tr_text) and re.search('建设单位|采购单位|业主', tr_text) and len(tr_text)<100:
            del_tag = tag.extract()
            # print('删除表格业绩内容', del_tag.text)
    del_trs = []
    '''删除表格某些行公布的业绩信息'''
    for tag in soup.find_all('table'):
        text = tag.text
        if re.search('业绩', text) == None:
            continue
        # for tr in tag.find_all('tr'):
        trs = tag.find_all('tr')
        i = 0
        while i < len(trs):
            tr = trs[i]
            if len(tr.find_all('td'))==2 and tr.td!=None and tr.td.findNextSibling()!=None:
                td1_text =tr.td.text
                td2_text =tr.td.findNextSibling().text
                if re.search('业绩', td1_text)!=None and len(td1_text)<10 and len(re.findall('(\d、|（\d）)?[-\w（）、]+(工程|项目|勘察|设计|施工|监理|总承包|采购|更新)', td2_text))>=2:
                    # del_tag = tr.extract()
                    # print('删除表格业绩内容', del_tag.text)
                    del_trs.append(tr)
            elif tr.td != None and re.search('^业绩|业绩$', tr.td.text.strip()) and len(tr.td.text.strip())<25:
                rows = tr.td.attrs.get('rowspan', '')
                cols = tr.td.attrs.get('colspan', '')
                if rows.isdigit() and int(rows)>2:
                    for j in range(int(rows)):
                        if i+j < len(trs):
                            del_trs.append(trs[i+j])
                    i += j
                elif cols.isdigit() and int(cols)>3 and len(tr.find_all('td'))==1 and i+2 < len(trs):
                    next_tr_cols = 0
                    td_num = 0
                    for td in trs[i+1].find_all('td'):
                        td_num += 1
                        if td.attrs.get('colspan', '').isdigit():
                            next_tr_cols += int(td.attrs.get('colspan', ''))
                    if next_tr_cols == int(cols):
                        del_trs.append(tr)
                        for j in range(1,len(trs)-i):
                            if len(trs[i+j].find_all('td')) == 1:
                                break
                            elif len(trs[i+j].find_all('td')) >= td_num-1:
                                del_trs.append(trs[i+j])
                            else:
                                break
                        i += j
            i += 1
        for tr in del_trs:
            del_tag = tr.extract()
            # print('删除表格业绩内容', del_tag.text)

def split_header(soup):
    '''
    处理 空格分割多个表头的情况 ： 主要标的名称      规格型号（或服务要求）      主要标的数量      主要标的单价      合同金额（万元）
    :param soup: bs4  soup 对象
    :return:
    '''
    header = []
    attrs = []
    flag = 0
    tag = None
    for p in soup.find_all('p'):
        text = p.get_text()
        if re.search('主要标的数量\s+主要标的单价(（万?元）)?\s+合同金额', text):
            header = re.split('\s{3,}', text) if re.search('\s{3,}', text) else re.split('\s+', text)
            flag = 1
            tag = p
            tag.string = ''
            continue
        if flag:
            attrs = re.split('\s{3,}', text) if re.search('\s{3,}', text) else re.split('\s+', text)
            if header and len(header) == len(attrs) and tag:
                s = ""
                for head, attr in zip(header, attrs):
                    s += head + '：' + attr + '，'
                # tag.string = s
                # p.extract()
                p.string = s
            else:
                break


def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
    '''
    :param articles: 待处理的article source html
    :param useselffool: 是否使用selffool
    :return: list_articles
    '''
    list_articles = []
    for article in articles:
        doc_id = article[0]
        sourceContent = article[1]
        sourceContent_raw = article[1] # 原始html数据，fingerprint计算MD5用
        sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
        sourceContent = re.sub("##attachment##","",sourceContent)
        sourceContent = sourceContent.replace('<br/>', '<br>')
        sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
        # for br_match in re.findall("[^>]+?<br>",sourceContent):
        #     _new = re.sub("<br>","",br_match)
        #     # <br>标签替换为<p>标签
        #     if not re.search("^\s+$",_new):
        #         _new = '<p>'+_new + '</p>'
        #         # print(br_match,_new)
        #         sourceContent = sourceContent.replace(br_match,_new,1)
        _send_doc_id = article[3]
        _title = article[4]
        _title_raw = article[4]
        page_time = article[5]
        web_source_no = article[6]
        '''特别数据源对 html 做特别修改'''
        if web_source_no in ['DX000202-1']:
            sourceContent = special_treatment(sourceContent, web_source_no)

        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
        article_processed = BeautifulSoup(sourceContent,"lxml")

        if re.search('主要标的数量(&nbsp;|\s)+主要标的单价(（万?元）)?(&nbsp;|\s)+合同金额', sourceContent): #处理 空格分割多个表头的情况
            split_header(article_processed)

        '''表格业绩内容删除'''
        del_tabel_achievement(article_processed)

        '''特别数据源对 BeautifulSoup(html) 做特别修改'''
        if web_source_no in ["00753-14","DX008357-11","18021-2"]:
            article_processed = special_treatment(article_processed, web_source_no)
        for _soup in article_processed.descendants:
            # 识别无标签文本，添加<span>标签
            if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
                # print(_soup.parent.string,_soup.string.strip())
                _soup.wrap(article_processed.new_tag("span"))
        # print(article_processed)
        # 正文和附件内容限制字数30000
        article_processed = article_limit(article_processed, limit_words=30000)
        # 把每个附件识别对应的html放回原来出现的位置
        article_processed = attachment_filelink(article_processed)

        article_processed = get_preprocessed_outline(article_processed)
        # print('article_processed')
        article_processed = tableToText(article_processed)
        article_processed = segment(article_processed)

        article_processed = article_processed.replace('(', '（').replace(')', '）')  #2022/8/10 统一为中文括号
        # article_processed = article_processed.replace(':', '：')  #2023/1/5 统一为中文冒号
        article_processed = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])", "：", article_processed)
        article_processed = article_processed.replace('．','.').replace('－', '-') # 2021/12/01 修正OCR识别PDF小数点错误问题
        article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
        article_processed = article_processed.replace('成交工程价款', '成交工程价')  # 2021/12/21 修正为中标价
        article_processed = re.sub('任务(?=编号[:：])', '项目',article_processed)  # 2022/08/10 修正为项目编号
        article_processed = article_processed.replace('招标（建设）单位', '招标单位')  #2022/8/10 修正预测不到表达
        article_processed = re.sub("采购商(?=[^\u4e00-\u9fa5]|名称)", "招标人", article_processed)
        article_processed = re.sub('(招标|采购)人(概况|信息)：?[，。]', '采购人信息：', article_processed)  # 2022/8/10统一表达
        article_processed = article_processed.replace('\（%）', '')    # 中标（成交）金额（元）\（%）：498888.00， 处理 江西省政府采购网  金额特殊问题
        article_processed = re.sub('金额：?（(可填写下浮率?、折扣率?或费率|拟签含税总单价总计|[^万元（）\d]{8,20})）：?', '金额：', article_processed)    # 中标（成交）金额：（可填写下浮率、折扣率或费率）：29.3万元  金额特殊问题
        article_processed = re.sub('（不?含(可抵扣增值|\w{,8})税）', '', article_processed)    # 120637247 投标报价（元），（含可抵扣增值税）：277,560.00。
        article_processed = re.sub('供应商的?(名称[及其、]{1,2}地址|联系方式：名称)', '供应商名称', article_processed)  # 18889217, 84422177
        article_processed = re.sub('，最高有效报价者：', '，中标人名称：', article_processed)  # 224678159 # 2023/7/4 四川站源特殊中标修改
        article_processed = re.sub('，最高有效报价：', '，投标报价：', article_processed)  # 224678159 # 2023/7/4 四川站源特殊中标修改
        article_processed = re.sub('备选中标人', '第二候选人', article_processed)  # 341344142 # 2023/7/17 特殊表达修改
        if web_source_no.startswith('DX002756-'):
            article_processed = re.sub('状态：(进行中|已结束)单位', '，项目单位', article_processed)  # 376225646
        if web_source_no.startswith('DX006116-') and re.search('结果公告如下：.{5,50}，单位名称：', article_processed):  # 2023/11/20 特殊处理 381591924 381592533 这种提取不到情况
            article_processed = re.sub('，单位名称：', '，供应商名称：', article_processed)
        ser = re.search('(采购|招标|比选)人(名称)?/(采购|招标|比选)?代理机构(名称)?：(?P<tenderee>[\w（）]{4,25}(/[\w（）]{4,25})?)/(?P<agency>[\w（）]{4,25})[，。]', article_processed)
        if ser:
            article_processed = article_processed.replace(ser.group(0), '采购人名称：%s，采购代理机构名称：%s，' % (ser.group('tenderee'), ser.group('agency')))

        ser2 = re.search('(采购|招标)人(名称)?/(采购|招标)?代理机构(名称)?：(?P<tenderee>[\w（）]{4,25})[，。/]', article_processed)
        if ser2:
            article_processed = article_processed.replace(ser2.group(0), '采购人名称：%s，采购代理机构名称：，' % (
            ser2.group('tenderee')))
        if re.search('中标单位名称：[\w（）]{5,25}，中标候选人名次：\d，', article_processed) and re.search('中标候选人名次：\d，中标单位名称：[\w（）]{5,25}，', article_processed)==None:  # 处理类似 304706608 此篇的数据源正文特殊表达
            for it in re.finditer('(?P<tenderer>(中标单位名称：[\w（）]{5,25}，))(?P<rank>(中标候选人名次：\d，))', article_processed):
                article_processed = article_processed.replace(it.group(0), it.group('rank')+it.group('tenderer'))

        '''去除业绩内容'''
        article_processed = del_achievement(article_processed)
        # 修复OCR金额中“，”、“。”识别错误
        article_processed_list = article_processed.split("##attachment##")
        if len(article_processed_list)>1:
            attachment_text = article_processed_list[1]
            for _match in re.finditer("\d。\d{2}",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
            # for _match in re.finditer("(\d，\d{3})[,，.]",attachment_text):
            for _match in re.finditer("\d，(?=\d{3}[^\d])",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("，",","),1)
            article_processed_list[1] = attachment_text
            article_processed = "##attachment##".join(article_processed_list)
        '''特别数据源对 预处理后文本 做特别修改'''
        if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2", '00811-8', '03795-1', '03795-2', 'DX000726-6']:
            article_processed = special_treatment(article_processed, web_source_no)

        # 提取bidway
        list_bidway = extract_bidway(article_processed, _title)
        if list_bidway:
            bidway = list_bidway[0].get("body")
            # bidway名称统一规范
            bidway = bidway_integrate(bidway)
        else:
            bidway = ""

        # 修正被"，"逗号分隔的时间
        repair_time = re.compile("[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[:时点]，?[0-6]\d分?|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[时点]|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]|"
                                 "[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d"
                                 )
        for _time in set(re.findall(repair_time,article_processed)):
            if re.search("，",_time):
                _time2 = re.sub("，", "", _time)
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time2)
                if item:
                    _time2 = _time2.replace(item.group(),item.group() + " ")
                article_processed = article_processed.replace(_time, _time2)
            else:
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time)
                if item:
                    _time2 = _time.replace(item.group(),item.group() + " ")
                    article_processed = article_processed.replace(_time, _time2)
        # print('re_rtime',re.findall(repair_time,article_processed))
        # log(article_processed)

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += round(time.time()-start_time,2)

        #article_processed = article[1]
        _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
                           bidway=bidway)
        _article.fingerprint = getFingerprint(_title_raw+sourceContent_raw)
        _article.page_time = page_time
        list_articles.append(_article)
    return list_articles

def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
    '''

    :param list_articles: 经过预处理的article text
    :return: list_sentences
    '''
    list_sentences = []
    list_outlines = []
    for article in list_articles:
        list_sentences_temp = []
        list_entitys_temp = []
        doc_id = article.id
        _send_doc_id = article.doc_id
        _title = article.title
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = article.content

        if len(_title)<100 and _title not in article_processed: # 把标题放到正文
            article_processed = _title + '，' + article_processed   # 2023/01/06 标题正文加逗号分割，预防标题后面是产品，正文开头是公司实体，实体识别把产品和公司作为整个角色实体

        attachment_begin_index = -1

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += time.time()-start_time

        #nlp处理
        if article_processed is not None and len(article_processed)!=0:
            split_patten = "。"
            sentences = []
            _begin = 0
            sentences_set = set()
            for _iter in re.finditer(split_patten,article_processed):
                _sen = article_processed[_begin:_iter.span()[1]]
                if len(_sen)>0 and _sen not in sentences_set:
                    # 标识在附件里的句子
                    if re.search("##attachment##",_sen):
                        attachment_begin_index = len(sentences)
                        # _sen = re.sub("##attachment##","",_sen)
                    sentences.append(_sen)
                    sentences_set.add(_sen)
                _begin = _iter.span()[1]
            _sen = article_processed[_begin:]
            if re.search("##attachment##", _sen):
                # _sen = re.sub("##attachment##", "", _sen)
                attachment_begin_index = len(sentences)
            if len(_sen)>0 and _sen not in sentences_set:
                sentences.append(_sen)
                sentences_set.add(_sen)
            # 解析outline大纲分段
            outline_list = []
            if re.search("##split##",article.content):
                temp_sentences = []
                last_sentence_index = (-1,-1)
                outline_index = 0
                for sentence_index in range(len(sentences)):
                    sentence_text = sentences[sentence_index]
                    for _ in re.findall("##split##", sentence_text):
                        _match = re.search("##split##", sentence_text)
                        if last_sentence_index[0] > -1:
                            sentence_begin_index,wordOffset_begin = last_sentence_index
                            sentence_end_index = sentence_index
                            wordOffset_end = _match.start()
                            if sentence_begin_index<attachment_begin_index and sentence_end_index>=attachment_begin_index:
                                outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1])))
                            else:
                                outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end))
                            outline_index += 1
                        sentence_text = re.sub("##split##，?", "", sentence_text,count=1)
                        last_sentence_index = (sentence_index,_match.start())
                    temp_sentences.append(sentence_text)
                if attachment_begin_index>-1 and last_sentence_index[0]<attachment_begin_index:
                    outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],attachment_begin_index-1,last_sentence_index[1],len(temp_sentences[attachment_begin_index-1])))
                else:
                    outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],len(sentences)-1,last_sentence_index[1],len(temp_sentences[-1])))
                sentences = temp_sentences
            #解析outline的outline_text内容
            for _outline in outline_list:
                if _outline.sentence_begin_index==_outline.sentence_end_index:
                    _text = sentences[_outline.sentence_begin_index][_outline.wordOffset_begin:_outline.wordOffset_end]
                else:
                    _text = ""
                    for idx in range(_outline.sentence_begin_index,_outline.sentence_end_index+1):
                        if idx==_outline.sentence_begin_index:
                            _text += sentences[idx][_outline.wordOffset_begin:]
                        elif idx==_outline.sentence_end_index:
                            _text += sentences[idx][:_outline.wordOffset_end]
                        else:
                            _text += sentences[idx]
                _outline.outline_text = _text
                _outline_summary = re.split("[:：，]",_text,1)[0]
                if len(_outline_summary)<30:
                    _outline.outline_summary = _outline_summary
                # print(_outline.outline_index,_outline.outline_text)

            article.content = "".join(sentences)
            # sentences.append(article_processed[_begin:])

            lemmas = []
            doc_offsets = []
            dep_types = []
            dep_tokens = []

            time1 = time.time()

            '''
            tokens_all = fool.cut(sentences)
            #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
            #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
            ner_entitys_all = fool.ner(sentences)
            '''
            #限流执行
            key_nerToken = "nerToken"
            start_time = time.time()
            # tokens_all = getTokens(sentences,useselffool=useselffool)
            tokens_all = getTokens([re.sub("##attachment_begin##|##attachment_end##","",_sen) for _sen in sentences],useselffool=useselffool)
            if key_nerToken not in cost_time:
                cost_time[key_nerToken] = 0
            cost_time[key_nerToken] += round(time.time()-start_time,2)

            in_attachment = False
            for sentence_index in range(len(sentences)):
                sentence_text = sentences[sentence_index]
                if re.search("##attachment_begin##",sentence_text):
                    in_attachment = True
                    sentence_text = re.sub("##attachment_begin##","",sentence_text)
                if re.search("##attachment_end##",sentence_text):
                    in_attachment = False
                    sentence_text = re.sub("##attachment_end##", "", sentence_text)
                if sentence_index >= attachment_begin_index and attachment_begin_index!=-1:
                    in_attachment = True
                tokens = tokens_all[sentence_index]

                #pos_tag = pos_all[sentence_index]
                pos_tag = ""

                ner_entitys = ""

                list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment))
        if len(list_sentences_temp)==0:
            list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
        list_sentences.append(list_sentences_temp)
        list_outlines.append(outline_list)
        article.content = re.sub("##attachment_begin##|##attachment_end##", "", article.content)
    return list_sentences,list_outlines

def get_money_entity(sentence_text, found_yeji, in_attachment=False):
    money_list = []
    # 使用正则识别金额
    entity_type = "money"
    list_money_pattern = {"cn": "(()(?P<filter_kw>百分之)?(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
                          "key_word": "((?P<text_key_word>(?:[￥¥]+，?|[单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果)(?:[,，\[（\(]*\s*(人民币|单位：)?/?(?P<unit_key_word_before>[万亿]?(?:[美日欧]元|元(/(M2|[\u4e00-\u9fa5]{1,3}))?)?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[\]）\)]?)\s*[，,:：]*(RMB|USD|EUR|JPY|CNY)?[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P<money_key_word>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?[百千]{,1})(?P<science_key_word>(E-?\d+))?(?:[（\(]?(?P<filter_>[%％‰折])*\s*，?((金额)?单位[:：])?(?P<unit_key_word_behind>[万亿]?(?:[美日欧]元|元)?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[）\)]?))",
                          "front_m": "((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万亿]?(?:[美日欧]元|元))\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,7}?))(?P<money_front_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?:，?)[百千]*)(?P<science_front_m>(E-?\d+))?())",
                          "behind_m": "(()()(?P<money_behind_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?:，?)[百千]*)(?P<science_behind_m>(E-?\d+))?(人民币)?[\(（]?(?P<unit_behind_m>[万亿]?(?:[美日欧]元|元)(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\)）]?)"}
    # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。

    pattern_money = re.compile("%s|%s|%s|%s" % (
    list_money_pattern["cn"], list_money_pattern["key_word"], list_money_pattern["behind_m"],
    list_money_pattern["front_m"]))

    if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text):
        found_yeji += 1
    if found_yeji >= 2:  # 过滤掉业绩后面的所有金额
        all_match = []
    else:
        ser = re.search('((收费标准|计算[方公]?式)：|\w{3,5}\s*=)+\s*[中标投标成交金额招标人预算价格万元\s（）()\[\]【】\d\.%％‰\+\-*×/]{20,}[，。]?', sentence_text)  # 过滤掉收费标准里面的金额
        if ser:
            all_match = re.finditer(pattern_money, sentence_text.replace(ser.group(0), ' ' * len(ser.group(0))))
        else:
            all_match = re.finditer(pattern_money, sentence_text)
    for _match in all_match:
        # print('_match: ', _match.group())
        if len(_match.group()) > 0:
            # print("===",_match.group())
            # # print(_match.groupdict())
            notes = ''  # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
            unit = ""
            entity_text = ""
            start_index = ""
            end_index = ""
            text_beforeMoney = ""
            filter = ""
            filter_unit = False
            notSure = False
            science = ""
            if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text[:_match.span()[0]]):  # 2021/7/21过滤掉业绩后面金额
                # print('金额在业绩后面: ', _match.group(0))
                found_yeji += 1
                break
            for k, v in _match.groupdict().items():
                if v != "" and v is not None:
                    if k == 'text_key_word':
                        notSure = True
                    if k.split("_")[0] == "money":
                        entity_text = v
                        # print(_match.group(k), 'entity_text: ', sentence_text[_match.start(k): _match.end(k)])
                        if entity_text.endswith(',00'):  # 金额逗号后面不可能为两个0结尾，应该小数点识别错，直接去掉
                            entity_text = entity_text[:-3]
                    if k.split("_")[0] == "unit":
                        if v == '万元' or unit == "":  # 处理  预算金额(元)：160万元 这种出现前后单位不一致情况
                            unit = v
                    if k.split("_")[0] == "text":
                        # print('text_before: ', _match.group(k))
                        text_beforeMoney = v
                    if k.split("_")[0] == "filter":
                        filter = v
                    if re.search("filter_unit", k) is not None:
                        filter_unit = True
                    if k.split("_")[0] == 'science':
                        science = v
            # print("金额：{0} ,单位：{1}, 前文：{2}, filter: {3}, filter_unit: {4}".format(entity_text,unit,text_beforeMoney,filter,filter_unit))
            # if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()):  # 2021/7/19 修正OCR识别小数点为逗号
            #     if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0] - 2):_match.span()[0]]):
            #         entity_text = re.sub('\d+,', '', entity_text)
            #     else:
            #         entity_text = entity_text.replace(',', '.')
            #     # print(' 修正OCR识别小数点为逗号')

            if filter != "":
                continue
            if len(entity_text)>30 or len(re.sub('[E-]', '', science))>2: # 限制数字长度，避免类似265339018附件金额错误，数值超大报错 decimal.InvalidOperation
                continue
            start_index, end_index = _match.span()
            start_index += len(text_beforeMoney)

            '''过滤掉手机号码作为金额'''
            if re.search('电话|手机|联系|方式|编号|编码|日期|数字|时间', text_beforeMoney):
                # print('过滤掉手机号码作为金额')
                continue
            elif re.search('^1[3-9]\d{9}$', entity_text) and re.search('：\w{1,3}$', text_beforeMoney): # 过滤掉类似 '13863441880', '金额（万元）：季勇13863441880'
                # print('过滤掉手机号码作为金额')
                continue

            if unit == "":  # 2021/7/21 有明显金额特征的补充单位，避免被过滤
                if (re.search('(￥|¥|RMB|CNY)[:：]?$', text_beforeMoney) or re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,}', entity_text)):
                    if entity_text.endswith('万元'):
                        unit = '万元'
                        entity_text = entity_text[:-2]
                    else:
                        unit = '元'
                    # print('1明显金额特征补充单位 元')
                elif re.search('USD[:：]?$', text_beforeMoney):
                    unit = '美元'
                elif re.search('EUR[:：]?$', text_beforeMoney):
                    unit = '欧元'
                elif re.search('JPY[:：]?$', text_beforeMoney):
                    unit = '日元'
                elif re.search('^[-—]+[\d,.]+万元', sentence_text[end_index:]):
                    # print('两个金额连接后面的有单位，用后面单位')
                    unit = '万元'
                elif re.search('([单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费)[:：为]*-?$', text_beforeMoney.strip()) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}', entity_text) == None:
                    if re.search('^[\d，,.]+$', entity_text) and float(re.sub('[,，]', '', entity_text))<500 and re.search('万元', sentence_text):
                        unit = '万元'
                        # print('金额较小且句子中有万元的，补充单位为万元')
                    elif re.search('^\d{1,3}\.\d{4,6}$', entity_text) and re.search('0000$', entity_text) == None:
                        unit = '万元'
                    else:
                        unit = '元'
                        # print('金额前面紧接关键词的补充单位 元')
                elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)', entity_text):
                    unit = '元'
                    # print('3明显金额特征补充单位 元')
                else:
                    # print('过滤掉没单位金额: ',entity_text)
                    continue
            elif unit == '万元':
                if end_index < len(sentence_text) and sentence_text[end_index] == '元' and re.search('\d$', entity_text):
                    unit = '元'
                elif re.search('^[5-9]\d{6,}\.\d{2}$', entity_text): # 五百亿以上的万元改为元
                    unit = '元'
            if unit.find("万") >= 0 and entity_text.find("万") >= 0:  # 2021/7/19修改为金额文本有万，不计算单位
                # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit)
                unit = "元"
            if re.search('.*万元万元', entity_text):  # 2021/7/19 修正两个万元
                # print(' 修正两个万元',entity_text)
                entity_text = entity_text.replace('万元万元', '万元')
            else:
                if filter_unit:
                    continue

            # symbol = '-' if entity_text.startswith('-') and not entity_text.startswith('--') and re.search('\d+$', sentence_text[:begin_index_temp]) == None else ''  # 负值金额前面保留负号 ，后面这些不作为负金额 起拍价：105.29-200.46万元  预 算 --- 350000.0 2023/04/14 取消符号

            entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", entity_text)
            # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
            if re.search('总投资|投资总额|总预算|总概算|投资规模|批复概算|投资额',
                         sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]):  # 2021/8/5过滤掉总投资金额
                # print('总投资金额: ', _match.group(0))
                notes = '总投资'
            elif re.search('投资|概算|建安费|其他费用|基本预备费',
                           sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/11/18 投资金额不作为招标金额
                notes = '投资'
            elif re.search('工程造价',
                           sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/12/20 工程造价不作为招标金额
                notes = '工程造价'
            elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
                  or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\(（]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\)）]*[:：为]',
                               sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
                  or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
                               sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
                notes = '保证金'
                # print('保证金信息：', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
            elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
                           sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
                notes = '成本警戒线'
            elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为：]', sentence_text[_match.span()[0]:_match.span()[1]]):
                cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
                notes = cost_re.group(1)
            elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
                notes = '单价'
            elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
                notes = '大写'
                if entity_text[0] == "拾":  # 2021/12/16 修正大写金额省略了数字转换错误问题
                    entity_text = "壹" + entity_text
                # print("补充备注：notes = 大写")
            if len(unit) > 0:
                if unit.find('万') >= 0 and len(entity_text.split('.')[0]) >= 8:  # 2021/7/19 修正万元金额过大的情况
                    # print('修正单位万元金额过大的情况 金额：', entity_text, '单位:', unit)
                    entity_text = str(
                        getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]) / 10000)
                    unit = '元'  # 修正金额后单位 重置为元
                else:
                    # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
                    entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]))
            else:
                if entity_text.find('万') >= 0 and entity_text.split('.')[0].isdigit() and len(
                        entity_text.split('.')[0]) >= 8:
                    entity_text = str(getUnifyMoney(entity_text) / 10000)
                    # print('修正金额字段含万 过大的情况')
                else:
                    entity_text = str(getUnifyMoney(entity_text))
            if science and re.search('^E-?\d+$', science):  # 科学计数
                entity_text = str(Decimal(entity_text + science)) if Decimal(entity_text + science) > 100 and Decimal(
                    entity_text + science) < 10000000000 else entity_text  # 结果大于100及小于100万才使用科学计算

            if float(entity_text) > 100000000000:  # float(entity_text)<100 or  2022/3/4 取消最小金额限制
                # print('过滤掉金额：float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
                continue

            if notSure and unit == "" and float(entity_text) > 100 * 10000:
                # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit)
                continue
            # print("金额：{0} ,单位：{1}, 前文：{2}, filter: {3}, filter_unit: {4}".format(entity_text, unit, text_beforeMoney,
            #                                                                      filter, filter_unit))
            if re.search('[%％‰折]|费率|下浮率', text_beforeMoney) and float(entity_text)<1000: # 过滤掉可能是费率的金额
                # print('过滤掉可能是费率的金额')
                continue
            money_list.append((entity_text, start_index, end_index, unit, notes))
    return money_list, found_yeji
def cut_repeat_name(s):
    '''
    公司连续重复名称去重
    :param s:
    :return:
    '''
    if len(s) >= 8:
        n = s.count(s[-4:])
        id = s.find(s[-4:]) + 4
        sub_s = s[:id]
        if n>=2 and s == sub_s * n:
            s = sub_s
    return s
def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
    '''

    :param list_sentences:分局情况
    :param cost_time:
    :return: list_entitys
    '''

    list_entitys = []
    not_extract_roles = ['黄埔军校', '国有资产管理处'] # 需要过滤掉的企业单位
    for list_sentence in list_sentences:
        sentences = []
        list_entitys_temp = []
        for _sentence in list_sentence:
            sentences.append(_sentence.sentence_text)

        time1 = time.time()

        '''
        tokens_all = fool.cut(sentences)
        #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
        #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
        ner_entitys_all = fool.ner(sentences)
        '''
        #限流执行
        key_nerToken = "nerToken"
        start_time = time.time()
        found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
        # found_pingbiao = False
        ner_entitys_all = getNers(sentences,useselffool=useselffool)
        if key_nerToken not in cost_time:
            cost_time[key_nerToken] = 0
        cost_time[key_nerToken] += round(time.time()-start_time,2)

        doctextcon_sentence_len = sum([1 for sentence in list_sentence if not sentence.in_attachment])
        company_dict = set()
        company_index = dict((i,set()) for i in range(len(list_sentence)))
        for sentence_index in range(len(list_sentence)):
            list_sentence_entitys = []

            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            in_attachment = list_sentence[sentence_index].in_attachment
            list_tokenbegin = []
            begin = 0
            for i in range(0,len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin+1)
            #pos_tag = pos_all[sentence_index]
            pos_tag = ""

            ner_entitys = ner_entitys_all[sentence_index]

            '''正则识别角色实体  经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
            for it in re.finditer(
                    '(?P<text_key_word>(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为:：]+)(?P<text>([（）\w]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[，。]',
                    sentence_text):
                for k, v in it.groupdict().items():
                    if k == 'text_key_word':
                        keyword = v
                    if k == 'text':
                        entity = v
                b = it.start() + len(keyword)
                e = it.end() - 1
                if (b, e, 'location', entity) in ner_entitys:
                    ner_entitys.remove((b, e, 'location', entity))
                    ner_entitys.append((b, e, 'company', entity))
                elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
                    ner_entitys.append((b, e, 'company', entity))

            for it in re.finditer(
                    '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为:：]+)(?P<text>\w{2,4}[省市县区镇]([（）\w]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[，。]',
                    sentence_text):
                for k, v in it.groupdict().items():
                    if k == 'text_key_word':
                        keyword = v
                    if k == 'text':
                        entity = v
                b = it.start() + len(keyword)
                e = it.end() - 1
                if (b, e, 'location', entity) in ner_entitys:
                    ner_entitys.remove((b, e, 'location', entity))
                    ner_entitys.append((b, e, 'org', entity))
                if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
                    ner_entitys.append((b, e, 'org', entity))

            for ner_entity in ner_entitys:
                if ner_entity[2] in ['company','org']:
                    company_dict.add((ner_entity[2],ner_entity[3]))
                    company_index[sentence_index].add((ner_entity[0],ner_entity[1]))
            #识别package

            ner_time_list = []
            #识别实体
            for ner_entity in ner_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]

                if entity_type=='time':
                    ner_time_list.append((begin_index_temp,end_index_temp))
                if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
                    continue
                # 实体长度限制
                if entity_type in ["org","company"] and len(entity_text)>30:
                    continue
                if entity_type == "person" and len(entity_text) > 20:
                    continue
                elif entity_type=="person" and len(entity_text)>10 and len(re.findall("[\u4e00-\u9fa5]",entity_text))<len(entity_text)/2:
                    continue
                # 识别不完整的组织机构补充
                if entity_type in ["org"]:
                    end_words = re.search("^[\u4e00-\u9fa5]{,5}(?:办公室|部|中心|处|会)",sentence_text[end_index_temp:end_index_temp+10])
                    if end_words:
                        entity_text = entity_text + end_words.group()

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                if entity_type!='time':
                    entity_text = re.sub("[,，。：!&@$\*\s]","",entity_text)
                entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text
                # 组织机构实体名称补充
                if entity_type in ["org", "company"]:
                    if entity_text in not_extract_roles: # 过滤掉名称在 需要过滤企业单位列表里的
                        continue
                    if not re.search("有限责任公司|有限公司",entity_text):
                        fix_name = re.search("(有限)([责贵]?任?)(公?司?)",entity_text)
                        if fix_name:
                            if len(fix_name.group(2))>0:
                                _text = fix_name.group()
                                if '司' in _text:
                                    entity_text = entity_text.replace(_text, "有限责任公司")
                                else:
                                    _text = re.search(_text + "[^司]{0,5}司", entity_text)
                                    if _text:
                                        _text = _text.group()
                                        entity_text = entity_text.replace(_text, "有限责任公司")
                                    else:
                                        entity_text = entity_text.replace(entity_text[fix_name.start():], "有限责任公司")
                            elif len(fix_name.group(3))>0:
                                _text = fix_name.group()
                                if '司' in _text:
                                    entity_text = entity_text.replace(_text, "有限公司")
                                else:
                                    _text = re.search(_text + "[^司]{0,3}司", entity_text)
                                    if _text:
                                        _text = _text.group()
                                        entity_text = entity_text.replace(_text, "有限公司")
                                    else:
                                        entity_text = entity_text.replace(entity_text[fix_name.start():], "有限公司")
                            elif re.search("有限$", entity_text):
                                entity_text = re.sub("有限$","有限公司",entity_text)
                    entity_text = entity_text.replace("有公司","有限公司")

                    '''下面对公司实体进行清洗'''
                    entity_text = re.sub('\s', '', entity_text)
                    if re.search('^(\d{4}年)?[\-\d月日份]*\w{2,3}分公司$', entity_text):  # 删除
                        # print('公司实体不符合规范：', entity_text)
                        continue
                    elif re.match('xx|XX', entity_text):  # 删除
                        # print('公司实体不符合规范：', entity_text)
                        continue
                    elif re.match('\.?(rar|zip|pdf|df|doc|docx|xls|xlsx|jpg|png)', entity_text):
                        entity_text = re.sub('\.?(rar|zip|pdf|df|doc|docx|xls|xlsx|jpg|png)', '', entity_text)
                    elif re.match(
                            '((\d{4}[年-])[\-\d:\s元月日份]*|\d{1,2}月[\d日.-]*(日?常?计划)?|\d{1,2}[.-]?|[A-Za-z](包|标段?)?|[a-zA-Z0-9]+-[a-zA-Z0-9-]*|[a-zA-Z]{1,2}|[①②③④⑤⑥⑦⑧⑨⑩]|\s|title\=|【[a-zA-Z0-9]+】|[^\w])[\u4e00-\u9fa5]+',
                            entity_text):
                        filter = re.match(
                            '((\d{4}[年-])[\-\d:\s元月日份]*|\d{1,2}月[\d日.-]*(日?常?计划)?|\d{1,2}[.-]?|[A-Za-z](包|标段?)?|[a-zA-Z0-9]+-[a-zA-Z0-9-]*|[a-zA-Z]{1,2}|[①②③④⑤⑥⑦⑧⑨⑩]|\s|title\=|【[a-zA-Z0-9]+】|[^\w])[\u4e00-\u9fa5]+',
                            entity_text).group(1)
                        entity_text = entity_text.replace(filter, '')
                    elif re.search('\]|\[|\]|[【】{}「?:∶〔·.\'#~_ΓΙεⅠ]', entity_text):
                        entity_text = re.sub('\]|\[|\]|[【】「?:∶〔·.\'#~_ΓΙεⅠ]', '', entity_text)
                    if len(re.sub('(项目|分|有限)?公司|集团|制造部|中心|医院|学校|大学|中学|小学|幼儿园', '', entity_text))<2:
                        # print('公司实体不符合规范：', entity_text)
                        continue

                entity_text = cut_repeat_name(entity_text) # 20231201 重复名称去重 如：中山大学附属第一医院中山大学附属第一医院中山大学附属第一医院

                list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
            # 标记文章末尾的"发布人”、“发布时间”实体
            if sentence_index==len(list_sentence)-1 or sentence_index==doctextcon_sentence_len-1:
                if len(list_sentence_entitys[-2:])==2:
                    second2last = list_sentence_entitys[-2]
                    last = list_sentence_entitys[-1]
                    if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
                            second2last.entity_type=="time" and last.entity_type in ["company",'org']):
                        if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
                            last.is_tail = True
                            second2last.is_tail = True

            #使用正则识别金额

            money_list, found_yeji = get_money_entity(sentence_text, found_yeji, in_attachment)
            entity_type = "money"
            for money in money_list:
                # print('money: ', money)
                entity_text, begin_index, end_index, unit, notes = money
                end_index = end_index - 1 if entity_text.endswith('，') else end_index
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                _exists = False
                for item in list_sentence_entitys:
                    if item.entity_id==entity_id and item.entity_type==entity_type:
                        _exists = True
                    if (begin_index >=item.wordOffset_begin and begin_index<item.wordOffset_end) or (end_index>item.wordOffset_begin and end_index<=item.wordOffset_end):
                        _exists = True
                        # print('_exists: ',begin_index, end_index, item.wordOffset_begin, item.wordOffset_end, item.entity_text, item.entity_type)
                if not _exists:
                    if float(entity_text)>1:
                        # if symbol == '-': # 负值金额保留负号
                        #     entity_text = '-'+entity_text   # 20230414 取消符号
                        begin_words = changeIndexFromWordToWords(tokens, begin_index)
                        end_words = changeIndexFromWordToWords(tokens, end_index)
                        # print('金额位置： ', begin_index, begin_words,end_index, end_words)
                        # print('金额召回: ', entity_text, sentence_text[begin_index:end_index], tokens[begin_words:end_words])
                        list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_words,end_words,begin_index,end_index,in_attachment=in_attachment))
                        list_sentence_entitys[-1].notes = notes  # 2021/7/20 新增金额备注
                        list_sentence_entitys[-1].money_unit = unit  # 2021/7/20 新增金额备注
                        # print('预处理中的 金额:%s, 单位：%s'%(entity_text,unit))
                        # print(entity_text,unit,notes)

            # "联系人"正则补充提取  2021/11/15 新增
            list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
            error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
                          '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理',
                          '代理人','采购','附件','注意','登录','报名','踏勘',"测试",'交货']
            list_person_text = set(list_person_text + error_text)
            re_person = re.compile("联系人[:：]([\u4e00-\u9fa5]工)|"
                                   "联系人[:：]([\u4e00-\u9fa5]{2,3})(?=，?联系)|"
                                   "联系人[:：]([\u4e00-\u9fa5]{2,3})(?=[，。；、])"
                                   )
            list_person = []
            if not in_attachment:
                for match_result in re_person.finditer(sentence_text):
                    match_text = match_result.group()
                    entity_text = match_text[4:]
                    wordOffset_begin = match_result.start() + 4
                    wordOffset_end = match_result.end()
                    # print(text[wordOffset_begin:wordOffset_end])
                    # 排除一些不为人名的实体
                    if re.search("^[\u4e00-\u9fa5]{7,}([，。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
                        continue
                    if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
                        _person = dict()
                        _person['body'] = entity_text
                        _person['begin_index'] = wordOffset_begin
                        _person['end_index'] = wordOffset_end
                        list_person.append(_person)
            entity_type = "person"
            for person in list_person:
                begin_index_temp = person['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = person['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = person['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 时间实体格式补充
            re_time_new = re.compile("20\d{2}-\d{1,2}-\d{1,2}|20\d{2}/\d{1,2}/\d{1,2}|20\d{2}\.\d{1,2}\.\d{1,2}|20\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[1-2][0-9]|3[0-1])")
            entity_type = "time"
            for _time in re.finditer(re_time_new,sentence_text):
                entity_text = _time.group()
                begin_index_temp = _time.start()
                end_index_temp = _time.end()
                is_same = False
                for t_index in ner_time_list:
                    if begin_index_temp>=t_index[0] and end_index_temp<=t_index[1]:
                        is_same = True
                        break
                if is_same:
                    continue
                if _time.start()!=0 and re.search("\d",sentence_text[_time.start()-1:_time.start()]):
                    continue
                # 纯数字格式，例：20190509
                if re.search("^\d{8}$",entity_text):
                    if _time.end()!=len(sentence_text) and re.search("[\da-zA-z]",sentence_text[_time.end():_time.end()+1]):
                        continue
                    entity_text = entity_text[:4] + "-" + entity_text[4:6] + "-" + entity_text[6:8]
                if not timeFormat(entity_text):
                    continue

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= end_index_temp:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp, in_attachment=in_attachment))

            # 资金来源提取  2020/12/30 新增
            list_moneySource = extract_moneySource(sentence_text)
            entity_type = "moneysource"
            for moneySource in list_moneySource:
                entity_text = moneySource['body']
                if len(entity_text)>50:
                    continue
                begin_index_temp = moneySource['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = moneySource['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment,prob=moneySource['prob']))

            # 电子邮箱提取 2021/11/04 新增
            list_email = extract_email(sentence_text)
            entity_type = "email" # 电子邮箱
            for email in list_email:
                begin_index_temp = email['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = email['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = email['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 服务期限提取 2020/12/30 新增
            list_servicetime = extract_servicetime(sentence_text)
            entity_type = "serviceTime"
            for servicetime in list_servicetime:
                entity_text = servicetime['body']
                begin_index_temp = servicetime['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = servicetime['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment, prob=servicetime["prob"]))

            # 2021/12/29 新增比率提取
            list_ratio = extract_ratio(sentence_text)
            entity_type = "ratio"
            for ratio in list_ratio:
                # print("ratio", ratio)
                begin_index_temp = ratio['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = ratio['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = ratio['body']
                ratio_value = (ratio['value'],ratio['type'])
                _entity = Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment)
                _entity.ratio_value = ratio_value
                list_sentence_entitys.append(_entity)

            list_sentence_entitys.sort(key=lambda x:x.begin_index)
            list_entitys_temp = list_entitys_temp+list_sentence_entitys
        # 补充ner模型未识别全的company/org实体
        for sentence_index in range(len(list_sentence)):
            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            in_attachment = list_sentence[sentence_index].in_attachment
            list_tokenbegin = []
            begin = 0
            for i in range(0, len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin + 1)
            add_sentence_entitys = []
            company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True)
            for company_type,company_text in company_dict:
                begin_index_list = findAllIndex(company_text,sentence_text)
                for begin_index in begin_index_list:
                    is_continue = False
                    for t_begin,t_end in list(company_index[sentence_index]):
                        if begin_index>=t_begin and begin_index+len(company_text)<=t_end:
                            is_continue = True
                            break
                    if not is_continue:
                        add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text))
                        company_index[sentence_index].add((begin_index,begin_index+len(company_text)))
                    else:
                        continue
            for ner_entity in add_sentence_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]

                if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
                    continue

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                entity_text = re.sub("[,，。：!&@$\*]","",entity_text)
                entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text
                list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
        list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index))
        list_entitys.append(list_entitys_temp)
    return list_entitys
    
def union_result(codeName,prem):
    '''
    @summary:模型的结果拼成字典
    @param:
        codeName:编号名称模型的结果字典
        prem:拿到属性的角色的字典
    @return:拼接起来的字典
    '''
    result = []
    assert len(codeName)==len(prem)
    for item_code,item_prem in zip(codeName,prem):
        result.append(dict(item_code,**item_prem))
    return result

def persistenceData(data):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for item_index in range(len(data)):
        item = data[item_index]
        doc_id = item[0]
        dic = item[1]
        code = dic['code']
        name = dic['name']
        prem = dic['prem']
        if len(code)==0:
            code_insert = ""
        else:
            code_insert = ";".join(code)
        prem_insert = ""
        for item in prem:
            for x in item:
                if isinstance(x, list):
                    if len(x)>0:
                        for x1 in x:
                            prem_insert+="/".join(x1)+","
                    prem_insert+="$"
                else:
                    prem_insert+=str(x)+"$"
            prem_insert+=";"
        sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
        cursor.execute(sql)
    conn.commit()
    conn.close()
   
def persistenceData1(list_entitys,list_sentences):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for list_entity in list_entitys:
        for entity in list_entity:
            if entity.values is not None:
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
            else:    
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
            cursor.execute(sql)
    for list_sentence in list_sentences:
        for sentence in list_sentence:
            str_tokens = "["
            for item in sentence.tokens:
                str_tokens += "'"
                if item=="'":
                    str_tokens += "''"
                else:
                    str_tokens += item
                str_tokens += "',"
            str_tokens = str_tokens[:-1]+"]"
            sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
            cursor.execute(sql)
    conn.commit()
    conn.close()


def _handle(item,result_queue):
    dochtml = item["dochtml"]
    docid = item["docid"]
    list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
    flag = False
    if list_innerTable:
        flag = True
        for table in list_innerTable:
            result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})

def getPredictTable():
    filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
    import pandas as pd
    import json
    from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
    df = pd.read_csv(filename)
    df_data = {"json_table":[],"docid":[]}
    _count = 0
    _sum = len(df["docid"])
    task_queue = Queue()
    result_queue = Queue()
    _index = 0
    for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
        task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
        _index += 1

    mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
    mh.run()
    while True:
        try:
            item = result_queue.get(block=True,timeout=1)
            df_data["docid"].append(item["docid"])
            df_data["json_table"].append(item["json_table"])
        except Exception as e:
            print(e)
            break
    df_1 = pd.DataFrame(df_data)
    df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])

if __name__=="__main__":
    '''
    import glob
    for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
        file_txt = str(file).replace("html","txt")
        with codecs.open(file_txt,"a+",encoding="utf8") as f:
            f.write("\n================\n")
            content = codecs.open(file,"r",encoding="utf8").read()
            f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
    '''        
    # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
    # getPredictTable()
    with open('D:/138786703.html', 'r', encoding='utf-8') as f:
        sourceContent = f.read()
        # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
        # print(article_processed)

        list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
        for entity in list_entitys[0]:
            print(entity.entity_type, entity.entity_text)