BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/Preprocessing.py

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup, Comment
import copy
import sys
import os
import time
import codecs

from BiddingKG.dl.ratio.re_ratio import extract_ratio

sys.setrecursionlimit(1000000)
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath(".."))
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Entitys import *
from BiddingKG.dl.interface.predictor import getPredictor
from BiddingKG.dl.common.nerUtils import *
from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
from BiddingKG.dl.time.re_servicetime import extract_servicetime
from BiddingKG.dl.relation_extraction.re_email import extract_email
from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate
from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
from BiddingKG.dl.entityLink.entityLink import *


#
def tableToText(soup):
    '''
    @param:
        soup:网页html的soup
    @return:处理完表格信息的网页text
    '''
    
    def getTrs(tbody):
    #获取所有的tr
        trs = []
        objs = tbody.find_all(recursive=False)
        for obj in objs:
            if obj.name=="tr":
                trs.append(obj)
            if obj.name=="tbody":
                for tr in obj.find_all("tr",recursive=False):
                    trs.append(tr)
        return trs

    def fixSpan(tbody):
        # 处理colspan, rowspan信息补全问题
        #trs = tbody.findChildren('tr', recursive=False)

        trs = getTrs(tbody)
        ths_len = 0
        ths = list()
        trs_set = set()
        #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱
        # 遍历每一个tr

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有colspan 则补全同一行下一个位置
                if 'colspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
                        col = int(re.sub("[^0-9]","",str(td['colspan'])))
                        if col<100 and len(td.get_text())<1000:
                            td['colspan'] = 1
                            for i in range(1, col, 1):
                                td.insert_after(copy.copy(td))

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有rowspan 则补全下一行同样位置
                if 'rowspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
                        row = int(re.sub("[^0-9]","",str(td['rowspan'])))
                        td['rowspan'] = 1
                        for i in range(1, row, 1):
                            # 获取下一行的所有td， 在对应的位置插入
                            if indtr+i<len(trs):
                                tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
                                if len(tds1) >= (indtd) and len(tds1)>0:
                                    if indtd > 0:
                                        tds1[indtd - 1].insert_after(copy.copy(td))
                                    else:
                                        tds1[0].insert_before(copy.copy(td))
                                elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1:  # 修正某些表格最后一列没补全
                                    tds1[indtd-2].insert_after(copy.copy(td))
    def getTable(tbody):
        #trs = tbody.findChildren('tr', recursive=False)
        trs = getTrs(tbody)
        inner_table = []
        for tr in trs:
            tr_line = []
            tds = tr.findChildren(['td','th'], recursive=False)
            if len(tds)==0:
                tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失
            for td in tds:
                tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
                #tr_line.append([td.get_text(),0])
            inner_table.append(tr_line)
        return inner_table                          
    
    #处理表格不对齐的问题
    def fixTable(inner_table,fix_value="~~"):
        maxWidth = 0
        for item in inner_table:
            if len(item)>maxWidth:
                maxWidth = len(item)
        for i in range(len(inner_table)):
            if len(inner_table[i])<maxWidth:
                for j in range(maxWidth-len(inner_table[i])):
                    inner_table[i].append([fix_value,0])
        return inner_table
    
    def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            point = ""
            for j in range(width):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_row
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        for j in range(width):
            point = ""
            for i in range(height):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_col
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        
    def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if inner_table[i][j][0]==pad_row:
                    inner_table[i][j][0] = inner_table[i][j-1][0]
                    inner_table[i][j][1] = inner_table[i][j-1][1]
                if inner_table[i][j][0]==pad_col:
                    inner_table[i][j][0] = inner_table[i-1][j][0]
                    inner_table[i][j][1] = inner_table[i-1][j][1]

    def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
        '''
        @summary: 修复表头识别，将明显错误的进行修正
        '''
        def repairNeeded(line):
            first_1 = -1
            last_1 = -1
            first_0 = -1
            last_0 = -1
            count_1 = 0
            count_0 = 0
            for i in range(len(line)):
                if line[i][0]==fix_value:
                    continue
                if line[i][1]==1:
                    if first_1==-1:
                        first_1 = i
                    last_1 = i
                    count_1 += 1
                if line[i][1]==0:
                    if first_0 == -1:
                        first_0 = i
                    last_0 = i
                    count_0 += 1
            if first_1 ==-1 or last_0 == -1:
                return False
            #异常情况：第一个不是表头；最后一个是表头；表头个数远大于属性值个数
            if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
                return True
            return False

        def getsimilarity(line,line1):
            same_count = 0
            for item,item1 in zip(line,line1):
                if item[1]==item1[1]:
                    same_count += 1
            return same_count/len(line)

        def selfrepair(inner_table,index,dye_set,key_set):
            '''
            @summary: 计算每个节点受到的挤压度来判断是否需要染色
            '''
            #print("B",inner_table[index])
            min_presure = 3
            list_dye = []
            first = None
            count = 0
            temp_set = set()
            _index = 0
            for item in inner_table[index]:
                if first is None:
                    first = item[1]
                    if item[0] not in temp_set:
                        count += 1
                        temp_set.add(item[0])
                else:
                    if first == item[1]:
                        if item[0] not in temp_set:
                            temp_set.add(item[0])
                            count += 1
                    else:
                        list_dye.append([first,count,_index])
                        first = item[1]
                        temp_set.add(item[0])
                        count = 1
                _index += 1
            list_dye.append([first,count,_index])
            if len(list_dye)>1:
                begin = 0
                end = 0
                for i in range(len(list_dye)):
                    end = list_dye[i][2]
                    dye_flag = False
                    #首尾要求压力减一
                    if i==0:
                        if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i+1][0]
                    elif i==len(list_dye)-1:
                        if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i-1][0]
                    else:
                        if list_dye[i][1]>1:
                            if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                        else:
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                    if dye_flag:
                        for h in range(begin,end):
                            inner_table[index][h][1] = dye_type
                            dye_set.add((inner_table[index][h][0],dye_type))
                            key_set.add(inner_table[index][h][0])
                    begin = end
                #print("E",inner_table[index])




        def otherrepair(inner_table,index,dye_set,key_set):
            list_provide_repair = []
            if index==0 and len(inner_table)>1:
                list_provide_repair.append(index+1)
            elif index==len(inner_table)-1:
                list_provide_repair.append(index-1)
            else:
                list_provide_repair.append(index+1)
                list_provide_repair.append(index-1)
            for provide_index in list_provide_repair:
                if not repairNeeded(inner_table[provide_index]):
                    same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
                    if same_prob>=0.8:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = inner_table[provide_index][i][1]
                    elif same_prob<=0.2:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]==inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
        len_dye_set = len(dye_set)
        height = len(inner_table)
        for i in range(height):
            if repairNeeded(inner_table[i]):
                selfrepair(inner_table,i,dye_set,key_set)
                #otherrepair(inner_table,i,dye_set,key_set)
        for h in range(len(inner_table)):
            for w in range(len(inner_table[0])):
                if inner_table[h][w][0] in key_set:
                    for item in dye_set:
                        if inner_table[h][w][0]==item[0]:
                            inner_table[h][w][1] = item[1]
        #如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代
        if len(dye_set)!=len(key_set):
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    selfrepair(inner_table,i,dye_set,key_set)
                    #otherrepair(inner_table,i,dye_set,key_set)
            return
        if len(dye_set)==len_dye_set:
            '''
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    otherrepair(inner_table,i,dye_set,key_set)
            '''
            return
        repairTable(inner_table, dye_set, key_set)

    def sliceTable(inner_table,fix_value="~~"):
        #进行分块
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        last_head = None
        last_is_same_value = False
        for h in range(height):
            is_all_key = True#是否是全表头行
            is_all_value = True#是否是全属性值
            is_same_with_lastHead = True#和上一行的结构是否相同
            is_same_value=True#一行的item都一样
            #is_same_first_item = True#与上一行的第一项是否相同
            same_value = inner_table[h][0][0]
            for w in range(width):
                if last_head is not None:
                    if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
                        is_all_key = False

                    if inner_table[h][w][0]==1:
                        is_all_value = False

                    if inner_table[h][w][1]!= inner_table[h-1][w][1]:
                        is_same_with_lastHead = False

                if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
                    is_same_value = False
                else:
                    if re.search("\d+",same_value) is not None:
                        is_same_value = False
            if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
                is_same_first_item = False

            last_head = h

            if last_is_same_value:
                last_is_same_value = is_same_value
                continue

            if is_same_value:
                head_list.append(h)
                last_is_same_value = is_same_value
                continue
            if not is_all_key:
                if not is_same_with_lastHead:
                    head_list.append(h)


        head_list.append(height)
        return head_list
                    
    def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):


        set_item = set()
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                set_item.add(item)
        list_item = list(set_item)
        x = []
        for item in list_item:
            x.append(getPredictor("form").encode(item))
        predict_y = getPredictor("form").predict(np.array(x),type="item")
        _dict = dict()
        
        for item,values in zip(list_item,list(predict_y)):
            _dict[item] = values[1]
            # print("##",item,values)
        #print(_dict)
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")

        repairTable(inner_table)
        head_list = sliceTable(inner_table)

        
        return inner_table,head_list

    def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):

        data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
        predict_y = getPredictor("form").getModel("context").predict(data_x)

        for _position,_y in zip(data_position,predict_y):
            _w = _position[0]
            _h = _position[1]
            if _y[1]>prob_min:
                inner_table[_h][_w][1] = 1
            else:
                inner_table[_h][_w][1] = 0
                _item = inner_table[_h][_w][0]
                if re.search(pat_head,_item) is not None and len(_item)<8:
                    inner_table[_h][_w][1] = 1

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if re.search("[:：]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
                    inner_table[i][j][1] = 1

        repairTable(inner_table)
        head_list = sliceTable(inner_table)
        # print("inner_table:",inner_table)

        return inner_table,head_list
                    
    #设置表头
    def setHead_inline(inner_table,prob_min=0.64):
        pad_row = "@@"
        pad_col = "##"
        removePadding(inner_table, pad_row, pad_col)
        pad_pattern = re.compile(pad_row+"|"+pad_col)
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            
            is_head = False
            is_long_value = False
            
            #判断是否是全padding值
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
                    is_same_value = False
                    break
                
            #predict is head or not with model
            temp_item = ""
            for j in range(width):
                temp_item += inner_table[i][j][0]+"|"
            temp_item = re.sub(pad_pattern,"",temp_item)
            form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
            if form_prob is not None:
                if form_prob[0][1]>prob_min:
                    is_head = True
                else:
                    is_head = False
            
            #print(temp_item,form_prob)
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                #不把连续表头分开
                if not is_head_last:
                    head_list.append(i)
                if is_long_value or is_same_value:
                    head_list.append(i+1)
            if is_head:
                for j in range(width):
                    inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                is_head = False
                
                #predict is head or not with model
                temp_item = ""
                for j in range(head_begin,head_end):
                    temp_item += inner_table[j][i][0]+"|"
                temp_item = re.sub(pad_pattern,"",temp_item)
                form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
                if form_prob is not None:
                    if form_prob[0][1]>prob_min:
                        is_head = True
                    else:
                        is_head = False
                
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        addPadding(inner_table, pad_row, pad_col)
        return inner_table,head_list
    
    #设置表头
    def setHead_withRule(inner_table,pattern,pat_value,count):
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            set_match = set()
            is_head = False
            is_long_value = False
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value:
                    is_same_value = False
                    break
            for j in range(width):
                if re.search(pat_value,inner_table[i][j][0]) is not None:
                    is_head = False
                    break
                str_find = re.findall(pattern,inner_table[i][j][0])
                if len(str_find)>0:
                    set_match.add(inner_table[i][j][0])
                if len(set_match)>=count:
                    is_head = True
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                if not is_head_last:
                    head_list.append(i)
                if is_head:
                    for j in range(width):
                        inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                set_match = set()
                is_head = False
                for j in range(head_begin,head_end):
                    if re.search(pat_value,inner_table[j][i][0]) is not None:
                        is_head = False
                        break
                    str_find = re.findall(pattern,inner_table[j][i][0])
                    if len(str_find)>0:
                        set_match.add(inner_table[j][i][0])
                    if len(set_match)>=count:
                        is_head = True
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        return inner_table,head_list
    
    #取得表格的处理方向
    def getDirect(inner_table,begin,end):
        '''
        column_head = set()
        row_head = set()
        widths = len(inner_table[0])
        for height in range(begin,end):
            for width in range(widths):
                if inner_table[height][width][1] ==1:
                    row_head.add(height)
                if inner_table[height][width][1] ==2:
                    column_head.add(width)
        company_pattern = re.compile("公司")
        if 0 in column_head and begin not in row_head:
            return "column"
        if 0 in column_head and begin in row_head:
            for height in range(begin,end):
                count = 0
                count_flag = True
                for width_index in range(width):
                    if inner_table[height][width_index][1]==0:
                        if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                            count += 1
                        else:
                            count_flag = False
                if count_flag and count>=2:
                    return "column"
        return "row"
        '''
        count_row_keys = 0
        count_column_keys = 0
        width = len(inner_table[0])
        if begin<end:
            for w in range(len(inner_table[begin])):
                if inner_table[begin][w][1]!=0:
                    count_row_keys += 1
        for h in range(begin,end):
            if inner_table[h][0][1]!=0:
                count_column_keys += 1
                
        company_pattern = re.compile("有限(责任)?公司")
        for height in range(begin,end):
            count_set = set()
            count_flag = True
            for width_index in range(width):
                if inner_table[height][width_index][1]==0:
                    if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                        count_set.add(inner_table[height][width_index][0])
                    else:
                        count_flag = False
            if count_flag and len(count_set)>=2:
                return "column"
        # if count_column_keys>count_row_keys:  #2022/2/15 此项不够严谨，造成很多错误，故取消
        #     return "column"
        return "row"
        
            
    #根据表格处理方向生成句子，        
    def getTableText(inner_table,head_list,key_direct=False):
        # packPattern = "(标包|[标包][号段名])"
        packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))"  # 2020/11/23 大网站规则，补充采购类包名
        rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)"  # 2020/11/23 大网站规则，添加序号为排序
        entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
        moneyPattern = "([中投]标|报价)(金额|价)"
        height = len(inner_table)
        width = len(inner_table[0])
        text = ""
         
        for head_i in range(len(head_list)-1):
            
            head_begin = head_list[head_i]
            head_end = head_list[head_i+1]
                
            direct = getDirect(inner_table, head_begin, head_end)

            #若只有一行，则直接按行读取
            if head_end-head_begin==1:
                text_line = ""
                for i in range(head_begin,head_end):
                    for w in range(len(inner_table[i])):
                        if inner_table[i][w][1]==1:
                            _punctuation = "："
                        else:
                            _punctuation = "，"  #2021/12/15 统一为中文标点，避免 206893924 国际F座1108,1,009,197.49元
                        if w>0:
                            if inner_table[i][w][0]!= inner_table[i][w-1][0]:
                                text_line += inner_table[i][w][0]+_punctuation
                        else:
                            text_line += inner_table[i][w][0]+_punctuation
                    text_line = text_line+"。" if text_line!="" else text_line
                text += text_line
            else:
                #构建一个共现矩阵
                table_occurence = []
                for i in range(head_begin,head_end):
                    line_oc = []
                    for j in range(width):
                        cell = inner_table[i][j]
                        line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
                    table_occurence.append(line_oc)


                occu_height = len(table_occurence)
                occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
                #为每个属性值寻找表头
                for i in range(occu_height):
                    for j in range(occu_width):
                        cell = table_occurence[i][j]
                        #是属性值
                        if cell["type"]==0 and cell["text"]!="":
                            left_head = ""
                            top_head = ""

                            find_flag = False
                            temp_head = ""
                            for loop_i in range(1,i+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [1]
                                if table_occurence[i-loop_i][j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i-loop_i][j]["text"]!=temp_head:
                                            top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head
                                    else:
                                        top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head
                                    find_flag = True
                                    temp_head = table_occurence[i-loop_i][j]["text"]
                                    table_occurence[i-loop_i][j]["occu_count"] += 1
                                else:
                                    #找到表头后遇到属性值就返回
                                    if find_flag:
                                        break


                            cell["top_head"] += top_head
                            find_flag = False
                            temp_head = ""



                            for loop_j in range(1,j+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [2]
                                if table_occurence[i][j-loop_j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i][j-loop_j]["text"]!=temp_head:
                                            left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head
                                    else:
                                        left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head
                                    find_flag = True
                                    temp_head = table_occurence[i][j-loop_j]["text"]
                                    table_occurence[i][j-loop_j]["occu_count"] += 1
                                else:
                                    if find_flag:
                                        break
                            cell["left_head"] += left_head
                if direct=="row":
                    for i in range(occu_height):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        money_text = ""
                        #在同一句话中重复的可以去掉
                        text_set = set()
                        for j in range(width):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["top_head"]+"：") if len(cell["top_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["left_head"] + head
                                else:
                                    head += cell["left_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None:
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    if re.search(moneyPattern,head) is not None and entity_text!="":
                                        money_text += head+cell["text"]+","
                                    else:
                                        text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))

                        text += pack_text+rank_text+entity_text+money_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text

                else:
                    for j in range(occu_width):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        text_set = set()
                        for i in range(occu_height):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["top_head"] + head
                                else:
                                    head += cell["top_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None and \
                                        re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))
                        text += pack_text+rank_text+entity_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text


                # if direct=="row":
                #     for i in range(head_begin,head_end):
                #         pack_text = ""
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         #在同一句话中重复的可以去掉
                #         text_set = set()
                #         for j in range(width):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 head = ""
                #
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         #找到表头后遇到属性值就返回
                #                         if find_flag:
                #                             break
                #
                #                 find_flag = False
                #                 temp_head = ""
                #
                #
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(packPattern,head) is not None:
                #                     pack_text += head+inner_table[i][j][0]+"，"
                #                 elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                #                     #排名替换为同一种表达
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += pack_text+rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
                # else:
                #     for j in range(width):
                #
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         text_set = set()
                #         for i in range(head_begin,head_end):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 find_flag = False
                #                 head = ""
                #                 temp_head = ""
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(rankPattern,head) is not None:
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
        return text
    
    def removeFix(inner_table,fix_value="~~"):
        height = len(inner_table)
        width = len(inner_table[0])
        for h in range(height):
            for w in range(width):
                if inner_table[h][w][0]==fix_value:
                    inner_table[h][w][0] = ""
    
    def trunTable(tbody):
        fixSpan(tbody)
        inner_table = getTable(tbody)
        inner_table = fixTable(inner_table)
        if len(inner_table)>0 and len(inner_table[0])>0:
            #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
            #inner_table,head_list = setHead_inline(inner_table)
            inner_table,head_list = setHead_initem(inner_table,pat_head)
            # inner_table,head_list = setHead_incontext(inner_table,pat_head)
            # print(inner_table)
            # for begin in range(len(head_list[:-1])):
            #     for item in inner_table[head_list[begin]:head_list[begin+1]]:
            #         print(item)
            #     print("====")

            removeFix(inner_table)
            
            # print("----")
            # print(head_list)
            # for item in inner_table:
            #     print(item)


            tbody.string = getTableText(inner_table,head_list)
            #print(tbody.string)
            tbody.name = "turntable"
            return inner_table
        return None


        
    pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$')
    #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
    pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")

    list_innerTable = []

    # 2022/2/9 删除干扰标签
    for tag in soup.find_all('option'): #例子： 216661412
        if 'selected' not in tag.attrs:
            tag.extract()
    for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题
        if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall(
                '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|开标评标|资格评审',
                ul.get_text(), re.S)))>3:
            ul.extract()

    tbodies = soup.find_all('table')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)

    tbodies = soup.find_all('tbody')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)

    return soup
    # return list_innerTable

re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]")
num_dict = {
    "一": 1, "二": 2,
    "三": 3, "四": 4,
    "五": 5, "六": 6,
    "七": 7, "八": 8,
    "九": 9, "十": 10}
# 一百以内的中文大写转换为数字
def change2num(text):
    result_num = -1
    # text = text[:6]
    match = re_num.search(text)
    if match:
        _num = match.group()
        if num_dict.get(_num):
            return num_dict.get(_num)
        else:
            tenths = 1
            the_unit = 0
            num_split = _num.split("十")
            if num_dict.get(num_split[0]):
                tenths = num_dict.get(num_split[0])
            if num_dict.get(num_split[1]):
                the_unit = num_dict.get(num_split[1])
            result_num = tenths * 10 + the_unit
    elif re.search("\d{1,2}",text):
        _num = re.search("\d{1,2}",text).group()
        result_num = int(_num)
    return result_num
#大纲分段处理
def get_preprocessed_outline(soup):
    pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、．\.]")
    pattern_1 = re.compile("^[\(（]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\)）]")
    pattern_2 = re.compile("^\d{1,2}[、．\.](?=[^\d]{1,2}|$)")
    pattern_3 = re.compile("^[\(（]?\d{1,2}[\)）]")
    pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3]

    body = soup.find("body")
    body_child = body.find_all(recursive=False)
    deal_part = body
    # print(body_child[0]['id'])
    if 'id' in body_child[0].attrs:
        if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent':
            deal_part = body_child[0]
    if len(deal_part.find_all(recursive=False))>2:
        deal_part = deal_part.parent
    skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot']
    for part in deal_part.find_all(recursive=False):
        # 查找解析文本的主干部分
        is_main_text = False
        through_text_num = 0
        while (not is_main_text and part.find_all(recursive=False)):
            while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \
                    part.find_all(recursive=False)[0].get_text(strip=True):
                part = part.find_all(recursive=False)[0]
            max_len = len(part.get_text(strip=True))
            is_main_text = True
            for t_part in part.find_all(recursive=False):
                if t_part.name not in skip_tag and t_part.get_text(strip=True)!="":
                    through_text_num += 1
                if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65:
                    if t_part.name not in skip_tag:
                        is_main_text = False
                        part = t_part
                        break
                    else:
                        while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \
                                t_part.find_all(recursive=False)[0].get_text(strip=True):
                            t_part = t_part.find_all(recursive=False)[0]
                        if through_text_num>2:
                            is_table = True
                            for _t_part in t_part.find_all(recursive=False):
                                if _t_part.name not in skip_tag:
                                    is_table = False
                                    break
                            if not is_table:
                                is_main_text = False
                                part = t_part
                                break
                        else:
                            is_main_text = False
                            part = t_part
                            break
        is_find = False
        for _pattern in pattern_list:
            last_index = 0
            handle_list = []
            for _part in part.find_all(recursive=False):
                if _part.name not in skip_tag and _part.get_text(strip=True) != "":
                    # print('text:', _part.get_text(strip=True))
                    re_match = re.search(_pattern, _part.get_text(strip=True))
                    if re_match:
                        outline_index = change2num(re_match.group())
                        if last_index < outline_index:
                            # _part.insert_before("##split##")
                            handle_list.append(_part)
                            last_index = outline_index
            if len(handle_list)>1:
                is_find = True
                for _part in handle_list:
                    _part.insert_before("##split##")
            if is_find:
                break
    # print(soup)
    return soup

#数据清洗
def segment(soup,final=True):
    # print("==")
    # print(soup)
    # print("====")
    #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
    subspaceList = ["td",'a',"span","p"]
    if soup.name in subspaceList:
        #判断有值叶子节点数
        _count = 0
        for child in soup.find_all(recursive=True):
            if child.get_text().strip()!="" and len(child.find_all())==0:
                _count += 1
        if _count<=1:
            text = soup.get_text()
            # 2020/11/24 大网站规则添加
            if 'title' in soup.attrs:
                if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']:
                    text = soup.attrs['title']

            _list = []
            for x in re.split("\s+",text):
                if x.strip()!="":
                    _list.append(len(x))
            if len(_list)>0:
                _minLength = min(_list)
                if _minLength>2:
                    _substr = "，"
                else:
                    _substr = ""
            else:
                _substr = ""

            text = text.replace("\r\n","，").replace("\n","，")
            text = re.sub("\s+",_substr,text)
            # text = re.sub("\s+","##space##",text)
            return text
    segList = ["title"]
    commaList = ["div","br","td","p","li"]
    #commaList = []
    spaceList = ["span"]

    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        # print(child.name,child.get_text())
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            child.insert_after("，")
        # if child.name == 'div' and 'class' in child.attrs:
        #     # 添加附件"attachment"标识
        #     if "richTextFetch" in child['class']:
        #         child.insert_before("##attachment##")
                # print(child.parent)
        # if child.name in subspaceList:
        #     child.insert_before("#subs"+str(child.name)+"#")
        #     child.insert_after("#sube"+str(child.name)+"#")
        # if child.name in spaceList:
        #     child.insert_after(" ")
    text = str(soup.get_text())
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
    # 感叹号替换为中文句号
    text = re.sub("(?<=[\u4e00-\u9fa5])[!！]|[!！](?=[\u4e00-\u9fa5])","。",text)
    #替换格式未识别的问号为" " ,update:2021/7/20
    text = re.sub("[？\?]{2,}"," ",text)


    #替换"""为"“",否则导入deepdive出错
    # text = text.replace('"',"“").replace("\r","").replace("\n","，")
    text = text.replace('"',"“").replace("\r","").replace("\n","")  #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
    # print('==1',text)
    # text = re.sub("\s{4,}","，",text)
    # 解决公告中的" "空格替换问题
    if re.search("\s{4,}",text):
        _text = ""
        for _sent in re.split("。+",text):
            for _sent2 in re.split('，+',_sent):
                for _sent3 in re.split("：+",_sent2):
                    for _t in re.split("\s{4,}",_sent3):
                        if len(_t)<3:
                            _text += _t
                        else:
                            _text += "，"+_t
                    _text += "："
                _text = _text[:-1]
                _text += "，"
            _text = _text[:-1]
            _text += "。"
        _text = _text[:-1]
        text = _text
    # print('==2',text)
    #替换标点

    #替换连续的标点

    if final:
        text = re.sub("##space##"," ",text)

    punc_pattern = "(?P<del>[。，；：:,\s]+)"

    list_punc = re.findall(punc_pattern,text)
    list_punc.sort(key=lambda x:len(x),reverse=True)
    for punc_del in list_punc:
        if len(punc_del)>1:
            if len(punc_del.strip())>0:
                if "：" in punc_del.strip():
                    if "。" in punc_del.strip():
                        text = re.sub(punc_del, "：。", text)
                    else:
                        text = re.sub(punc_del,"：",text)
                else:
                    text = re.sub(punc_del,punc_del.strip()[0],text)   #2021/12/09 修正由于某些标签后插入符号把原来符号替换
            else:
                text = re.sub(punc_del,"",text)
        

    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)

    # #删除标签中的所有空格
    # for subs in subspaceList:
    #     patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
    #     while(True):
    #         oneMatch = re.search(re.compile(patten),text)
    #         if oneMatch is not None:
    #             _match = oneMatch.group(1)
    #             text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
    #         else:
    #             break

    # text过大报错
    LOOP_LEN = 10000
    LOOP_BEGIN = 0
    _text = ""



    if len(text)<10000000:
        while(LOOP_BEGIN<len(text)):
            _text += re.sub("）",")",re.sub("（","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
            LOOP_BEGIN += LOOP_LEN
        text = _text
    # 附件标识前修改为句号，避免正文和附件内容混合在一起
    text = re.sub("[^。](?=##attachment##)","。",text)
    return text

'''
#数据清洗
def segment(soup):
    segList = ["title"]
    commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]    
    spaceList = ["span"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        if child.name == 'br':
            child.insert_before('，')
        child_text = re.sub('\s', '', child.get_text())
        if child_text == '' or child_text[-1] in ['。','，','：','；']:
            continue
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            if len(child_text)>3 and len(child_text) <50:   # 先判断是否字数少于50，成立加逗号，否则加句号
                child.insert_after("，")           
            elif len(child_text) >=50:
                child.insert_after("。")
        #if child.name in spaceList:
            #child.insert_after(" ")  
    text = str(soup.get_text())
    text = re.sub("\s{5,}","，",text)
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“")    
    #text = text.replace('"',"“").replace("\r","").replace("\n","")
    
    #删除所有空格
    text = re.sub("\s+","#nbsp#",text)
    text_list = text.split('#nbsp#')
    new_text = ''
    for i in range(len(text_list)-1):
        if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']:
            new_text += text_list[i]  
        elif  re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '。' 
        elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '；' 
        elif text_list[i].isdigit() and text_list[i+1].isdigit():
            new_text += text_list[i] + ' '           
        elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']:
            new_text += text_list[i]       
        elif len(text_list[i]) >= 3 and  len(text_list[i+1]) >= 3:
            new_text += text_list[i] + '，'
        else:
            new_text += text_list[i]       
    new_text += text_list[-1]      
    text = new_text
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
            
    #替换标点
    while(True):
        #替换连续的标点
        punc = re.search("，(?P<punc>：|。|，|；)\s*",text)
        if punc is not None:
            text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text)
        
        punc = re.search("(?P<punc>：|。|，|；)\s*，",text)
        if punc is not None:
            text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text)
        else:
            #替换标点之后的空格
            punc = re.search("(?P<punc>：|。|，|；)\s+",text)
            if punc is not None:
                text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
            else:
                break
    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)
    #替换中文括号为英文括号
    text = re.sub("（","(",text)
    text = re.sub("）",")",text)
    return text
'''

#连续实体合并（弃用）
def union_ner(list_ner):
    result_list = []
    union_index = []
    union_index_set = set()
    for i in range(len(list_ner)-1):
        if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
            if list_ner[i][1]-list_ner[i+1][0]==1:
                union_index_set.add(i)
                union_index_set.add(i+1)
                union_index.append((i,i+1))
    for i in range(len(list_ner)):
        if i not in union_index_set:
            result_list.append(list_ner[i])
    for item in union_index:
        #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
        result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
    return result_list
                

# def get_preprocessed(articles,useselffool=False):
#     '''
#     @summary:预处理步骤，NLP处理、实体识别
#     @param:
#         articles:待处理的文章list [[id,source,jointime,doc_id,title]]
#     @return:list of articles,list of each article of sentences,list of each article of entitys
#     '''
#     list_articles = []
#     list_sentences = []
#     list_entitys = []
#     cost_time = dict()
#     for article in articles:
#         list_sentences_temp = []
#         list_entitys_temp = []
#         doc_id = article[0]
#         sourceContent = article[1]
#         _send_doc_id = article[3]
#         _title = article[4]
#         #表格处理
#         key_preprocess = "tableToText"
#         start_time = time.time()
#         article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
#
#         # log(article_processed)
#
#         if key_preprocess not in cost_time:
#             cost_time[key_preprocess] = 0
#         cost_time[key_preprocess] += time.time()-start_time
#
#         #article_processed = article[1]
#         list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
#         #nlp处理
#         if article_processed is not None and len(article_processed)!=0:
#             split_patten = "。"
#             sentences = []
#             _begin = 0
#             for _iter in re.finditer(split_patten,article_processed):
#                 sentences.append(article_processed[_begin:_iter.span()[1]])
#                 _begin = _iter.span()[1]
#             sentences.append(article_processed[_begin:])
#
#             lemmas = []
#             doc_offsets = []
#             dep_types = []
#             dep_tokens = []
#
#             time1 = time.time()
#
#             '''
#             tokens_all = fool.cut(sentences)
#             #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
#             #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
#             ner_entitys_all = fool.ner(sentences)
#             '''
#             #限流执行
#             key_nerToken = "nerToken"
#             start_time = time.time()
#             tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
#             if key_nerToken not in cost_time:
#                 cost_time[key_nerToken] = 0
#             cost_time[key_nerToken] += time.time()-start_time
#
#
#             for sentence_index in range(len(sentences)):
#
#
#
#                 list_sentence_entitys = []
#                 sentence_text = sentences[sentence_index]
#                 tokens = tokens_all[sentence_index]
#
#                 list_tokenbegin = []
#                 begin = 0
#                 for i in range(0,len(tokens)):
#                     list_tokenbegin.append(begin)
#                     begin += len(str(tokens[i]))
#                 list_tokenbegin.append(begin+1)
#                 #pos_tag = pos_all[sentence_index]
#                 pos_tag = ""
#
#                 ner_entitys = ner_entitys_all[sentence_index]
#
#                 list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
#
#                 #识别package
#
#
#                 #识别实体
#                 for ner_entity in ner_entitys:
#                     begin_index_temp = ner_entity[0]
#                     end_index_temp = ner_entity[1]
#                     entity_type = ner_entity[2]
#                     entity_text = ner_entity[3]
#
#                     for j in range(len(list_tokenbegin)):
#                         if list_tokenbegin[j]==begin_index_temp:
#                             begin_index = j
#                             break
#                         elif list_tokenbegin[j]>begin_index_temp:
#                             begin_index = j-1
#                             break
#                     begin_index_temp += len(str(entity_text))
#                     for j in range(begin_index,len(list_tokenbegin)):
#                         if list_tokenbegin[j]>=begin_index_temp:
#                             end_index = j-1
#                             break
#                     entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#                     #去掉标点符号
#                     entity_text = re.sub("[,，。：]","",entity_text)
#                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
#
#
#                 #使用正则识别金额
#                 entity_type = "money"
#
#                 #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"
#
#                 list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
#                                       "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"}
#
#                 set_begin = set()
#                 for pattern_key in list_money_pattern.keys():
#                     pattern = re.compile(list_money_pattern[pattern_key])
#                     all_match = re.findall(pattern, sentence_text)
#                     index = 0
#                     for i in range(len(all_match)):
#                         if len(all_match[i][0])>0:
#                             # print("===",all_match[i])
#                             #print(all_match[i][0])
#                             unit = ""
#                             entity_text = all_match[i][3]
#                             if pattern_key in ["key_word","front_m"]:
#                                 unit = all_match[i][1]
#                             else:
#                                 unit = all_match[i][4]
#                             if entity_text.find("元")>=0:
#                                 unit = ""
#
#                             index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
#
#                             begin_index_temp = index
#                             for j in range(len(list_tokenbegin)):
#                                 if list_tokenbegin[j]==index:
#                                     begin_index = j
#                                     break
#                                 elif list_tokenbegin[j]>index:
#                                     begin_index = j-1
#                                     break
#                             index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
#                             end_index_temp = index
#                             #index += len(str(all_match[i][0]))
#                             for j in range(begin_index,len(list_tokenbegin)):
#                                 if list_tokenbegin[j]>=index:
#                                     end_index = j-1
#                                     break
#                             entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#
#                             entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
#                             if len(unit)>0:
#                                 entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
#                             else:
#                                 entity_text = str(getUnifyMoney(entity_text))
#
#                             _exists = False
#                             for item in list_sentence_entitys:
#                                 if item.entity_id==entity_id and item.entity_type==entity_type:
#                                     _exists = True
#                             if not _exists:
#                                 if float(entity_text)>10:
#                                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
#
#                         else:
#                             index += 1
#
#                 list_sentence_entitys.sort(key=lambda x:x.begin_index)
#                 list_entitys_temp = list_entitys_temp+list_sentence_entitys
#         list_sentences.append(list_sentences_temp)
#         list_entitys.append(list_entitys_temp)
#     return list_articles,list_sentences,list_entitys,cost_time

def get_preprocessed(articles, useselffool=False):
    '''
    @summary:预处理步骤，NLP处理、实体识别
    @param:
        articles:待处理的文章list [[id,source,jointime,doc_id,title]]
    @return:list of articles,list of each article of sentences,list of each article of entitys
    '''
    cost_time = dict()
    list_articles = get_preprocessed_article(articles,cost_time)
    list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
    list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)

    calibrateEnterprise(list_articles,list_sentences,list_entitys)

    return list_articles,list_sentences,list_entitys,list_outlines,cost_time

def special_treatment(sourceContent, web_source_no):
    if web_source_no == 'DX000202-1':
         ser = re.search('中标供应商及中标金额：【((\w{5,20}-[\d,.]+,)+)】', sourceContent)
         if ser:
             new = ""
             l = ser.group(1).split(',')
             for i in range(len(l)):
                 it = l[i]
                 if '-' in it:
                     role, money = it.split('-')
                     new += '标段%d, 中标供应商: ' % (i + 1) + role + '，中标金额：' + money + '。'
             sourceContent = sourceContent.replace(ser.group(0), new, 1)
    elif web_source_no == '00753-14':
        pcontent = sourceContent.find("div", id="pcontent")
        pcontent = pcontent.find_all(recursive=False)[0]
        first_table = None
        for idx in range(len(pcontent.find_all(recursive=False))):
            t_part = pcontent.find_all(recursive=False)[idx]
            if t_part.name != "table":
                break
            if idx == 0:
                first_table = t_part
            else:
                for _tr in t_part.find("tbody").find_all(recursive=False):
                    first_table.find("tbody").append(_tr)
                t_part.clear()
    elif web_source_no == 'DX008357-11':
        pcontent = sourceContent.find("div", id="pcontent")
        pcontent = pcontent.find_all(recursive=False)[0]
        error_table = []
        is_error_table = False
        for part in pcontent.find_all(recursive=False):
            if is_error_table:
                if part.name == "table":
                    error_table.append(part)
                else:
                    break
            if part.name == "div" and part.get_text(strip=True) == "中标候选单位：":
                is_error_table = True
        first_table = None
        for idx in range(len(error_table)):
            t_part = error_table[idx]
            # if t_part.name != "table":
            #     break
            if idx == 0:
                for _tr in t_part.find("tbody").find_all(recursive=False):
                    if _tr.get_text(strip=True) == "":
                        _tr.decompose()
                first_table = t_part
            else:
                for _tr in t_part.find("tbody").find_all(recursive=False):
                    if _tr.get_text(strip=True) != "":
                        first_table.find("tbody").append(_tr)
                t_part.clear()
    elif web_source_no == '18021-2':
        pcontent = sourceContent.find("div", id="pcontent")
        td = pcontent.find_all("td")
        for _td in td:
            if str(_td.string).strip() == "报价金额":
                _td.string = "单价"
    elif web_source_no == '13740-2':
        # “xxx成为成交供应商”
        re_match = re.search("[^，。]+成为[^，。]*成交供应商", sourceContent)
        if re_match:
            sourceContent = sourceContent.replace(re_match.group(), "成交人：" + re_match.group(), sourceContent)
    elif web_source_no == '03786-10':
        ser1 = re.search('中标价：([\d,.]+)', sourceContent)
        ser2 = re.search('合同金额[(（]万元[)）]：([\d,.]+)', sourceContent)
        if ser1 and ser2:
            m1 = ser1.group(1).replace(',', '')
            m2 = ser2.group(1).replace(',', '')
            if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'):
                new = '中标价(万元)：' + m1
                sourceContent = sourceContent.replace(ser1.group(0), new, 1)
    elif web_source_no=='00076-4':
        ser = re.search('主要标的数量：([0-9一]+)\w{,3}，主要标的单价：([\d,.]+)元?，合同金额：(.00)，', sourceContent)
        if ser:
            num = ser.group(1).replace('一', '1')
            try:
                num = 1 if num == '0' else num
                unit_price = ser.group(2).replace(',', '')
                total_price = str(int(num) * float(unit_price))
                new = '合同金额：' + total_price
                sourceContent = sourceContent.replace('合同金额：.00', new, 1)
            except Exception as e:
                log('preprocessing.py special_treatment exception')
    elif web_source_no=='DX000105-2':
        if re.search("成交公示", sourceContent) and re.search('，投标人：', sourceContent) and re.search('，成交人：', sourceContent)==None:
            sourceContent = sourceContent.replace('，投标人：', '，成交人：')
    elif web_source_no in ['04080-3', '04080-4']:
        ser = re.search('合同金额：([0-9,]+.[0-9]{3,})(.{,4})', sourceContent)
        if ser and '万' not in ser.group(2):
            sourceContent = sourceContent.replace('合同金额：', '合同金额(万元)：')
    elif web_source_no=='03761-3':
        ser = re.search('中标价，([0-9]+)[.0-9]*%', sourceContent)
        if ser and int(ser.group(1))>100:
            sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元')
    elif web_source_no=='00695-7':
        ser = re.search('支付金额：', sourceContent)
        if ser:
            sourceContent = sourceContent.replace('支付金额：', '合同金额：')
    return sourceContent

def article_limit(soup,limit_words=30000):
    sub_space = re.compile("\s+")
    def soup_limit(_soup,_count,max_count=30000,max_gap=500):
        """
        :param _soup: soup
        :param _count: 当前字数
        :param max_count: 字数最大限制
        :param max_gap: 超过限制后的最大误差
        :return:
        """
        _gap = _count - max_count
        _is_skip = False
        next_soup = None
        while len(_soup.find_all(recursive=False)) == 1 and \
                _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
            _soup = _soup.find_all(recursive=False)[0]
        try:
            for _soup_part in _soup.find_all(recursive=False):
                if not _is_skip:
                    _count += len(re.sub(sub_space, "", _soup_part.get_text()))
                    if _count >= max_count:
                        _gap = _count - max_count
                        if _gap <= max_gap:
                            _is_skip = True
                        else:
                            next_soup = _soup_part
                            _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
                            break
                else:
                    _soup_part.decompose()
        except:
            return _count,_gap,None
        return _count,_gap,next_soup

    text_count = 0
    have_attachment = False
    attachment_part = None
    for child in soup.find_all(recursive=True):
        if child.name == 'div' and 'class' in child.attrs:
            if "richTextFetch" in child['class']:
                child.insert_before("##attachment##")
                attachment_part = child
                have_attachment = True
                break
    if not have_attachment:
        # 无附件
        if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
            text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
    else:
        # 有附件
        _text = re.sub(sub_space, "", soup.get_text())
        _text_split = _text.split("##attachment##")
        if len(_text_split[0])>limit_words:
            main_soup = attachment_part.parent
            main_text = main_soup.find_all(recursive=False)[0]
            text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
        if len(_text_split[1])>limit_words:
            attachment_text_nums = 0
            attachment_skip = False
            for part in attachment_part.find_all(recursive=False):
                if not attachment_skip:
                    attachment_text_nums += len(re.sub(sub_space, "", part.get_text()))
                    if attachment_text_nums>=limit_words:
                        attachment_skip = True
                else:
                    part.decompose()

    return soup

def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
    '''
    :param articles: 待处理的article source html
    :param useselffool: 是否使用selffool
    :return: list_articles
    '''
    list_articles = []
    for article in articles:
        doc_id = article[0]
        sourceContent = article[1]
        sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)

        sourceContent = sourceContent.replace('<br/>', '<br>')
        sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
        # for br_match in re.findall("[^>]+?<br>",sourceContent):
        #     _new = re.sub("<br>","",br_match)
        #     # <br>标签替换为<p>标签
        #     if not re.search("^\s+$",_new):
        #         _new = '<p>'+_new + '</p>'
        #         # print(br_match,_new)
        #         sourceContent = sourceContent.replace(br_match,_new,1)
        _send_doc_id = article[3]
        _title = article[4]
        page_time = article[5]
        web_source_no = article[6]
        '''特别数据源对 html 做特别修改'''
        if web_source_no in ['DX000202-1']:
            sourceContent = special_treatment(sourceContent, web_source_no)

        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
        article_processed = BeautifulSoup(sourceContent,"lxml")
        '''特别数据源对 BeautifulSoup(html) 做特别修改'''
        if web_source_no in ["00753-14","DX008357-11","18021-2"]:
            article_processed = special_treatment(article_processed, web_source_no)
        for _soup in article_processed.descendants:
            # 识别无标签文本，添加<span>标签
            if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
                # print(_soup.parent.string,_soup.string.strip())
                _soup.wrap(article_processed.new_tag("span"))
        # print(article_processed)
        # 正文和附件内容限制字数30000
        article_processed = article_limit(article_processed,limit_words=30000)
        article_processed = get_preprocessed_outline(article_processed)
        article_processed = tableToText(article_processed)
        # print(article_processed)
        article_processed = segment(article_processed)
        article_processed = article_processed.replace('．','.') # 2021/12/01 修正OCR识别PDF小数点错误问题
        article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
        article_processed = article_processed.replace('成交工程价款', '成交工程价')  # 2021/12/21 修正为中标价
        # 修复OCR金额中“，”、“。”识别错误
        article_processed_list = article_processed.split("##attachment##")
        if len(article_processed_list)>1:
            attachment_text = article_processed_list[1]
            for _match in re.finditer("\d。\d{2}",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
            for _match in re.finditer("(\d，\d{3})[,，.]",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("，",","),1)
            article_processed_list[1] = attachment_text
            article_processed = "##attachment##".join(article_processed_list)
        '''特别数据源对 预处理后文本 做特别修改'''
        if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2"]:
            article_processed = special_treatment(article_processed, web_source_no)

        # 提取bidway
        list_bidway = extract_bidway(article_processed, _title)
        if list_bidway:
            bidway = list_bidway[0].get("body")
            # bidway名称统一规范
            bidway = bidway_integrate(bidway)
        else:
            bidway = ""

        # 修正被"，"逗号分隔的时间
        repair_time = re.compile("[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[:时点]，?[0-6]\d分?|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[时点]|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]|"
                                 "[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d"
                                 )
        for _time in set(re.findall(repair_time,article_processed)):
            if re.search("，",_time):
                _time2 = re.sub("，", "", _time)
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time2)
                if item:
                    _time2 = _time2.replace(item.group(),item.group() + " ")
                article_processed = article_processed.replace(_time, _time2)
            else:
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time)
                if item:
                    _time2 = _time.replace(item.group(),item.group() + " ")
                    article_processed = article_processed.replace(_time, _time2)
        # print('re_rtime',re.findall(repair_time,article_processed))
        # log(article_processed)

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += round(time.time()-start_time,2)

        #article_processed = article[1]
        _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
                           bidway=bidway)
        _article.fingerprint = getFingerprint(_title+sourceContent)
        _article.page_time = page_time
        list_articles.append(_article)
    return list_articles

def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
    '''

    :param list_articles: 经过预处理的article text
    :return: list_sentences
    '''
    list_sentences = []
    list_outlines = []
    for article in list_articles:
        list_sentences_temp = []
        list_entitys_temp = []
        doc_id = article.id
        _send_doc_id = article.doc_id
        _title = article.title
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = article.content
        attachment_begin_index = -1

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += time.time()-start_time

        #nlp处理
        if article_processed is not None and len(article_processed)!=0:
            split_patten = "。"
            sentences = []
            _begin = 0
            sentences_set = set()
            for _iter in re.finditer(split_patten,article_processed):
                _sen = article_processed[_begin:_iter.span()[1]]
                if len(_sen)>0 and _sen not in sentences_set:
                    # 标识在附件里的句子
                    if re.search("##attachment##",_sen):
                        attachment_begin_index = len(sentences)
                        # _sen = re.sub("##attachment##","",_sen)
                    sentences.append(_sen)
                    sentences_set.add(_sen)
                _begin = _iter.span()[1]
            _sen = article_processed[_begin:]
            if re.search("##attachment##", _sen):
                # _sen = re.sub("##attachment##", "", _sen)
                attachment_begin_index = len(sentences)
            if len(_sen)>0 and _sen not in sentences_set:
                sentences.append(_sen)
                sentences_set.add(_sen)
            # 解析outline大纲分段
            outline_list = []
            if re.search("##split##",article.content):
                temp_sentences = []
                last_sentence_index = (-1,-1)
                outline_index = 0
                for sentence_index in range(len(sentences)):
                    sentence_text = sentences[sentence_index]
                    for _ in re.findall("##split##", sentence_text):
                        _match = re.search("##split##", sentence_text)
                        if last_sentence_index[0] > -1:
                            sentence_begin_index,wordOffset_begin = last_sentence_index
                            sentence_end_index = sentence_index
                            wordOffset_end = _match.start()
                            if sentence_begin_index<attachment_begin_index and sentence_end_index>=attachment_begin_index:
                                outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1])))
                            else:
                                outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end))
                            outline_index += 1
                        sentence_text = re.sub("##split##", "", sentence_text,count=1)
                        last_sentence_index = (sentence_index,_match.start())
                    temp_sentences.append(sentence_text)
                if attachment_begin_index>-1 and last_sentence_index[0]<attachment_begin_index:
                    outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],attachment_begin_index-1,last_sentence_index[1],len(temp_sentences[attachment_begin_index-1])))
                else:
                    outline_list.append(Outline(doc_id,outline_index,'',last_sentence_index[0],len(sentences)-1,last_sentence_index[1],len(temp_sentences[-1])))
                sentences = temp_sentences
            #解析outline的outline_text内容
            for _outline in outline_list:
                if _outline.sentence_begin_index==_outline.sentence_end_index:
                    _text = sentences[_outline.sentence_begin_index][_outline.wordOffset_begin:_outline.wordOffset_end]
                else:
                    _text = ""
                    for idx in range(_outline.sentence_begin_index,_outline.sentence_end_index+1):
                        if idx==_outline.sentence_begin_index:
                            _text += sentences[idx][_outline.wordOffset_begin:]
                        elif idx==_outline.sentence_end_index:
                            _text += sentences[idx][:_outline.wordOffset_end]
                        else:
                            _text += sentences[idx]
                _outline.outline_text = _text
                _outline_summary = re.split("[:：，]",_text,1)[0]
                if len(_outline_summary)<20:
                    _outline.outline_summary = _outline_summary
                # print(_outline.outline_index,_outline.outline_text)

            article.content = "".join(sentences)
            # sentences.append(article_processed[_begin:])

            lemmas = []
            doc_offsets = []
            dep_types = []
            dep_tokens = []

            time1 = time.time()

            '''
            tokens_all = fool.cut(sentences)
            #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
            #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
            ner_entitys_all = fool.ner(sentences)
            '''
            #限流执行
            key_nerToken = "nerToken"
            start_time = time.time()
            tokens_all = getTokens(sentences,useselffool=useselffool)
            if key_nerToken not in cost_time:
                cost_time[key_nerToken] = 0
            cost_time[key_nerToken] += round(time.time()-start_time,2)

            in_attachment = False
            for sentence_index in range(len(sentences)):
                if sentence_index == attachment_begin_index:
                    in_attachment = True
                sentence_text = sentences[sentence_index]
                tokens = tokens_all[sentence_index]

                #pos_tag = pos_all[sentence_index]
                pos_tag = ""

                ner_entitys = ""

                list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment))
        if len(list_sentences_temp)==0:
            list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
        list_sentences.append(list_sentences_temp)
        list_outlines.append(outline_list)
    return list_sentences,list_outlines

def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
    '''

    :param list_sentences:分局情况
    :param cost_time:
    :return: list_entitys
    '''

    list_entitys = []
    for list_sentence in list_sentences:
        sentences = []
        list_entitys_temp = []
        for _sentence in list_sentence:
            sentences.append(_sentence.sentence_text)

        lemmas = []
        doc_offsets = []
        dep_types = []
        dep_tokens = []

        time1 = time.time()

        '''
        tokens_all = fool.cut(sentences)
        #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
        #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
        ner_entitys_all = fool.ner(sentences)
        '''
        #限流执行
        key_nerToken = "nerToken"
        start_time = time.time()
        found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
        # found_pingbiao = False
        ner_entitys_all = getNers(sentences,useselffool=useselffool)
        if key_nerToken not in cost_time:
            cost_time[key_nerToken] = 0
        cost_time[key_nerToken] += round(time.time()-start_time,2)

        company_dict = set()
        company_index = dict((i,set()) for i in range(len(list_sentence)))
        for sentence_index in range(len(list_sentence)):
            list_sentence_entitys = []

            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            in_attachment = list_sentence[sentence_index].in_attachment
            list_tokenbegin = []
            begin = 0
            for i in range(0,len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin+1)
            #pos_tag = pos_all[sentence_index]
            pos_tag = ""

            ner_entitys = ner_entitys_all[sentence_index]

            '''正则识别角色实体  经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处'''
            for it in re.finditer(
                    '(?P<text_key_word>(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为:：]+)(?P<text>([^，。、；《:：]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[，。]',
                    sentence_text):
                for k, v in it.groupdict().items():
                    if k == 'text_key_word':
                        keyword = v
                    if k == 'text':
                        entity = v
                b = it.start() + len(keyword)
                e = it.end() - 1
                if (b, e, 'location', entity) in ner_entitys:
                    ner_entitys.remove((b, e, 'location', entity))
                    ner_entitys.append((b, e, 'company', entity))
                elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
                    ner_entitys.append((b, e, 'company', entity))

            for it in re.finditer(
                    '(?P<text_key_word>((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为:：]+)(?P<text>\w{2,4}[省市县区镇]([^，。、；《]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[，。]',
                    sentence_text):
                for k, v in it.groupdict().items():
                    if k == 'text_key_word':
                        keyword = v
                    if k == 'text':
                        entity = v
                b = it.start() + len(keyword)
                e = it.end() - 1
                if (b, e, 'location', entity) in ner_entitys:
                    ner_entitys.remove((b, e, 'location', entity))
                    ner_entitys.append((b, e, 'org', entity))
                if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys:
                    ner_entitys.append((b, e, 'org', entity))

            for ner_entity in ner_entitys:
                if ner_entity[2] in ['company','org']:
                    company_dict.add((ner_entity[2],ner_entity[3]))
                    company_index[sentence_index].add((ner_entity[0],ner_entity[1]))
            #识别package

            #识别实体
            for ner_entity in ner_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]

                if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
                    continue

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                entity_text = re.sub("[,，。：!&@$\*]","",entity_text)
                entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text
                list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
            # 标记文章末尾的"发布人”、“发布时间”实体
            if sentence_index==len(list_sentence)-1:
                if len(list_sentence_entitys[-2:])>2:
                    second2last = list_sentence_entitys[-2]
                    last = list_sentence_entitys[-1]
                    if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
                            second2last.entity_type=="time" and last.entity_type in ["company",'org']):
                        if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
                            last.is_tail = True
                            second2last.is_tail = True

            #使用正则识别金额
            entity_type = "money"
            #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"

            # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
            #                       "key_word":"((?P<text_key_word>(?:[￥¥]+，?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果：)(?:[,（\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[）\)]?))",
            #                       "front_m":"((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)())",
            #                       "behind_m":"(()()(?P<money_behind_m>[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\)）]?)"}
            list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
                                  "key_word": "((?P<text_key_word>(?:[￥¥]+，?|[单报标限总]价|金额|成交报?价|价格|预算|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,，（\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]{,1})(?:[（\(]?(?P<filter_>[%])*\s*(单位[:：])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[）\)]?))",
                                  "front_m":"((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]*)())",
                                  "behind_m":"(()()(?P<money_behind_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]*)(人民币)?[\(（]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\)）]?)"}
            # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。

            pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
            set_begin = set()
            # for pattern_key in list_money_pattern.keys():
            # for pattern_key in ["cn","key_word","behind_m","front_m"]:
            #     # pattern = re.compile(list_money_pattern[pattern_key])
            #     pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?\s*([万元]*)\s*[）\)]?))*|(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*|((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*")
            #     all_match = re.findall(pattern, sentence_text)
                # index = 0
                # for i in range(len(all_match)):
                #     if len(all_match[i][0])>0:
                #         print("===",all_match[i])
                #         #print(all_match[i][0])
                #         unit = ""
                #         entity_text = all_match[i][3]
                #         if pattern_key in ["key_word","front_m"]:
                #             unit = all_match[i][1]
                #             if pattern_key=="key_word":
                #                 if all_match[i][1]=="" and all_match[i][4]!="":
                #                     unit = all_match[i][4]
                #         else:
                #             unit = all_match[i][4]
                #         if entity_text.find("元")>=0:
                #             unit = ""
                #
                #         index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
                #         begin_index_temp = index
                #         for j in range(len(list_tokenbegin)):
                #             if list_tokenbegin[j]==index:
                #                 begin_index = j
                #                 break
                #             elif list_tokenbegin[j]>index:
                #                 begin_index = j-1
                #                 break
                #         index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
                #         end_index_temp = index
                #         #index += len(str(all_match[i][0]))
                #         for j in range(begin_index,len(list_tokenbegin)):
                #             if list_tokenbegin[j]>=index:
                #                 end_index = j-1
                #                 break
                #         entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
                #
                #         entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
                #         if len(unit)>0:
                #             entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                #         else:
                #             entity_text = str(getUnifyMoney(entity_text))
                #
                #         _exists = False
                #         for item in list_sentence_entitys:
                #             if item.entity_id==entity_id and item.entity_type==entity_type:
                #                 _exists = True
                #         if not _exists:
                #             if float(entity_text)>1:
                #                 list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
                #
                #     else:
                #         index += 1

            # if re.search('评标结果|候选人公示', sentence_text):
            #     found_pingbiao = True
            if re.search('业绩', sentence_text):
                found_yeji += 1
            if found_yeji >= 2: # 过滤掉业绩后面的所有金额
                all_match = []
            else:
                all_match = re.finditer(pattern_money, sentence_text)
            index = 0
            for _match in all_match:
                if len(_match.group())>0:
                    # print("===",_match.group())
                    # # print(_match.groupdict())
                    notes = ''  # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
                    unit = ""
                    entity_text = ""
                    text_beforeMoney = ""
                    filter = ""
                    filter_unit = False
                    notSure = False
                    if re.search('业绩', sentence_text[:_match.span()[0]]):  # 2021/7/21过滤掉业绩后面金额
                        # print('金额在业绩后面: ', _match.group(0))
                        found_yeji += 1
                        break

                    for k,v in _match.groupdict().items():
                        if v!="" and v is not None:
                            if k=='text_key_word':
                                notSure = True
                            if k.split("_")[0]=="money":
                                entity_text = v
                            if k.split("_")[0]=="unit":
                                unit = v
                            if k.split("_")[0]=="text":
                                text_beforeMoney = v
                            if k.split("_")[0]=="filter":
                                filter = v
                            if re.search("filter_unit",k) is not None:
                                filter_unit = True
                    # print(_match.group())
                    # print(entity_text,unit,text_beforeMoney,filter,filter_unit)
                    if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()):  # 2021/7/19 修正OCR识别小数点为逗号
                        if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
                             entity_text = re.sub('\d+,', '', entity_text)
                        else:
                            entity_text = entity_text.replace(',', '.')
                        # print(' 修正OCR识别小数点为逗号')

                    if entity_text.find("元")>=0:
                        unit = ""
                    if unit == "":  #2021/7/21 有明显金额特征的补充单位，避免被过滤
                        if ('￥' in text_beforeMoney or '¥' in text_beforeMoney):
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                        elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[:：为]+$', text_beforeMoney.strip()) and \
                                re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                        elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)',entity_text):
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                    if unit.find("万") >= 0 and entity_text.find("万") >= 0:  #2021/7/19修改为金额文本有万，不计算单位
                        # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit)
                        unit = "元"
                    if re.search('.*万元万元', entity_text):  #2021/7/19 修正两个万元
                        # print(' 修正两个万元',entity_text)
                        entity_text = entity_text.replace('万元万元','万元')
                    else:
                        if filter_unit:
                            continue
                    if filter!="":
                        continue

                    index = _match.span()[0]+len(text_beforeMoney)
                    begin_index_temp = index
                    for j in range(len(list_tokenbegin)):
                        if list_tokenbegin[j]==index:
                            begin_index = j
                            break
                        elif list_tokenbegin[j]>index:
                            begin_index = j-1
                            break
                    index = _match.span()[1]
                    end_index_temp = index
                    #index += len(str(all_match[i][0]))
                    for j in range(begin_index,len(list_tokenbegin)):
                        if list_tokenbegin[j]>=index:
                            end_index = j-1
                            break
                    entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                    entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
                    # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
                    if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/8/5过滤掉总投资金额
                        # print('总投资金额: ', _match.group(0))
                        notes = '总投资'
                    elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/11/18 投资金额不作为招标金额
                        notes = '投资'
                    elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/12/20 工程造价不作为招标金额
                        notes = '工程造价'
                    elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]])
                          or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\(（]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\)）]*[:：为]',
                                       sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]])
                          or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为',
                                       sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])):
                        notes = '保证金'
                        # print('保证金信息：', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])
                    elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}',
                                   sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]):
                        notes = '成本警戒线'
                    elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为：]', sentence_text[_match.span()[0]:_match.span()[1]]):
                        cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
                        notes = cost_re.group(1)
                    elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]):
                        notes = '单价'
                    elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
                        notes = '大写'
                        if entity_text[0] == "拾":  # 2021/12/16 修正大写金额省略了数字转换错误问题
                            entity_text = "壹"+entity_text
                        # print("补充备注：notes = 大写")
                    if len(unit)>0:
                        if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
                            # print('修正单位万元金额过大的情况 金额：', entity_text, '单位:', unit)
                            entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
                            unit = '元' # 修正金额后单位 重置为元
                        else:
                            # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
                            entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                    else:
                        if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
                            entity_text = str(getUnifyMoney(entity_text)/10000)
                            # print('修正金额字段含万 过大的情况')
                        else:
                            entity_text = str(getUnifyMoney(entity_text))

                    if float(entity_text)>100000000000:  # float(entity_text)<100 or  2022/3/4 取消最小金额限制
                        # print('过滤掉金额：float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
                        continue

                    if notSure and unit=="" and float(entity_text)>100*10000:
                        # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit)
                        continue

                    _exists = False
                    for item in list_sentence_entitys:
                        if item.entity_id==entity_id and item.entity_type==entity_type:
                            _exists = True
                        if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
                            _exists = True
                    if not _exists:
                        if float(entity_text)>1:
                            list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp,in_attachment=in_attachment))
                            list_sentence_entitys[-1].notes = notes  # 2021/7/20 新增金额备注
                            list_sentence_entitys[-1].money_unit = unit  # 2021/7/20 新增金额备注
                            # print('预处理中的 金额:%s, 单位：%s'%(entity_text,unit))
                            # print(entity_text,unit,notes)

                else:
                    index += 1

            # "联系人"正则补充提取  2021/11/15 新增
            list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person']
            error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮',
                          '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理',
                          '代理人','采购','附件','注意','登录','报名','踏勘']
            list_person_text = set(list_person_text + error_text)
            re_person = re.compile("联系人[:：]([\u4e00-\u9fa5]工)|"
                                   "联系人[:：]([\u4e00-\u9fa5]{2,3})(?=联系)|"
                                   "联系人[:：]([\u4e00-\u9fa5]{2,3})")
            list_person = []
            for match_result in re_person.finditer(sentence_text):
                match_text = match_result.group()
                entity_text = match_text[4:]
                wordOffset_begin = match_result.start() + 4
                wordOffset_end = match_result.end()
                # print(text[wordOffset_begin:wordOffset_end])
                # 排除一些不为人名的实体
                if re.search("^[\u4e00-\u9fa5]{7,}([，。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]):
                    continue
                if entity_text not in list_person_text and entity_text[:2] not in list_person_text:
                    _person = dict()
                    _person['body'] = entity_text
                    _person['begin_index'] = wordOffset_begin
                    _person['end_index'] = wordOffset_end
                    list_person.append(_person)
            entity_type = "person"
            for person in list_person:
                begin_index_temp = person['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = person['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = person['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 资金来源提取  2020/12/30 新增
            list_moneySource = extract_moneySource(sentence_text)
            entity_type = "moneysource"
            for moneySource in list_moneySource:
                begin_index_temp = moneySource['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = moneySource['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = moneySource['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 电子邮箱提取 2021/11/04 新增
            list_email = extract_email(sentence_text)
            entity_type = "email" # 电子邮箱
            for email in list_email:
                begin_index_temp = email['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = email['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = email['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 服务期限提取 2020/12/30 新增
            list_servicetime = extract_servicetime(sentence_text)
            entity_type = "serviceTime"
            for servicetime in list_servicetime:
                begin_index_temp = servicetime['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = servicetime['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = servicetime['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            # 招标方式提取 2020/12/30 新增
            # list_bidway = extract_bidway(sentence_text, )
            # entity_type = "bidway"
            # for bidway in list_bidway:
            #     begin_index_temp = bidway['begin_index']
            #     end_index_temp = bidway['end_index']
            #     begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
            #     end_index = changeIndexFromWordToWords(tokens, end_index_temp)
            #     if begin_index is None or end_index is None:
            #         continue
            #     print(begin_index_temp,end_index_temp,begin_index,end_index)
            #     entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
            #     entity_text = bidway['body']
            #     list_sentence_entitys.append(
            #         Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
            #                begin_index_temp, end_index_temp))

            # 2021/12/29 新增比率提取
            list_ratio = extract_ratio(sentence_text)
            entity_type = "ratio"
            for ratio in list_ratio:
                # print("ratio", ratio)
                begin_index_temp = ratio['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = ratio['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = ratio['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp,in_attachment=in_attachment))

            list_sentence_entitys.sort(key=lambda x:x.begin_index)
            list_entitys_temp = list_entitys_temp+list_sentence_entitys
        # 补充ner模型未识别全的company/org实体
        for sentence_index in range(len(list_sentence)):
            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            in_attachment = list_sentence[sentence_index].in_attachment
            list_tokenbegin = []
            begin = 0
            for i in range(0, len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin + 1)
            add_sentence_entitys = []
            company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True)
            for company_type,company_text in company_dict:
                begin_index_list = findAllIndex(company_text,sentence_text)
                for begin_index in begin_index_list:
                    is_continue = False
                    for t_begin,t_end in list(company_index[sentence_index]):
                        if begin_index>=t_begin and begin_index+len(company_text)<=t_end:
                            is_continue = True
                            break
                    if not is_continue:
                        add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text))
                        company_index[sentence_index].add((begin_index,begin_index+len(company_text)))
                    else:
                        continue
            for ner_entity in add_sentence_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]

                if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
                    continue

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                entity_text = re.sub("[,，。：!&@$\*]","",entity_text)
                entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text
                list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment))
        list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index))
        list_entitys.append(list_entitys_temp)
    return list_entitys
    
def union_result(codeName,prem):
    '''
    @summary:模型的结果拼成字典
    @param:
        codeName:编号名称模型的结果字典
        prem:拿到属性的角色的字典
    @return:拼接起来的字典
    '''
    result = []
    assert len(codeName)==len(prem)
    for item_code,item_prem in zip(codeName,prem):
        result.append(dict(item_code,**item_prem))
    return result

def persistenceData(data):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for item_index in range(len(data)):
        item = data[item_index]
        doc_id = item[0]
        dic = item[1]
        code = dic['code']
        name = dic['name']
        prem = dic['prem']
        if len(code)==0:
            code_insert = ""
        else:
            code_insert = ";".join(code)
        prem_insert = ""
        for item in prem:
            for x in item:
                if isinstance(x, list):
                    if len(x)>0:
                        for x1 in x:
                            prem_insert+="/".join(x1)+","
                    prem_insert+="$"
                else:
                    prem_insert+=str(x)+"$"
            prem_insert+=";"
        sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
        cursor.execute(sql)
    conn.commit()
    conn.close()
   
def persistenceData1(list_entitys,list_sentences):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for list_entity in list_entitys:
        for entity in list_entity:
            if entity.values is not None:
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
            else:    
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
            cursor.execute(sql)
    for list_sentence in list_sentences:
        for sentence in list_sentence:
            str_tokens = "["
            for item in sentence.tokens:
                str_tokens += "'"
                if item=="'":
                    str_tokens += "''"
                else:
                    str_tokens += item
                str_tokens += "',"
            str_tokens = str_tokens[:-1]+"]"
            sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
            cursor.execute(sql)
    conn.commit()
    conn.close()


def _handle(item,result_queue):
    dochtml = item["dochtml"]
    docid = item["docid"]
    list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
    flag = False
    if list_innerTable:
        flag = True
        for table in list_innerTable:
            result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})

def getPredictTable():
    filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
    import pandas as pd
    import json
    from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
    df = pd.read_csv(filename)
    df_data = {"json_table":[],"docid":[]}
    _count = 0
    _sum = len(df["docid"])
    task_queue = Queue()
    result_queue = Queue()
    _index = 0
    for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
        task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
        _index += 1

    mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
    mh.run()
    while True:
        try:
            item = result_queue.get(block=True,timeout=1)
            df_data["docid"].append(item["docid"])
            df_data["json_table"].append(item["json_table"])
        except Exception as e:
            print(e)
            break
    df_1 = pd.DataFrame(df_data)
    df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])

if __name__=="__main__":
    '''
    import glob
    for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
        file_txt = str(file).replace("html","txt")
        with codecs.open(file_txt,"a+",encoding="utf8") as f:
            f.write("\n================\n")
            content = codecs.open(file,"r",encoding="utf8").read()
            f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
    '''        
    # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
    # getPredictTable()
    with open('D:/138786703.html', 'r', encoding='utf-8') as f:
        sourceContent = f.read()
        # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
        # print(article_processed)

        list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
        for entity in list_entitys[0]:
            print(entity.entity_type, entity.entity_text)