BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/Preprocessing.py

# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup, Comment
import copy
import sys
import os
import time
import codecs
sys.setrecursionlimit(1000000)
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath(".."))
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Entitys import *
from BiddingKG.dl.interface.predictor import getPredictor
from BiddingKG.dl.common.nerUtils import *
from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
from BiddingKG.dl.time.re_servicetime import extract_servicetime
from BiddingKG.dl.bidway.re_bidway import extract_bidway
from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
from BiddingKG.dl.entityLink.entityLink import *


#
def tableToText(soup):
    '''
    @param:
        soup:网页html的soup
    @return:处理完表格信息的网页text
    '''
    
    def getTrs(tbody):
    #获取所有的tr
        trs = []
        objs = tbody.find_all(recursive=False)
        for obj in objs:
            if obj.name=="tr":
                trs.append(obj)
            if obj.name=="tbody":
                for tr in obj.find_all("tr",recursive=False):
                    trs.append(tr)
        return trs

    def fixSpan(tbody):
        # 处理colspan, rowspan信息补全问题
        #trs = tbody.findChildren('tr', recursive=False)

        trs = getTrs(tbody)
        ths_len = 0
        ths = list()
        trs_set = set()
        #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱
        # 遍历每一个tr

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有colspan 则补全同一行下一个位置
                if 'colspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
                        col = int(re.sub("[^0-9]","",str(td['colspan'])))
                        if col<100 and len(td.get_text())<1000:
                            td['colspan'] = 1
                            for i in range(1, col, 1):
                                td.insert_after(copy.copy(td))

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有rowspan 则补全下一行同样位置
                if 'rowspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
                        row = int(re.sub("[^0-9]","",str(td['rowspan'])))
                        td['rowspan'] = 1
                        for i in range(1, row, 1):
                            # 获取下一行的所有td， 在对应的位置插入
                            if indtr+i<len(trs):
                                tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
                                if len(tds1) >= (indtd) and len(tds1)>0:
                                    if indtd > 0:
                                        tds1[indtd - 1].insert_after(copy.copy(td))
                                    else:
                                        tds1[0].insert_before(copy.copy(td))
                                elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1:  # 修正某些表格最后一列没补全
                                    tds1[indtd-2].insert_after(copy.copy(td))
    def getTable(tbody):
        #trs = tbody.findChildren('tr', recursive=False)
        trs = getTrs(tbody)
        inner_table = []
        for tr in trs:
            tr_line = []
            tds = tr.findChildren(['td','th'], recursive=False)
            for td in tds:
                tr_line.append([re.sub('\xa0','',segment(td,final=False)),0])
                #tr_line.append([td.get_text(),0])
            inner_table.append(tr_line)
        return inner_table                          
    
    #处理表格不对齐的问题
    def fixTable(inner_table,fix_value="~~"):
        maxWidth = 0
        for item in inner_table:
            if len(item)>maxWidth:
                maxWidth = len(item)
        for i in range(len(inner_table)):
            if len(inner_table[i])<maxWidth:
                for j in range(maxWidth-len(inner_table[i])):
                    inner_table[i].append([fix_value,0])
        return inner_table
    
    def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            point = ""
            for j in range(width):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_row
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        for j in range(width):
            point = ""
            for i in range(height):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_col
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        
    def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if inner_table[i][j][0]==pad_row:
                    inner_table[i][j][0] = inner_table[i][j-1][0]
                    inner_table[i][j][1] = inner_table[i][j-1][1]
                if inner_table[i][j][0]==pad_col:
                    inner_table[i][j][0] = inner_table[i-1][j][0]
                    inner_table[i][j][1] = inner_table[i-1][j][1]

    def repairTable(inner_table,dye_set = set(),key_set = set(),fix_value="~~"):
        '''
        @summary: 修复表头识别，将明显错误的进行修正
        '''
        def repairNeeded(line):
            first_1 = -1
            last_1 = -1
            first_0 = -1
            last_0 = -1
            count_1 = 0
            count_0 = 0
            for i in range(len(line)):
                if line[i][0]==fix_value:
                    continue
                if line[i][1]==1:
                    if first_1==-1:
                        first_1 = i
                    last_1 = i
                    count_1 += 1
                if line[i][1]==0:
                    if first_0 == -1:
                        first_0 = i
                    last_0 = i
                    count_0 += 1
            if first_1 ==-1 or last_0 == -1:
                return False
            #异常情况：第一个不是表头；最后一个是表头；表头个数远大于属性值个数
            if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
                return True
            return False

        def getsimilarity(line,line1):
            same_count = 0
            for item,item1 in zip(line,line1):
                if item[1]==item1[1]:
                    same_count += 1
            return same_count/len(line)

        def selfrepair(inner_table,index,dye_set,key_set):
            '''
            @summary: 计算每个节点受到的挤压度来判断是否需要染色
            '''
            #print("B",inner_table[index])
            min_presure = 3
            list_dye = []
            first = None
            count = 0
            temp_set = set()
            _index = 0
            for item in inner_table[index]:
                if first is None:
                    first = item[1]
                    if item[0] not in temp_set:
                        count += 1
                        temp_set.add(item[0])
                else:
                    if first == item[1]:
                        if item[0] not in temp_set:
                            temp_set.add(item[0])
                            count += 1
                    else:
                        list_dye.append([first,count,_index])
                        first = item[1]
                        temp_set.add(item[0])
                        count = 1
                _index += 1
            list_dye.append([first,count,_index])
            if len(list_dye)>1:
                begin = 0
                end = 0
                for i in range(len(list_dye)):
                    end = list_dye[i][2]
                    dye_flag = False
                    #首尾要求压力减一
                    if i==0:
                        if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i+1][0]
                    elif i==len(list_dye)-1:
                        if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
                            dye_flag = True
                            dye_type = list_dye[i-1][0]
                    else:
                        if list_dye[i][1]>1:
                            if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                        else:
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                            if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                    if dye_flag:
                        for h in range(begin,end):
                            inner_table[index][h][1] = dye_type
                            dye_set.add((inner_table[index][h][0],dye_type))
                            key_set.add(inner_table[index][h][0])
                    begin = end
                #print("E",inner_table[index])




        def otherrepair(inner_table,index,dye_set,key_set):
            list_provide_repair = []
            if index==0 and len(inner_table)>1:
                list_provide_repair.append(index+1)
            elif index==len(inner_table)-1:
                list_provide_repair.append(index-1)
            else:
                list_provide_repair.append(index+1)
                list_provide_repair.append(index-1)
            for provide_index in list_provide_repair:
                if not repairNeeded(inner_table[provide_index]):
                    same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
                    if same_prob>=0.8:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = inner_table[provide_index][i][1]
                    elif same_prob<=0.2:
                        for i in range(len(inner_table[provide_index])):
                            if inner_table[index][i][1]==inner_table[provide_index][i][1]:
                                dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                key_set.add(inner_table[index][i][0])
                            inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
        len_dye_set = len(dye_set)
        height = len(inner_table)
        for i in range(height):
            if repairNeeded(inner_table[i]):
                selfrepair(inner_table,i,dye_set,key_set)
                #otherrepair(inner_table,i,dye_set,key_set)
        for h in range(len(inner_table)):
            for w in range(len(inner_table[0])):
                if inner_table[h][w][0] in key_set:
                    for item in dye_set:
                        if inner_table[h][w][0]==item[0]:
                            inner_table[h][w][1] = item[1]
        #如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代
        if len(dye_set)!=len(key_set):
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    selfrepair(inner_table,i,dye_set,key_set)
                    #otherrepair(inner_table,i,dye_set,key_set)
            return
        if len(dye_set)==len_dye_set:
            '''
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    otherrepair(inner_table,i,dye_set,key_set)
            '''
            return
        repairTable(inner_table, dye_set, key_set)

    def sliceTable(inner_table,fix_value="~~"):
        #进行分块
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        last_head = None
        last_is_same_value = False
        for h in range(height):
            is_all_key = True#是否是全表头行
            is_all_value = True#是否是全属性值
            is_same_with_lastHead = True#和上一行的结构是否相同
            is_same_value=True#一行的item都一样
            #is_same_first_item = True#与上一行的第一项是否相同
            same_value = inner_table[h][0][0]
            for w in range(width):
                if last_head is not None:
                    if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
                        is_all_key = False

                    if inner_table[h][w][0]==1:
                        is_all_value = False

                    if inner_table[h][w][1]!= inner_table[h-1][w][1]:
                        is_same_with_lastHead = False

                if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
                    is_same_value = False
                else:
                    if re.search("\d+",same_value) is not None:
                        is_same_value = False
            if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
                is_same_first_item = False

            last_head = h

            if last_is_same_value:
                last_is_same_value = is_same_value
                continue

            if is_same_value:
                head_list.append(h)
                last_is_same_value = is_same_value
                continue
            if not is_all_key:
                if not is_same_with_lastHead:
                    head_list.append(h)


        head_list.append(height)
        return head_list
                    
    def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):


        set_item = set()
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                set_item.add(item)
        list_item = list(set_item)
        x = []
        for item in list_item:
            x.append(getPredictor("form").encode(item))
        predict_y = getPredictor("form").predict(np.array(x),type="item")
        _dict = dict()
        
        for item,values in zip(list_item,list(predict_y)):
            _dict[item] = values[1]
            # print("##",item,values)
        #print(_dict)
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")
        
        repairTable(inner_table)
        head_list = sliceTable(inner_table)

        
        return inner_table,head_list

    def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5):

        data_x,data_position = getPredictor("form").getModel("context").encode(inner_table)
        predict_y = getPredictor("form").getModel("context").predict(data_x)

        for _position,_y in zip(data_position,predict_y):
            _w = _position[0]
            _h = _position[1]
            if _y[1]>prob_min:
                inner_table[_h][_w][1] = 1
            else:
                inner_table[_h][_w][1] = 0
                _item = inner_table[_h][_w][0]
                if re.search(pat_head,_item) is not None and len(_item)<8:
                    inner_table[_h][_w][1] = 1

        # print("=====")
        # for item in inner_table:
        #     print(item)
        # print("======")
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if re.search("[:：]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8:
                    inner_table[i][j][1] = 1

        repairTable(inner_table)
        head_list = sliceTable(inner_table)


        return inner_table,head_list
                    
    #设置表头
    def setHead_inline(inner_table,prob_min=0.64):
        pad_row = "@@"
        pad_col = "##"
        removePadding(inner_table, pad_row, pad_col)
        pad_pattern = re.compile(pad_row+"|"+pad_col)
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            
            is_head = False
            is_long_value = False
            
            #判断是否是全padding值
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
                    is_same_value = False
                    break
                
            #predict is head or not with model
            temp_item = ""
            for j in range(width):
                temp_item += inner_table[i][j][0]+"|"
            temp_item = re.sub(pad_pattern,"",temp_item)
            form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
            if form_prob is not None:
                if form_prob[0][1]>prob_min:
                    is_head = True
                else:
                    is_head = False
            
            #print(temp_item,form_prob)
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                #不把连续表头分开
                if not is_head_last:
                    head_list.append(i)
                if is_long_value or is_same_value:
                    head_list.append(i+1)
            if is_head:
                for j in range(width):
                    inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                is_head = False
                
                #predict is head or not with model
                temp_item = ""
                for j in range(head_begin,head_end):
                    temp_item += inner_table[j][i][0]+"|"
                temp_item = re.sub(pad_pattern,"",temp_item)
                form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
                if form_prob is not None:
                    if form_prob[0][1]>prob_min:
                        is_head = True
                    else:
                        is_head = False
                
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        addPadding(inner_table, pad_row, pad_col)
        return inner_table,head_list
    
    #设置表头
    def setHead_withRule(inner_table,pattern,pat_value,count):
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            set_match = set()
            is_head = False
            is_long_value = False
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value:
                    is_same_value = False
                    break
            for j in range(width):
                if re.search(pat_value,inner_table[i][j][0]) is not None:
                    is_head = False
                    break
                str_find = re.findall(pattern,inner_table[i][j][0])
                if len(str_find)>0:
                    set_match.add(inner_table[i][j][0])
                if len(set_match)>=count:
                    is_head = True
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                if not is_head_last:
                    head_list.append(i)
                if is_head:
                    for j in range(width):
                        inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                set_match = set()
                is_head = False
                for j in range(head_begin,head_end):
                    if re.search(pat_value,inner_table[j][i][0]) is not None:
                        is_head = False
                        break
                    str_find = re.findall(pattern,inner_table[j][i][0])
                    if len(str_find)>0:
                        set_match.add(inner_table[j][i][0])
                    if len(set_match)>=count:
                        is_head = True
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        return inner_table,head_list
    
    #取得表格的处理方向
    def getDirect(inner_table,begin,end):
        '''
        column_head = set()
        row_head = set()
        widths = len(inner_table[0])
        for height in range(begin,end):
            for width in range(widths):
                if inner_table[height][width][1] ==1:
                    row_head.add(height)
                if inner_table[height][width][1] ==2:
                    column_head.add(width)
        company_pattern = re.compile("公司")
        if 0 in column_head and begin not in row_head:
            return "column"
        if 0 in column_head and begin in row_head:
            for height in range(begin,end):
                count = 0
                count_flag = True
                for width_index in range(width):
                    if inner_table[height][width_index][1]==0:
                        if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                            count += 1
                        else:
                            count_flag = False
                if count_flag and count>=2:
                    return "column"
        return "row"
        '''
        count_row_keys = 0
        count_column_keys = 0
        width = len(inner_table[0])
        if begin<end:
            for w in range(len(inner_table[begin])):
                if inner_table[begin][w][1]!=0:
                    count_row_keys += 1
        for h in range(begin,end):
            if inner_table[h][0][1]!=0:
                count_column_keys += 1
                
        company_pattern = re.compile("有限(责任)?公司")
        for height in range(begin,end):
            count_set = set()
            count_flag = True
            for width_index in range(width):
                if inner_table[height][width_index][1]==0:
                    if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                        count_set.add(inner_table[height][width_index][0])
                    else:
                        count_flag = False
            if count_flag and len(count_set)>=2:
                return "column"
        if count_column_keys>count_row_keys:
            return "column"
        return "row"
        
            
    #根据表格处理方向生成句子，        
    def getTableText(inner_table,head_list,key_direct=False):
        # packPattern = "(标包|[标包][号段名])"
        packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))"  # 2020/11/23 大网站规则，补充采购类包名
        rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)"  # 2020/11/23 大网站规则，添加序号为排序
        entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))"
        moneyPattern = "([中投]标|报价)(金额|价)"
        height = len(inner_table)
        width = len(inner_table[0])
        text = ""
         
        for head_i in range(len(head_list)-1):
            
            head_begin = head_list[head_i]
            head_end = head_list[head_i+1]
                
            direct = getDirect(inner_table, head_begin, head_end)


            #若只有一行，则直接按行读取
            if head_end-head_begin==1:
                text_line = ""
                for i in range(head_begin,head_end):
                    for w in range(len(inner_table[i])):
                        if inner_table[i][w][1]==1:
                            _punctuation = ":"
                        else:
                            _punctuation = ","
                        if w>0:
                            if inner_table[i][w][0]!= inner_table[i][w-1][0]:
                                text_line += inner_table[i][w][0]+_punctuation
                        else:
                            text_line += inner_table[i][w][0]+_punctuation
                    text_line = text_line+"。" if text_line!="" else text_line
                text += text_line
            else:
                #构建一个共现矩阵
                table_occurence = []
                for i in range(head_begin,head_end):
                    line_oc = []
                    for j in range(width):
                        cell = inner_table[i][j]
                        line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0})
                    table_occurence.append(line_oc)


                occu_height = len(table_occurence)
                occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0
                #为每个属性值寻找表头
                for i in range(occu_height):
                    for j in range(occu_width):
                        cell = table_occurence[i][j]
                        #是属性值
                        if cell["type"]==0 and cell["text"]!="":
                            left_head = ""
                            top_head = ""

                            find_flag = False
                            temp_head = ""
                            for loop_i in range(1,i+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [1]
                                if table_occurence[i-loop_i][j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i-loop_i][j]["text"]!=temp_head:
                                            top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
                                    else:
                                        top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head
                                    find_flag = True
                                    temp_head = table_occurence[i-loop_i][j]["text"]
                                    table_occurence[i-loop_i][j]["occu_count"] += 1
                                else:
                                    #找到表头后遇到属性值就返回
                                    if find_flag:
                                        break


                            cell["top_head"] += top_head
                            find_flag = False
                            temp_head = ""



                            for loop_j in range(1,j+1):
                                if not key_direct:
                                    key_values = [1,2]
                                else:
                                    key_values = [2]
                                if table_occurence[i][j-loop_j]["type"] in key_values:
                                    if find_flag:
                                        if table_occurence[i][j-loop_j]["text"]!=temp_head:
                                            left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
                                    else:
                                        left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head
                                    find_flag = True
                                    temp_head = table_occurence[i][j-loop_j]["text"]
                                    table_occurence[i][j-loop_j]["occu_count"] += 1
                                else:
                                    if find_flag:
                                        break
                            cell["left_head"] += left_head
                if direct=="row":
                    for i in range(occu_height):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        money_text = ""
                        #在同一句话中重复的可以去掉
                        text_set = set()
                        for j in range(width):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["top_head"]+":") if len(cell["top_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["left_head"] + head
                                else:
                                    head += cell["left_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None:
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    if re.search(moneyPattern,head) is not None and entity_text!="":
                                        money_text += head+cell["text"]+","
                                    else:
                                        text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))

                        text += pack_text+rank_text+entity_text+money_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text

                else:
                    for j in range(occu_width):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        text_set = set()
                        for i in range(occu_height):
                            cell = table_occurence[i][j]
                            if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0):

                                cell = table_occurence[i][j]
                                head = (cell["left_head"]+"") if len(cell["left_head"])>0 else ""
                                if re.search("单报标限总]价|金额|成交报?价|报价", head):
                                    head = cell["top_head"] + head
                                else:
                                    head += cell["top_head"]
                                if str(head+cell["text"]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+cell["text"]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+cell["text"]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None and \
                                        re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题
                                    entity_text += head+cell["text"]+"，"
                                    #print(entity_text)
                                else:
                                    text_line += head+cell["text"]+"，"
                                text_set.add(str(head+cell["text"]))
                        text += pack_text+rank_text+entity_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text


                # if direct=="row":
                #     for i in range(head_begin,head_end):
                #         pack_text = ""
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         #在同一句话中重复的可以去掉
                #         text_set = set()
                #         for j in range(width):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 head = ""
                #
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         #找到表头后遇到属性值就返回
                #                         if find_flag:
                #                             break
                #
                #                 find_flag = False
                #                 temp_head = ""
                #
                #
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(packPattern,head) is not None:
                #                     pack_text += head+inner_table[i][j][0]+"，"
                #                 elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                #                     #排名替换为同一种表达
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += pack_text+rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
                # else:
                #     for j in range(width):
                #
                #         rank_text = ""
                #         entity_text = ""
                #         text_line = ""
                #         text_set = set()
                #         for i in range(head_begin,head_end):
                #             cell = inner_table[i][j]
                #             #是属性值
                #             if cell[1]==0 and cell[0]!="":
                #                 find_flag = False
                #                 head = ""
                #                 temp_head = ""
                #
                #                 for loop_j in range(1,j+1):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [2]
                #                     if inner_table[i][j-loop_j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i][j-loop_j][0]!=temp_head:
                #                                 head = inner_table[i][j-loop_j][0]+":"+head
                #                         else:
                #                             head = inner_table[i][j-loop_j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i][j-loop_j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 find_flag = False
                #                 temp_head = ""
                #                 for loop_i in range(0,i+1-head_begin):
                #                     if not key_direct:
                #                         key_values = [1,2]
                #                     else:
                #                         key_values = [1]
                #                     if inner_table[i-loop_i][j][1] in key_values:
                #                         if find_flag:
                #                             if inner_table[i-loop_i][j][0]!=temp_head:
                #                                 head = inner_table[i-loop_i][j][0]+":"+head
                #                         else:
                #                             head = inner_table[i-loop_i][j][0]+":"+head
                #                         find_flag = True
                #                         temp_head = inner_table[i-loop_i][j][0]
                #                     else:
                #                         if find_flag:
                #                             break
                #                 if str(head+inner_table[i][j][0]) in text_set:
                #                     continue
                #                 if re.search(rankPattern,head) is not None:
                #                     rank_text += head+inner_table[i][j][0]+"，"
                #                     #print(rank_text)
                #                 elif re.search(entityPattern,head) is not None:
                #                     entity_text += head+inner_table[i][j][0]+"，"
                #                     #print(entity_text)
                #                 else:
                #                     text_line += head+inner_table[i][j][0]+"，"
                #                 text_set.add(str(head+inner_table[i][j][0]))
                #         text += rank_text+entity_text+text_line
                #         text = text[:-1]+"。" if len(text)>0 else text
        return text
    
    def removeFix(inner_table,fix_value="~~"):
        height = len(inner_table)
        width = len(inner_table[0])
        for h in range(height):
            for w in range(width):
                if inner_table[h][w][0]==fix_value:
                    inner_table[h][w][0] = ""
    
    def trunTable(tbody):
        fixSpan(tbody)
        inner_table = getTable(tbody)
        inner_table = fixTable(inner_table)
        if len(inner_table)>0 and len(inner_table[0])>0:
            #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
            #inner_table,head_list = setHead_inline(inner_table)
            # inner_table,head_list = setHead_initem(inner_table,pat_head)
            inner_table,head_list = setHead_incontext(inner_table,pat_head)
            # print(inner_table)
            # for begin in range(len(head_list[:-1])):
            #     for item in inner_table[head_list[begin]:head_list[begin+1]]:
            #         print(item)
            #     print("====")

            removeFix(inner_table)
            
            # print("----")
            # print(head_list)
            # for item in inner_table:
            #     print(item)


            tbody.string = getTableText(inner_table,head_list)
            #print(tbody.string)
            tbody.name = "turntable"
            return inner_table
        return None


        
    pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
    #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
    pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")

    list_innerTable = []
    tbodies = soup.find_all('tbody')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)
    '''2021/10/19先找tbody 再找table,避免一个table内多个tbody造成数据丢失'''
    tbodies = soup.find_all('table')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)

    return soup
    # return list_innerTable

#数据清洗
def segment(soup,final=True):
    # print("==")
    # print(soup)
    # print("====")
    #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
    subspaceList = ["td",'a',"span","p"]
    if soup.name in subspaceList:
        #判断有值叶子节点数
        _count = 0
        for child in soup.find_all(recursive=True):
            if child.get_text().strip()!="" and len(child.find_all())==0:
                _count += 1
        if _count<=1:
            text = soup.get_text()
            # 2020/11/24 大网站规则添加
            if 'title' in soup.attrs:
                if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']:
                    text = soup.attrs['title']

            _list = []
            for x in re.split("\s+",text):
                if x.strip()!="":
                    _list.append(len(x))
            if len(_list)>0:
                _minLength = min(_list)
                if _minLength>2:
                    _substr = "，"
                else:
                    _substr = ""
            else:
                _substr = ""

            text = text.replace("\r\n","，").replace("\n","，")
            text = re.sub("\s+",_substr,text)
            # text = re.sub("\s+","##space##",text)
            return text
    segList = ["title"]
    commaList = ["div","br","td","p"]
    #commaList = []
    spaceList = ["span"]

    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):

        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            child.insert_after("，")
        # if child.name in subspaceList:
        #     child.insert_before("#subs"+str(child.name)+"#")
        #     child.insert_after("#sube"+str(child.name)+"#")
        # if child.name in spaceList:
        #     child.insert_after(" ")
    text = str(soup.get_text())

    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
    #替换"？"为 " " ,update:2021/7/20
    text = re.sub("？+"," ",text)
         

    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    # print('==1',text)
    # text = re.sub("\s{4,}","，",text)
    # 解决公告中的" "空格替换问题
    if re.search("\s{4,}",text):
        _text = ""
        for _sent in re.split("。+",text):
            for _sent2 in re.split('，+',_sent):
                for _sent3 in re.split("：+",_sent2):
                    for _t in re.split("\s{4,}",_sent3):
                        if len(_t)<3:
                            _text += _t
                        else:
                            _text += "，"+_t
                    _text += "："
                _text = _text[:-1]
                _text += "，"
            _text = _text[:-1]
            _text += "。"
        _text = _text[:-1]
        text = _text
    # print('==2',text)
    #替换标点

    #替换连续的标点

    if final:
        text = re.sub("##space##"," ",text)

    punc_pattern = "(?P<del>[。，；：:,\s]+)"

    list_punc = re.findall(punc_pattern,text)
    list_punc.sort(key=lambda x:len(x),reverse=True)
    for punc_del in list_punc:
        if len(punc_del)>1:
            if len(punc_del.strip())>0:
                if "：" in punc_del.strip():
                    text = re.sub(punc_del,"：",text)
                else:
                    text = re.sub(punc_del,punc_del.strip()[-1],text)
            else:
                text = re.sub(punc_del,"",text)
        

    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)

    # #删除标签中的所有空格
    # for subs in subspaceList:
    #     patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
    #     while(True):
    #         oneMatch = re.search(re.compile(patten),text)
    #         if oneMatch is not None:
    #             _match = oneMatch.group(1)
    #             text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match)
    #         else:
    #             break

    # text过大报错
    LOOP_LEN = 10000
    LOOP_BEGIN = 0
    _text = ""



    if len(text)<10000000:
        while(LOOP_BEGIN<len(text)):
            _text += re.sub("）",")",re.sub("（","(",re.sub("\s+","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
            LOOP_BEGIN += LOOP_LEN
        text = _text

    return text

'''
#数据清洗
def segment(soup):
    segList = ["title"]
    commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]    
    spaceList = ["span"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        if child.name == 'br':
            child.insert_before('，')
        child_text = re.sub('\s', '', child.get_text())
        if child_text == '' or child_text[-1] in ['。','，','：','；']:
            continue
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            if len(child_text)>3 and len(child_text) <50:   # 先判断是否字数少于50，成立加逗号，否则加句号
                child.insert_after("，")           
            elif len(child_text) >=50:
                child.insert_after("。")
        #if child.name in spaceList:
            #child.insert_after(" ")  
    text = str(soup.get_text())
    text = re.sub("\s{5,}","，",text)
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“")    
    #text = text.replace('"',"“").replace("\r","").replace("\n","")
    
    #删除所有空格
    text = re.sub("\s+","#nbsp#",text)
    text_list = text.split('#nbsp#')
    new_text = ''
    for i in range(len(text_list)-1):
        if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']:
            new_text += text_list[i]  
        elif  re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '。' 
        elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '；' 
        elif text_list[i].isdigit() and text_list[i+1].isdigit():
            new_text += text_list[i] + ' '           
        elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']:
            new_text += text_list[i]       
        elif len(text_list[i]) >= 3 and  len(text_list[i+1]) >= 3:
            new_text += text_list[i] + '，'
        else:
            new_text += text_list[i]       
    new_text += text_list[-1]      
    text = new_text
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
            
    #替换标点
    while(True):
        #替换连续的标点
        punc = re.search("，(?P<punc>：|。|，|；)\s*",text)
        if punc is not None:
            text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text)
        
        punc = re.search("(?P<punc>：|。|，|；)\s*，",text)
        if punc is not None:
            text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text)
        else:
            #替换标点之后的空格
            punc = re.search("(?P<punc>：|。|，|；)\s+",text)
            if punc is not None:
                text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
            else:
                break
    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)
    #替换中文括号为英文括号
    text = re.sub("（","(",text)
    text = re.sub("）",")",text)
    return text
'''

#连续实体合并（弃用）
def union_ner(list_ner):
    result_list = []
    union_index = []
    union_index_set = set()
    for i in range(len(list_ner)-1):
        if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
            if list_ner[i][1]-list_ner[i+1][0]==1:
                union_index_set.add(i)
                union_index_set.add(i+1)
                union_index.append((i,i+1))
    for i in range(len(list_ner)):
        if i not in union_index_set:
            result_list.append(list_ner[i])
    for item in union_index:
        #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
        result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
    return result_list
                

# def get_preprocessed(articles,useselffool=False):
#     '''
#     @summary:预处理步骤，NLP处理、实体识别
#     @param:
#         articles:待处理的文章list [[id,source,jointime,doc_id,title]]
#     @return:list of articles,list of each article of sentences,list of each article of entitys
#     '''
#     list_articles = []
#     list_sentences = []
#     list_entitys = []
#     cost_time = dict()
#     for article in articles:
#         list_sentences_temp = []
#         list_entitys_temp = []
#         doc_id = article[0]
#         sourceContent = article[1]
#         _send_doc_id = article[3]
#         _title = article[4]
#         #表格处理
#         key_preprocess = "tableToText"
#         start_time = time.time()
#         article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
#
#         # log(article_processed)
#
#         if key_preprocess not in cost_time:
#             cost_time[key_preprocess] = 0
#         cost_time[key_preprocess] += time.time()-start_time
#
#         #article_processed = article[1]
#         list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
#         #nlp处理
#         if article_processed is not None and len(article_processed)!=0:
#             split_patten = "。"
#             sentences = []
#             _begin = 0
#             for _iter in re.finditer(split_patten,article_processed):
#                 sentences.append(article_processed[_begin:_iter.span()[1]])
#                 _begin = _iter.span()[1]
#             sentences.append(article_processed[_begin:])
#
#             lemmas = []
#             doc_offsets = []
#             dep_types = []
#             dep_tokens = []
#
#             time1 = time.time()
#
#             '''
#             tokens_all = fool.cut(sentences)
#             #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
#             #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
#             ner_entitys_all = fool.ner(sentences)
#             '''
#             #限流执行
#             key_nerToken = "nerToken"
#             start_time = time.time()
#             tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
#             if key_nerToken not in cost_time:
#                 cost_time[key_nerToken] = 0
#             cost_time[key_nerToken] += time.time()-start_time
#
#
#             for sentence_index in range(len(sentences)):
#
#
#
#                 list_sentence_entitys = []
#                 sentence_text = sentences[sentence_index]
#                 tokens = tokens_all[sentence_index]
#
#                 list_tokenbegin = []
#                 begin = 0
#                 for i in range(0,len(tokens)):
#                     list_tokenbegin.append(begin)
#                     begin += len(str(tokens[i]))
#                 list_tokenbegin.append(begin+1)
#                 #pos_tag = pos_all[sentence_index]
#                 pos_tag = ""
#
#                 ner_entitys = ner_entitys_all[sentence_index]
#
#                 list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
#
#                 #识别package
#
#
#                 #识别实体
#                 for ner_entity in ner_entitys:
#                     begin_index_temp = ner_entity[0]
#                     end_index_temp = ner_entity[1]
#                     entity_type = ner_entity[2]
#                     entity_text = ner_entity[3]
#
#                     for j in range(len(list_tokenbegin)):
#                         if list_tokenbegin[j]==begin_index_temp:
#                             begin_index = j
#                             break
#                         elif list_tokenbegin[j]>begin_index_temp:
#                             begin_index = j-1
#                             break
#                     begin_index_temp += len(str(entity_text))
#                     for j in range(begin_index,len(list_tokenbegin)):
#                         if list_tokenbegin[j]>=begin_index_temp:
#                             end_index = j-1
#                             break
#                     entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#                     #去掉标点符号
#                     entity_text = re.sub("[,，。：]","",entity_text)
#                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
#
#
#                 #使用正则识别金额
#                 entity_type = "money"
#
#                 #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"
#
#                 list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
#                                       "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"}
#
#                 set_begin = set()
#                 for pattern_key in list_money_pattern.keys():
#                     pattern = re.compile(list_money_pattern[pattern_key])
#                     all_match = re.findall(pattern, sentence_text)
#                     index = 0
#                     for i in range(len(all_match)):
#                         if len(all_match[i][0])>0:
#                             # print("===",all_match[i])
#                             #print(all_match[i][0])
#                             unit = ""
#                             entity_text = all_match[i][3]
#                             if pattern_key in ["key_word","front_m"]:
#                                 unit = all_match[i][1]
#                             else:
#                                 unit = all_match[i][4]
#                             if entity_text.find("元")>=0:
#                                 unit = ""
#
#                             index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
#
#                             begin_index_temp = index
#                             for j in range(len(list_tokenbegin)):
#                                 if list_tokenbegin[j]==index:
#                                     begin_index = j
#                                     break
#                                 elif list_tokenbegin[j]>index:
#                                     begin_index = j-1
#                                     break
#                             index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
#                             end_index_temp = index
#                             #index += len(str(all_match[i][0]))
#                             for j in range(begin_index,len(list_tokenbegin)):
#                                 if list_tokenbegin[j]>=index:
#                                     end_index = j-1
#                                     break
#                             entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#
#                             entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
#                             if len(unit)>0:
#                                 entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
#                             else:
#                                 entity_text = str(getUnifyMoney(entity_text))
#
#                             _exists = False
#                             for item in list_sentence_entitys:
#                                 if item.entity_id==entity_id and item.entity_type==entity_type:
#                                     _exists = True
#                             if not _exists:
#                                 if float(entity_text)>10:
#                                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
#
#                         else:
#                             index += 1
#
#                 list_sentence_entitys.sort(key=lambda x:x.begin_index)
#                 list_entitys_temp = list_entitys_temp+list_sentence_entitys
#         list_sentences.append(list_sentences_temp)
#         list_entitys.append(list_entitys_temp)
#     return list_articles,list_sentences,list_entitys,cost_time

def get_preprocessed(articles,useselffool=False):
    '''
    @summary:预处理步骤，NLP处理、实体识别
    @param:
        articles:待处理的文章list [[id,source,jointime,doc_id,title]]
    @return:list of articles,list of each article of sentences,list of each article of entitys
    '''
    cost_time = dict()
    list_articles = get_preprocessed_article(articles,cost_time)
    list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
    list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)

    calibrateEnterprise(list_articles,list_sentences,list_entitys)

    return list_articles,list_sentences,list_entitys,cost_time
    

def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
    '''
    :param articles: 待处理的article source html
    :param useselffool: 是否使用selffool
    :return: list_articles
    '''
    list_articles = []
    for article in articles:
        doc_id = article[0]
        sourceContent = article[1]
        sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
        _send_doc_id = article[3]
        _title = article[4]
        page_time = article[5]
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))

        # 提取bidway
        list_bidway = extract_bidway(article_processed, _title)
        if list_bidway:
            bidway = list_bidway[0].get("body")
        else:
            bidway = ""

        # 修正被"，"逗号分隔的时间
        repair_time = re.compile("[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[:时点]，?[0-6]\d分?|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[时点]|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]|"
                                 "[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d"
                                 )
        for _time in set(re.findall(repair_time,article_processed)):
            if re.search("，",_time):
                _time2 = re.sub("，", "", _time)
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time2)
                if item:
                    _time2 = _time2.replace(item.group(),item.group() + " ")
                article_processed = article_processed.replace(_time, _time2)
            else:
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time)
                if item:
                    _time2 = _time.replace(item.group(),item.group() + " ")
                    article_processed = article_processed.replace(_time, _time2)
        # print('re_rtime',re.findall(repair_time,article_processed))
        # log(article_processed)

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += round(time.time()-start_time,2)

        #article_processed = article[1]
        _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
                           bidway=bidway)
        _article.fingerprint = getFingerprint(_title+sourceContent)
        _article.page_time = page_time
        list_articles.append(_article)
    return list_articles

def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
    '''

    :param list_articles: 经过预处理的article text
    :return: list_sentences
    '''
    list_sentences = []
    for article in list_articles:
        list_sentences_temp = []
        list_entitys_temp = []
        doc_id = article.id
        _send_doc_id = article.doc_id
        _title = article.title
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = article.content


        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += time.time()-start_time

        #nlp处理
        if article_processed is not None and len(article_processed)!=0:
            split_patten = "。"
            sentences = []
            _begin = 0
            sentences_set = set()
            for _iter in re.finditer(split_patten,article_processed):
                _sen = article_processed[_begin:_iter.span()[1]]
                if len(_sen)>0 and _sen not in sentences_set:
                    sentences.append(_sen)
                    sentences_set.add(_sen)
                _begin = _iter.span()[1]
            _sen = article_processed[_begin:]
            if len(_sen)>0 and _sen not in sentences_set:
                sentences.append(_sen)
                sentences_set.add(_sen)
            article.content = "".join(sentences)
            # sentences.append(article_processed[_begin:])

            lemmas = []
            doc_offsets = []
            dep_types = []
            dep_tokens = []

            time1 = time.time()

            '''
            tokens_all = fool.cut(sentences)
            #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
            #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
            ner_entitys_all = fool.ner(sentences)
            '''
            #限流执行
            key_nerToken = "nerToken"
            start_time = time.time()
            tokens_all = getTokens(sentences,useselffool=useselffool)
            if key_nerToken not in cost_time:
                cost_time[key_nerToken] = 0
            cost_time[key_nerToken] += time.time()-start_time


            for sentence_index in range(len(sentences)):

                sentence_text = sentences[sentence_index]
                tokens = tokens_all[sentence_index]

                #pos_tag = pos_all[sentence_index]
                pos_tag = ""

                ner_entitys = ""

                list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))

        if len(list_sentences_temp)==0:
            list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
        list_sentences.append(list_sentences_temp)
    return list_sentences

def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
    '''

    :param list_sentences:分局情况
    :param cost_time:
    :return: list_entitys
    '''

    list_entitys = []
    for list_sentence in list_sentences:
        sentences = []
        list_entitys_temp = []
        for _sentence in list_sentence:
            sentences.append(_sentence.sentence_text)

        lemmas = []
        doc_offsets = []
        dep_types = []
        dep_tokens = []

        time1 = time.time()

        '''
        tokens_all = fool.cut(sentences)
        #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
        #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
        ner_entitys_all = fool.ner(sentences)
        '''
        #限流执行
        key_nerToken = "nerToken"
        start_time = time.time()
        found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额
        # found_pingbiao = False
        ner_entitys_all = getNers(sentences,useselffool=useselffool)
        if key_nerToken not in cost_time:
            cost_time[key_nerToken] = 0
        cost_time[key_nerToken] += round(time.time()-start_time,2)


        for sentence_index in range(len(list_sentence)):
            list_sentence_entitys = []

            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            list_tokenbegin = []
            begin = 0
            for i in range(0,len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin+1)
            #pos_tag = pos_all[sentence_index]
            pos_tag = ""

            ner_entitys = ner_entitys_all[sentence_index]


            #识别package

            #识别实体
            for ner_entity in ner_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]

                if entity_type in ["org","company"] and not isLegalEnterprise(entity_text):
                    continue

                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                entity_text = re.sub("[,，。：!&@$\*]","",entity_text)
                entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text
                list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
            # 标记文章末尾的"发布人”、“发布时间”实体
            if sentence_index==len(list_sentence)-1:
                if len(list_sentence_entitys[-2:])>2:
                    second2last = list_sentence_entitys[-2]
                    last = list_sentence_entitys[-1]
                    if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or (
                            second2last.entity_type=="time" and last.entity_type in ["company",'org']):
                        if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6:
                            last.is_tail = True
                            second2last.is_tail = True

            #使用正则识别金额
            entity_type = "money"
            #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"

            # list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
            #                       "key_word":"((?P<text_key_word>(?:[￥¥]+，?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果：)(?:[,（\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[）\)]?))",
            #                       "front_m":"((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)())",
            #                       "behind_m":"(()()(?P<money_behind_m>[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\)）]?)"}
            list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
                                  "key_word": "((?P<text_key_word>(?:[￥¥]+，?|[单报标限总]价|金额|成交报?价|价格|预算|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果：)(?:[,，（\(]*\s*(人民币)?(?P<unit_key_word_before>[万亿]?元?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]{,1})(?:[（\(]?(?P<filter_>[%])*\s*(单位[:：])?(?P<unit_key_word_behind>[万亿]?元?(?P<filter_unit1>[台只吨斤棵株页亩方条米]*))\s*[）\)]?))",
                                  "front_m":"((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万亿]?元)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]*)())",
                                  "behind_m":"(()()(?P<money_behind_m>[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千]*)[\(（]?(?P<unit_behind_m>[万亿]?元(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\)）]?)"}
            # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。

            pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
            set_begin = set()
            # for pattern_key in list_money_pattern.keys():
            # for pattern_key in ["cn","key_word","behind_m","front_m"]:
            #     # pattern = re.compile(list_money_pattern[pattern_key])
            #     pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?\s*([万元]*)\s*[）\)]?))*|(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*|((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*")
            #     all_match = re.findall(pattern, sentence_text)
                # index = 0
                # for i in range(len(all_match)):
                #     if len(all_match[i][0])>0:
                #         print("===",all_match[i])
                #         #print(all_match[i][0])
                #         unit = ""
                #         entity_text = all_match[i][3]
                #         if pattern_key in ["key_word","front_m"]:
                #             unit = all_match[i][1]
                #             if pattern_key=="key_word":
                #                 if all_match[i][1]=="" and all_match[i][4]!="":
                #                     unit = all_match[i][4]
                #         else:
                #             unit = all_match[i][4]
                #         if entity_text.find("元")>=0:
                #             unit = ""
                #
                #         index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
                #         begin_index_temp = index
                #         for j in range(len(list_tokenbegin)):
                #             if list_tokenbegin[j]==index:
                #                 begin_index = j
                #                 break
                #             elif list_tokenbegin[j]>index:
                #                 begin_index = j-1
                #                 break
                #         index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
                #         end_index_temp = index
                #         #index += len(str(all_match[i][0]))
                #         for j in range(begin_index,len(list_tokenbegin)):
                #             if list_tokenbegin[j]>=index:
                #                 end_index = j-1
                #                 break
                #         entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
                #
                #         entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
                #         if len(unit)>0:
                #             entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                #         else:
                #             entity_text = str(getUnifyMoney(entity_text))
                #
                #         _exists = False
                #         for item in list_sentence_entitys:
                #             if item.entity_id==entity_id and item.entity_type==entity_type:
                #                 _exists = True
                #         if not _exists:
                #             if float(entity_text)>1:
                #                 list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
                #
                #     else:
                #         index += 1

            # if re.search('评标结果|候选人公示', sentence_text):
            #     found_pingbiao = True
            if re.search('业绩', sentence_text):
                found_yeji += 1
            if found_yeji >= 2: # 过滤掉业绩后面的所有金额
                all_match = []
            else:
                all_match = re.finditer(pattern_money, sentence_text)
            index = 0
            for _match in all_match:
                if len(_match.group())>0:
                    # print("===",_match.group())
                    # # print(_match.groupdict())
                    notes = ''  # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
                    unit = ""
                    entity_text = ""
                    text_beforeMoney = ""
                    filter = ""
                    filter_unit = False
                    notSure = False
                    if re.search('业绩', sentence_text[:_match.span()[0]]):  # 2021/7/21过滤掉业绩后面金额
                        # print('金额在业绩后面: ', _match.group(0))
                        found_yeji += 1
                        break

                    for k,v in _match.groupdict().items():
                        if v!="" and v is not None:
                            if k=='text_key_word':
                                notSure = True
                            if k.split("_")[0]=="money":
                                entity_text = v
                            if k.split("_")[0]=="unit":
                                unit = v
                            if k.split("_")[0]=="text":
                                text_beforeMoney = v
                            if k.split("_")[0]=="filter":
                                filter = v
                            if re.search("filter_unit",k) is not None:
                                filter_unit = True

                    if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()):  # 2021/7/19 修正OCR识别小数点为逗号
                        if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]):
                            entity_text = re.sub('\d+,', '', entity_text)
                        else:
                            entity_text = entity_text.replace(',', '.')
                        # print(' 修正OCR识别小数点为逗号')

                    if entity_text.find("元")>=0:
                        unit = ""
                    if unit == "":  #2021/7/21 有明显金额特征的补充单位，避免被过滤
                        if ('￥' in text_beforeMoney or '¥' in text_beforeMoney):
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                        elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[:：为]+$', text_beforeMoney.strip()) and \
                                re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None:
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                        elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)',entity_text):
                            unit = '元'
                            # print('明显金额特征补充单位 元')
                    if unit.find("万") >= 0 and entity_text.find("万") >= 0:  #2021/7/19修改为金额文本有万，不计算单位
                        # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit)
                        unit = "元"
                    if re.search('.*万元万元', entity_text):  #2021/7/19 修正两个万元
                        # print(' 修正两个万元',entity_text)
                        entity_text = entity_text.replace('万元万元','万元')
                    else:
                        if filter_unit:
                            continue
                    if filter!="":
                        continue

                    index = _match.span()[0]+len(text_beforeMoney)
                    begin_index_temp = index
                    for j in range(len(list_tokenbegin)):
                        if list_tokenbegin[j]==index:
                            begin_index = j
                            break
                        elif list_tokenbegin[j]>index:
                            begin_index = j-1
                            break
                    index = _match.span()[1]
                    end_index_temp = index
                    #index += len(str(all_match[i][0]))
                    for j in range(begin_index,len(list_tokenbegin)):
                        if list_tokenbegin[j]>=index:
                            end_index = j-1
                            break
                    entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                    entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
                    # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
                    if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/8/5过滤掉总投资金额
                        # print('总投资金额: ', _match.group(0))
                        notes = '总投资'
                    elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/11/18 投资金额不作为招标金额
                        notes = '投资'
                    elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为：]', sentence_text[_match.span()[0]:_match.span()[1]]):
                        cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]])
                        notes = cost_re.group(1)
                    elif re.search('单价', sentence_text[_match.span()[0]:_match.span()[1]]):
                        notes = '单价'
                    elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None:
                        notes = '大写'
                        # print("补充备注：notes = 大写")
                    if len(unit)>0:
                        if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况
                            # print('修正单位万元金额过大的情况 金额：', entity_text, '单位:', unit)
                            entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000)
                            unit = '元' # 修正金额后单位 重置为元
                        else:
                            # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):')
                            entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                    else:
                        if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8:
                            entity_text = str(getUnifyMoney(entity_text)/10000)
                            # print('修正金额字段含万 过大的情况')
                        else:
                            entity_text = str(getUnifyMoney(entity_text))

                    if float(entity_text)<100 or float(entity_text)>100000000000:
                        # print('过滤掉金额：float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit)
                        continue

                    if notSure and unit=="" and float(entity_text)>100*10000:
                        # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit)
                        continue

                    _exists = False
                    for item in list_sentence_entitys:
                        if item.entity_id==entity_id and item.entity_type==entity_type:
                            _exists = True
                        if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index):
                            _exists = True
                    if not _exists:
                        if float(entity_text)>1:
                            list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
                            list_sentence_entitys[-1].notes = notes  # 2021/7/20 新增金额备注
                            list_sentence_entitys[-1].money_unit = unit  # 2021/7/20 新增金额备注
                            # print('预处理中的 金额:%s, 单位：%s'%(entity_text,unit))
                else:
                    index += 1

            # 资金来源提取  2020/12/30 新增
            list_moneySource = extract_moneySource(sentence_text)
            entity_type = "moneysource"
            for moneySource in list_moneySource:
                begin_index_temp = moneySource['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = moneySource['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = moneySource['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp))

            # 服务期限提取 2020/12/30 新增
            list_servicetime = extract_servicetime(sentence_text)
            entity_type = "serviceTime"
            for servicetime in list_servicetime:
                begin_index_temp = servicetime['begin_index']
                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j] == begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j] > begin_index_temp:
                        begin_index = j - 1
                        break
                index = servicetime['end_index']
                end_index_temp = index
                for j in range(begin_index, len(list_tokenbegin)):
                    if list_tokenbegin[j] >= index:
                        end_index = j - 1
                        break
                entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
                entity_text = servicetime['body']
                list_sentence_entitys.append(
                    Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
                           begin_index_temp, end_index_temp))

            # 招标方式提取 2020/12/30 新增
            # list_bidway = extract_bidway(sentence_text, )
            # entity_type = "bidway"
            # for bidway in list_bidway:
            #     begin_index_temp = bidway['begin_index']
            #     end_index_temp = bidway['end_index']
            #     begin_index = changeIndexFromWordToWords(tokens, begin_index_temp)
            #     end_index = changeIndexFromWordToWords(tokens, end_index_temp)
            #     if begin_index is None or end_index is None:
            #         continue
            #     print(begin_index_temp,end_index_temp,begin_index,end_index)
            #     entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index)
            #     entity_text = bidway['body']
            #     list_sentence_entitys.append(
            #         Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index,
            #                begin_index_temp, end_index_temp))

            list_sentence_entitys.sort(key=lambda x:x.begin_index)
            list_entitys_temp = list_entitys_temp+list_sentence_entitys
        list_entitys.append(list_entitys_temp)
    return list_entitys
    
def union_result(codeName,prem):
    '''
    @summary:模型的结果拼成字典
    @param:
        codeName:编号名称模型的结果字典
        prem:拿到属性的角色的字典
    @return:拼接起来的字典
    '''
    result = []
    assert len(codeName)==len(prem)
    for item_code,item_prem in zip(codeName,prem):
        result.append(dict(item_code,**item_prem))
    return result

def persistenceData(data):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for item_index in range(len(data)):
        item = data[item_index]
        doc_id = item[0]
        dic = item[1]
        code = dic['code']
        name = dic['name']
        prem = dic['prem']
        if len(code)==0:
            code_insert = ""
        else:
            code_insert = ";".join(code)
        prem_insert = ""
        for item in prem:
            for x in item:
                if isinstance(x, list):
                    if len(x)>0:
                        for x1 in x:
                            prem_insert+="/".join(x1)+","
                    prem_insert+="$"
                else:
                    prem_insert+=str(x)+"$"
            prem_insert+=";"
        sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
        cursor.execute(sql)
    conn.commit()
    conn.close()
   
def persistenceData1(list_entitys,list_sentences):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for list_entity in list_entitys:
        for entity in list_entity:
            if entity.values is not None:
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
            else:    
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
            cursor.execute(sql)
    for list_sentence in list_sentences:
        for sentence in list_sentence:
            str_tokens = "["
            for item in sentence.tokens:
                str_tokens += "'"
                if item=="'":
                    str_tokens += "''"
                else:
                    str_tokens += item
                str_tokens += "',"
            str_tokens = str_tokens[:-1]+"]"
            sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
            cursor.execute(sql)
    conn.commit()
    conn.close()


def _handle(item,result_queue):
    dochtml = item["dochtml"]
    docid = item["docid"]
    list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
    flag = False
    if list_innerTable:
        flag = True
        for table in list_innerTable:
            result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})

def getPredictTable():
    filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
    import pandas as pd
    import json
    from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
    df = pd.read_csv(filename)
    df_data = {"json_table":[],"docid":[]}
    _count = 0
    _sum = len(df["docid"])
    task_queue = Queue()
    result_queue = Queue()
    _index = 0
    for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
        task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
        _index += 1

    mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
    mh.run()
    while True:
        try:
            item = result_queue.get(block=True,timeout=1)
            df_data["docid"].append(item["docid"])
            df_data["json_table"].append(item["json_table"])
        except Exception as e:
            print(e)
            break
    df_1 = pd.DataFrame(df_data)
    df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])

if __name__=="__main__":
    '''
    import glob
    for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
        file_txt = str(file).replace("html","txt")
        with codecs.open(file_txt,"a+",encoding="utf8") as f:
            f.write("\n================\n")
            content = codecs.open(file,"r",encoding="utf8").read()
            f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
    '''        
    # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
    # getPredictTable()
    with open('D:/138786703.html', 'r', encoding='utf-8') as f:
        sourceContent = f.read()
        # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml")))
        # print(article_processed)

        list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True)
        for entity in list_entitys[0]:
            print(entity.entity_type, entity.entity_text)