BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/Preprocessing.py

from bs4 import BeautifulSoup, Comment
import copy
import sys
import os
import time
import codecs
sys.setrecursionlimit(1000000)
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath(".."))
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Entitys import *
from BiddingKG.dl.interface.predictor import *
from BiddingKG.dl.foolnltk import selffool



#
def tableToText(soup):
    '''
    @param:
        soup:网页html的soup
    @return:处理完表格信息的网页text
    '''
    
    def getTrs(tbody):
    #获取所有的tr
        trs = []
        objs = tbody.find_all(recursive=False)
        for obj in objs:
            if obj.name=="tr":
                trs.append(obj)
            if obj.name=="tbody":
                for tr in obj.find_all("tr",recursive=False):
                    trs.append(tr)
        return trs

    def fixSpan(tbody):
        # 处理colspan, rowspan信息补全问题
        #trs = tbody.findChildren('tr', recursive=False)

        trs = getTrs(tbody)
        ths_len = 0
        ths = list()
        trs_set = set()
        #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱
        # 遍历每一个tr

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有colspan 则补全同一行下一个位置
                if 'colspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['colspan'])))!="":
                        col = int(re.sub("[^0-9]","",str(td['colspan'])))
                        if col<100 and len(td.get_text())<1000:
                            td['colspan'] = 1
                            for i in range(1, col, 1):
                                td.insert_after(copy.copy(td))

        for indtr, tr in enumerate(trs):
            ths_tmp = tr.findChildren('th', recursive=False)
            #不补全含有表格的tr
            if len(tr.findChildren('table'))>0:
                continue
            if len(ths_tmp) > 0:
                ths_len = ths_len + len(ths_tmp)
                for th in ths_tmp:
                    ths.append(th)
                trs_set.add(tr)
            # 遍历每行中的element
            tds = tr.findChildren(recursive=False)
            for indtd, td in enumerate(tds):
                # 若有rowspan 则补全下一行同样位置
                if 'rowspan' in td.attrs:
                    if str(re.sub("[^0-9]","",str(td['rowspan'])))!="":
                        row = int(re.sub("[^0-9]","",str(td['rowspan'])))
                        td['rowspan'] = 1
                        for i in range(1, row, 1):
                            # 获取下一行的所有td， 在对应的位置插入
                            if indtr+i<len(trs):
                                tds1 = trs[indtr + i].findChildren(['td','th'], recursive=False)
                                if len(tds1) >= (indtd) and len(tds1)>0:
                                    if indtd > 0:
                                        tds1[indtd - 1].insert_after(copy.copy(td))
                                    else:
                                        tds1[0].insert_before(copy.copy(td))
    def getTable(tbody):
        #trs = tbody.findChildren('tr', recursive=False)
        trs = getTrs(tbody)
        inner_table = []
        for tr in trs:
            tr_line = []
            tds = tr.findChildren(['td','th'], recursive=False)
            for td in tds:
                tr_line.append([re.sub('\xa0','',segment(td)),0])
                #tr_line.append([td.get_text(),0])
            inner_table.append(tr_line)
        return inner_table                          
    
    #处理表格不对齐的问题
    def fixTable(inner_table,fix_value="~~"):
        maxWidth = 0
        for item in inner_table:
            if len(item)>maxWidth:
                maxWidth = len(item)
        for i in range(len(inner_table)):
            if len(inner_table[i])<maxWidth:
                for j in range(maxWidth-len(inner_table[i])):
                    inner_table[i].append([fix_value,0])
        return inner_table
    
    def removePadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            point = ""
            for j in range(width):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_row
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        for j in range(width):
            point = ""
            for i in range(height):
                if inner_table[i][j][0]==point and point!="":
                    inner_table[i][j][0] = pad_col
                else:
                    if inner_table[i][j][0] not in [pad_row,pad_col]:
                        point = inner_table[i][j][0]
        
    def addPadding(inner_table,pad_row = "@@",pad_col = "##"):
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                if inner_table[i][j][0]==pad_row:
                    inner_table[i][j][0] = inner_table[i][j-1][0]
                    inner_table[i][j][1] = inner_table[i][j-1][1]
                if inner_table[i][j][0]==pad_col:
                    inner_table[i][j][0] = inner_table[i-1][j][0]
                    inner_table[i][j][1] = inner_table[i-1][j][1]
                    
    def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5):
        def repairTable(inner_table,dye_set = set(),key_set = set()):
            '''
            @summary: 修复表头识别，将明显错误的进行修正
            '''
            def repairNeeded(line):
                first_1 = -1
                last_1 = -1
                first_0 = -1
                last_0 = -1
                count_1 = 0
                count_0 = 0
                for i in range(len(line)):
                    if line[i][0]==fix_value:
                        continue
                    if line[i][1]==1:
                        if first_1==-1:
                            first_1 = i
                        last_1 = i
                        count_1 += 1
                    if line[i][1]==0:
                        if first_0 == -1:
                            first_0 = i
                        last_0 = i
                        count_0 += 1
                if first_1 ==-1 or last_0 == -1:
                    return False
                #异常情况：第一个不是表头；最后一个是表头；表头个数远大于属性值个数
                if first_1-0>0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3:
                    return True
                return False
            
            def getsimilarity(line,line1):
                same_count = 0
                for item,item1 in zip(line,line1):
                    if item[1]==item1[1]:
                        same_count += 1
                return same_count/len(line)
            
            def selfrepair(inner_table,index,dye_set,key_set):
                '''
                @summary: 计算每个节点受到的挤压度来判断是否需要染色
                '''
                #print("B",inner_table[index])
                min_presure = 3
                list_dye = []
                first = None
                count = 0
                temp_set = set()
                _index = 0
                for item in inner_table[index]:
                    if first is None:
                        first = item[1]
                        if item[0] not in temp_set:
                            count += 1
                            temp_set.add(item[0])
                    else:
                        if first == item[1]:
                            if item[0] not in temp_set:
                                temp_set.add(item[0])
                                count += 1
                        else:
                            list_dye.append([first,count,_index])
                            first = item[1]
                            temp_set.add(item[0])
                            count = 1
                    _index += 1
                list_dye.append([first,count,_index])
                if len(list_dye)>1:
                    begin = 0
                    end = 0
                    for i in range(len(list_dye)):
                        end = list_dye[i][2]
                        dye_flag = False
                        #首尾要求压力减一
                        if i==0:
                            if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1:
                                dye_flag = True
                                dye_type = list_dye[i+1][0]
                        elif i==len(list_dye)-1:
                            if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1:
                                dye_flag = True
                                dye_type = list_dye[i-1][0]
                        else:
                            if list_dye[i][1]>1:
                                if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure:
                                    dye_flag = True
                                    dye_type = list_dye[i+1][0]
                                if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                    dye_flag = True
                                    dye_type = list_dye[i-1][0]
                            else:
                                if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                    dye_flag = True
                                    dye_type = list_dye[i+1][0]
                                if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure:
                                    dye_flag = True
                                    dye_type = list_dye[i-1][0]
                        if dye_flag:
                            for h in range(begin,end):
                                inner_table[index][h][1] = dye_type
                                dye_set.add((inner_table[index][h][0],dye_type))
                                key_set.add(inner_table[index][h][0])
                        begin = end
                    #print("E",inner_table[index])
                        
                            
                                
                
            def otherrepair(inner_table,index,dye_set,key_set):
                list_provide_repair = []
                if index==0 and len(inner_table)>1:
                    list_provide_repair.append(index+1)
                elif index==len(inner_table)-1:
                    list_provide_repair.append(index-1)
                else:
                    list_provide_repair.append(index+1)
                    list_provide_repair.append(index-1)
                for provide_index in list_provide_repair:
                    if not repairNeeded(inner_table[provide_index]):
                        same_prob = getsimilarity(inner_table[index], inner_table[provide_index])
                        if same_prob>=0.8:
                            for i in range(len(inner_table[provide_index])):
                                if inner_table[index][i][1]!=inner_table[provide_index][i][1]:
                                    dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                    key_set.add(inner_table[index][i][0])
                                inner_table[index][i][1] = inner_table[provide_index][i][1]
                        elif same_prob<=0.2:
                            for i in range(len(inner_table[provide_index])):
                                if inner_table[index][i][1]==inner_table[provide_index][i][1]:
                                    dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1]))
                                    key_set.add(inner_table[index][i][0])
                                inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1
            len_dye_set = len(dye_set)
            height = len(inner_table)
            for i in range(height):
                if repairNeeded(inner_table[i]):
                    selfrepair(inner_table,i,dye_set,key_set)
                    #otherrepair(inner_table,i,dye_set,key_set)
            for h in range(len(inner_table)):
                for w in range(len(inner_table[0])):
                    if inner_table[h][w][0] in key_set:
                        for item in dye_set:
                            if inner_table[h][w][0]==item[0]:
                                inner_table[h][w][1] = item[1]
            #如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代
            if len(dye_set)!=len(key_set):
                for i in range(height):
                    if repairNeeded(inner_table[i]):
                        selfrepair(inner_table,i,dye_set,key_set)
                        #otherrepair(inner_table,i,dye_set,key_set)
                return
            if len(dye_set)==len_dye_set:
                '''
                for i in range(height):
                    if repairNeeded(inner_table[i]):
                        otherrepair(inner_table,i,dye_set,key_set)
                '''
                return
            repairTable(inner_table, dye_set, key_set)
            
                    
                        
                
                    
        set_item = set()
        height = len(inner_table)
        width = len(inner_table[0])
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                set_item.add(item)
        list_item = list(set_item)
        x = []
        for item in list_item:
            x.append(getPredictor("form").encode(item))
        predict_y = getPredictor("form").predict(np.array(x),type="item")
        _dict = dict()
        
        for item,values in zip(list_item,list(predict_y)):
            _dict[item] = values[1]
            # print("##",item,values)
        #print(_dict)
        for i in range(height):
            for j in range(width):
                item = inner_table[i][j][0]
                inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0)
        
        repairTable(inner_table)
        
        #进行分块
        head_list = []
        head_list.append(0)
        last_head = None
        for h in range(height):
            is_all_key = True#是否是全表头行
            is_all_value = True#是否是全属性值
            is_same_with_lastHead = True#和上一行的结构是否相同
            is_same_value=True#一行的item都一样
            #is_same_first_item = True#与上一行的第一项是否相同
            same_value = inner_table[h][0][0]
            for w in range(width):
                if last_head is not None:
                    if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0:
                        is_all_key = False
                        
                    if inner_table[h][w][0]==1:
                        is_all_value = False
                    
                    if inner_table[h][w][1]!= inner_table[h-1][w][1]:
                        is_same_with_lastHead = False
                    
                if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value:
                    is_same_value = False
                else:
                    if re.search("\d+",same_value) is not None:
                        is_same_value = False
            if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]:
                is_same_first_item = False
                
            last_head = h
                
            if is_same_value:
                head_list.append(h)
                head_list.append(h+1)
                continue
            if not is_all_key:
                if not is_same_with_lastHead:
                    head_list.append(h)
                    
                    
        head_list.append(height)
        
        return inner_table,head_list
                    
    #设置表头
    def setHead_inline(inner_table,prob_min=0.64):
        pad_row = "@@"
        pad_col = "##"
        removePadding(inner_table, pad_row, pad_col)
        pad_pattern = re.compile(pad_row+"|"+pad_col)
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            
            is_head = False
            is_long_value = False
            
            #判断是否是全padding值
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row:
                    is_same_value = False
                    break
                
            #predict is head or not with model
            temp_item = ""
            for j in range(width):
                temp_item += inner_table[i][j][0]+"|"
            temp_item = re.sub(pad_pattern,"",temp_item)
            form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
            if form_prob is not None:
                if form_prob[0][1]>prob_min:
                    is_head = True
                else:
                    is_head = False
            
            #print(temp_item,form_prob)
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                #不把连续表头分开
                if not is_head_last:
                    head_list.append(i)
                if is_long_value or is_same_value:
                    head_list.append(i+1)
            if is_head:
                for j in range(width):
                    inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                is_head = False
                
                #predict is head or not with model
                temp_item = ""
                for j in range(head_begin,head_end):
                    temp_item += inner_table[j][i][0]+"|"
                temp_item = re.sub(pad_pattern,"",temp_item)
                form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line")
                if form_prob is not None:
                    if form_prob[0][1]>prob_min:
                        is_head = True
                    else:
                        is_head = False
                
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        addPadding(inner_table, pad_row, pad_col)
        return inner_table,head_list
    
    #设置表头
    def setHead_withRule(inner_table,pattern,pat_value,count):
        height = len(inner_table)
        width = len(inner_table[0])
        head_list = []
        head_list.append(0)
        #行表头
        is_head_last = False
        for i in range(height):
            set_match = set()
            is_head = False
            is_long_value = False
            is_same_value = True
            same_value = inner_table[i][0][0]
            for j in range(width):
                if inner_table[i][j][0]!=same_value:
                    is_same_value = False
                    break
            for j in range(width):
                if re.search(pat_value,inner_table[i][j][0]) is not None:
                    is_head = False
                    break
                str_find = re.findall(pattern,inner_table[i][j][0])
                if len(str_find)>0:
                    set_match.add(inner_table[i][j][0])
                if len(set_match)>=count:
                    is_head = True
            if len(inner_table[i][0][0])>40:
                is_long_value = True
            if is_head or is_long_value or is_same_value:
                if not is_head_last:
                    head_list.append(i)
                if is_head:
                    for j in range(width):
                        inner_table[i][j][1] = 1
            is_head_last = is_head
        head_list.append(height)
        #列表头
        for i in range(len(head_list)-1):
            head_begin = head_list[i]
            head_end = head_list[i+1]
            #最后一列不设置为列表头
            for i in range(width-1):
                set_match = set()
                is_head = False
                for j in range(head_begin,head_end):
                    if re.search(pat_value,inner_table[j][i][0]) is not None:
                        is_head = False
                        break
                    str_find = re.findall(pattern,inner_table[j][i][0])
                    if len(str_find)>0:
                        set_match.add(inner_table[j][i][0])
                    if len(set_match)>=count:
                        is_head = True
                if is_head:
                    for j in range(head_begin,head_end):
                        inner_table[j][i][1] = 2
        return inner_table,head_list
    
    #取得表格的处理方向
    def getDirect(inner_table,begin,end):
        '''
        column_head = set()
        row_head = set()
        widths = len(inner_table[0])
        for height in range(begin,end):
            for width in range(widths):
                if inner_table[height][width][1] ==1:
                    row_head.add(height)
                if inner_table[height][width][1] ==2:
                    column_head.add(width)
        company_pattern = re.compile("公司")
        if 0 in column_head and begin not in row_head:
            return "column"
        if 0 in column_head and begin in row_head:
            for height in range(begin,end):
                count = 0
                count_flag = True
                for width_index in range(width):
                    if inner_table[height][width_index][1]==0:
                        if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                            count += 1
                        else:
                            count_flag = False
                if count_flag and count>=2:
                    return "column"
        return "row"
        '''
        count_row_keys = 0
        count_column_keys = 0
        width = len(inner_table[0])
        if begin<end:
            for w in range(len(inner_table[begin])):
                if inner_table[begin][w][1]!=0:
                    count_row_keys += 1
        for h in range(begin,end):
            if inner_table[h][0][1]!=0:
                count_column_keys += 1
                
        company_pattern = re.compile("有限(责任)?公司")
        for height in range(begin,end):
            count_set = set()
            count_flag = True
            for width_index in range(width):
                if inner_table[height][width_index][1]==0:
                    if re.search(company_pattern,inner_table[height][width_index][0])  is not None:
                        count_set.add(inner_table[height][width_index][0])
                    else:
                        count_flag = False
            if count_flag and len(count_set)>=2:
                return "column"
        if count_column_keys>count_row_keys:
            return "column"
        return "row"
        
            
    #根据表格处理方向生成句子，        
    def getTableText(inner_table,head_list,key_direct=False):
        # packPattern = "(标包|[标包][号段名])"
        packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))"  # 2020/11/23 大网站规则，补充采购类包名
        rankPattern = "(排名|排序|名次|序号|评标结果|评审结果)"  # 2020/11/23 大网站规则，添加序号为排序
        entityPattern = "(候选|([中投]标|报价)(人|单位|候选)|单位名称|供应商)"
        height = len(inner_table)
        width = len(inner_table[0])
        text = ""
         
        for head_i in range(len(head_list)-1):
            
            head_begin = head_list[head_i]
            head_end = head_list[head_i+1]
                
            direct = getDirect(inner_table, head_begin, head_end)
            
            #若只有一行，则直接按行读取
            if head_end-head_begin==1:
                text_line = ""
                for i in range(head_begin,head_end):
                    for w in range(len(inner_table[i])):
                        if inner_table[i][w][1]==1:
                            _punctuation = ":"
                        else:
                            _punctuation = ","
                        if w>0:
                            if inner_table[i][w][0]!= inner_table[i][w-1][0]:
                                text_line += inner_table[i][w][0]+_punctuation
                        else:
                            text_line += inner_table[i][w][0]+_punctuation
                    text_line = text_line+"。" if text_line!="" else text_line
                text += text_line
            else:
        
                if direct=="row":
                    for i in range(head_begin,head_end):
                        pack_text = ""
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        #在同一句话中重复的可以去掉
                        text_set = set()
                        for j in range(width):
                            cell = inner_table[i][j]
                            #是属性值
                            if cell[1]==0 and cell[0]!="":
                                head = ""
                                
                                find_flag = False
                                temp_head = ""
                                for loop_i in range(0,i+1-head_begin):
                                    if not key_direct:
                                        key_values = [1,2]
                                    else:
                                        key_values = [1]
                                    if inner_table[i-loop_i][j][1] in key_values:
                                        if find_flag:
                                            if inner_table[i-loop_i][j][0]!=temp_head:
                                                head = inner_table[i-loop_i][j][0]+":"+head
                                        else:
                                            head = inner_table[i-loop_i][j][0]+":"+head
                                        find_flag = True
                                        temp_head = inner_table[i-loop_i][j][0]
                                    else:
                                        #找到表头后遇到属性值就返回
                                        if find_flag:
                                            break
                                
                                find_flag = False
                                temp_head = ""
                                
                                
                                
                                for loop_j in range(1,j+1):
                                    if not key_direct:
                                        key_values = [1,2]
                                    else:
                                        key_values = [2]
                                    if inner_table[i][j-loop_j][1] in key_values:
                                        if find_flag:
                                            if inner_table[i][j-loop_j][0]!=temp_head:
                                                head = inner_table[i][j-loop_j][0]+":"+head
                                        else:
                                            head = inner_table[i][j-loop_j][0]+":"+head
                                        find_flag = True
                                        temp_head = inner_table[i][j-loop_j][0]
                                    else:
                                        if find_flag:
                                            break
                                
                                if str(head+inner_table[i][j][0]) in text_set:
                                    continue
                                if re.search(packPattern,head) is not None:
                                    pack_text += head+inner_table[i][j][0]+"，"
                                elif re.search(rankPattern,head) is not None:   # 2020/11/23 大网站规则发现问题，if 改elif
                                    #排名替换为同一种表达
                                    rank_text += head+inner_table[i][j][0]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None:
                                    entity_text += head+inner_table[i][j][0]+"，"
                                    #print(entity_text)
                                else:
                                    text_line += head+inner_table[i][j][0]+"，"
                                text_set.add(str(head+inner_table[i][j][0]))
                        text += pack_text+rank_text+entity_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text
                else:
                    for j in range(width):
                    
                        rank_text = ""
                        entity_text = ""
                        text_line = ""
                        text_set = set()
                        for i in range(head_begin,head_end):
                            cell = inner_table[i][j]
                            #是属性值
                            if cell[1]==0 and cell[0]!="":
                                find_flag = False
                                head = ""
                                temp_head = ""
                                
                                for loop_j in range(1,j+1):
                                    if not key_direct:
                                        key_values = [1,2]
                                    else:
                                        key_values = [2]
                                    if inner_table[i][j-loop_j][1] in key_values:
                                        if find_flag:
                                            if inner_table[i][j-loop_j][0]!=temp_head:
                                                head = inner_table[i][j-loop_j][0]+":"+head
                                        else:
                                            head = inner_table[i][j-loop_j][0]+":"+head
                                        find_flag = True
                                        temp_head = inner_table[i][j-loop_j][0]
                                    else:
                                        if find_flag:
                                            break
                                find_flag = False
                                temp_head = ""
                                for loop_i in range(0,i+1-head_begin):
                                    if not key_direct:
                                        key_values = [1,2]
                                    else:
                                        key_values = [1]
                                    if inner_table[i-loop_i][j][1] in key_values:
                                        if find_flag:
                                            if inner_table[i-loop_i][j][0]!=temp_head:
                                                head = inner_table[i-loop_i][j][0]+":"+head
                                        else:
                                            head = inner_table[i-loop_i][j][0]+":"+head
                                        find_flag = True
                                        temp_head = inner_table[i-loop_i][j][0]
                                    else:
                                        if find_flag:
                                            break
                                if str(head+inner_table[i][j][0]) in text_set:
                                    continue
                                if re.search(rankPattern,head) is not None:
                                    rank_text += head+inner_table[i][j][0]+"，"
                                    #print(rank_text)
                                elif re.search(entityPattern,head) is not None:
                                    entity_text += head+inner_table[i][j][0]+"，"
                                    #print(entity_text)
                                else:
                                    text_line += head+inner_table[i][j][0]+"，"
                                text_set.add(str(head+inner_table[i][j][0]))
                        text += rank_text+entity_text+text_line
                        text = text[:-1]+"。" if len(text)>0 else text
        return text
    
    def removeFix(inner_table,fix_value="~~"):
        height = len(inner_table)
        width = len(inner_table[0])
        for h in range(height):
            for w in range(width):
                if inner_table[h][w][0]==fix_value:
                    inner_table[h][w][0] = ""
    
    def trunTable(tbody):
        fixSpan(tbody)
        inner_table = getTable(tbody)
        inner_table = fixTable(inner_table)
        if len(inner_table)>0 and len(inner_table[0])>0:
            #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
            #inner_table,head_list = setHead_inline(inner_table)
            inner_table,head_list = setHead_initem(inner_table,pat_head)
            # print(inner_table)
            # for begin in range(len(head_list[:-1])):
            #     for item in inner_table[head_list[begin]:head_list[begin+1]]:
            #         print(item)
            #     print("====")

            removeFix(inner_table)
            
            '''
            print("----")
            print(head_list)
            for item in inner_table:
                print(item)
            '''

            tbody.string = getTableText(inner_table,head_list)
            #print(tbody.string)
            tbody.name = "turntable"
            return inner_table
        return None


        
    pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$')
    #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)')
    pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)")

    list_innerTable = []
    tbodies = soup.find_all('table')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)
    tbodies = soup.find_all('tbody')
    # 遍历表格中的每个tbody
    #逆序处理嵌套表格
    for tbody_index in range(1,len(tbodies)+1):
        tbody = tbodies[len(tbodies)-tbody_index]
        inner_table = trunTable(tbody)
        list_innerTable.append(inner_table)
    return soup
    # return list_innerTable

#数据清洗
def segment(soup):
    # print("==")
    # print(soup)
    # print("====")
    #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"]
    if soup.name=="td":
        #判断有值叶子节点数
        _count = 0
        for child in soup.find_all(recursive=True):
            if child.get_text().strip()!="" and len(child.find_all())==0:
                _count += 1
        if _count<=1:
            text = soup.get_text()
            # 2020/11/24 大网站规则添加
            if 'title' in soup.attrs:
                if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']:
                    text = soup.attrs['title']

            _list = []
            for x in re.split("\s+",text):
                if x.strip()!="":
                    _list.append(len(x))
            if len(_list)>0:
                _minLength = min(_list)
                if _minLength>2:
                    _substr = "，"
                else:
                    _substr = ""
            else:
                _substr = ""
            text = _substr.join(re.split("(\s+)",text))
            text = text.replace("\r\n","，").replace("\n","，")
            text = re.sub("^[，\s]*|[，\s]*$","",text)
            return text
    segList = ["title"]
    commaList = ["div","br","td","p"]
    #commaList = []
    spaceList = ["span"]
    subspaceList = ["td",'a',"span","p"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):

        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            child.insert_after("，")
        # if child.name in subspaceList:
        #     child.insert_before("#subs"+str(child.name)+"#")
        #     child.insert_after("#sube"+str(child.name)+"#")
        if child.name in spaceList:
            child.insert_after(" ")
    text = str(soup.get_text())

    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
    
         
    #删除标签中的所有空格
    for subs in subspaceList:
        patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#"
        while(True):
            oneMatch = re.search(re.compile(patten),text)
            if oneMatch is not None:
                _match = oneMatch.group(1)
                _minLength = min([len(x) for x in re.split("(\s*)",_match)])
                if _minLength>2:
                    _substr = "，"
                else:
                    _substr = ""
                text = text.replace("#subs"+str(subs)+"#"+oneMatch.group(1)+"#sube"+str(subs)+"#",re.sub("\s",_substr,oneMatch.group(1)))
            else:
                break
    
    
    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    text = re.sub("\s{4,}","，",text)   
    #替换标点
    while(True):
        #替换连续的标点
        punc = re.search("，(?P<punc>：|。|，|；)\s*",text)
        if punc is not None:
            text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text)
        
        punc = re.search("(?P<punc>：|。|，|；)\s*，",text)
        if punc is not None:
            text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text)
        else:
            #替换标点之后的空格
            punc = re.search("(?P<punc>：|。|，|；)\s+",text)
            if punc is not None:
                text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
            else:
                break
    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    list_text = []
    # for _t in text_split:
    #     list_text.append(re.sub("）",")",re.sub("（","(",re.sub("\s*","",_t))))
    text = "。".join(text_split)
    # text = text.replace('）',")").replace("（","(").replace("\s","")
    #删除所有空格
    # text过大报错
    LOOP_LEN = 10000
    LOOP_BEGIN = 0
    _text = ""
    if len(text)<10000000:
        while(LOOP_BEGIN<len(text)):
            _text += re.sub("）",")",re.sub("（","(",re.sub("\s*","",text[LOOP_BEGIN:LOOP_BEGIN+LOOP_LEN])))
            LOOP_BEGIN += LOOP_LEN
    else:
        return text
    # text = re.sub("\s*","",text)
    # #替换中文括号为英文括号
    # text = re.sub("（","(",text)
    # text = re.sub("）",")",text)
    return _text

'''
#数据清洗
def segment(soup):
    segList = ["title"]
    commaList = ["p","div","h1", "h2", "h3", "h4", "h5", "h6", "header", "dl", "ul", "label"]    
    spaceList = ["span"]
    tbodies = soup.find_all('tbody')
    if len(tbodies) == 0:
        tbodies = soup.find_all('table')
    # 递归遍历所有节点,插入符号
    for child in soup.find_all(recursive=True):
        if child.name == 'br':
            child.insert_before('，')
        child_text = re.sub('\s', '', child.get_text())
        if child_text == '' or child_text[-1] in ['。','，','：','；']:
            continue
        if child.name in segList:
            child.insert_after("。")
        if child.name in commaList:
            if len(child_text)>3 and len(child_text) <50:   # 先判断是否字数少于50，成立加逗号，否则加句号
                child.insert_after("，")           
            elif len(child_text) >=50:
                child.insert_after("。")
        #if child.name in spaceList:
            #child.insert_after(" ")  
    text = str(soup.get_text())
    text = re.sub("\s{5,}","，",text)
    text = text.replace('"',"“").replace("\r","").replace("\n","，")
    #替换"""为"“",否则导入deepdive出错
    text = text.replace('"',"“")    
    #text = text.replace('"',"“").replace("\r","").replace("\n","")
    
    #删除所有空格
    text = re.sub("\s+","#nbsp#",text)
    text_list = text.split('#nbsp#')
    new_text = ''
    for i in range(len(text_list)-1):
        if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']:
            new_text += text_list[i]  
        elif  re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '。' 
        elif re.findall('([0-9]、)', text_list[i+1][:4]) != []:
            new_text += text_list[i] + '；' 
        elif text_list[i].isdigit() and text_list[i+1].isdigit():
            new_text += text_list[i] + ' '           
        elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']:
            new_text += text_list[i]       
        elif len(text_list[i]) >= 3 and  len(text_list[i+1]) >= 3:
            new_text += text_list[i] + '，'
        else:
            new_text += text_list[i]       
    new_text += text_list[-1]      
    text = new_text
    #替换英文冒号为中文冒号
    text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text)
    #替换为中文逗号
    text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text)
    #替换为中文分号
    text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text)
            
    #替换标点
    while(True):
        #替换连续的标点
        punc = re.search("，(?P<punc>：|。|，|；)\s*",text)
        if punc is not None:
            text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text)
        
        punc = re.search("(?P<punc>：|。|，|；)\s*，",text)
        if punc is not None:
            text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text)
        else:
            #替换标点之后的空格
            punc = re.search("(?P<punc>：|。|，|；)\s+",text)
            if punc is not None:
                text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text)
            else:
                break
    #将连续的中文句号替换为一个
    text_split = text.split("。")
    text_split = [x for x in text_split if len(x)>0]
    text = "。".join(text_split)
    #替换中文括号为英文括号
    text = re.sub("（","(",text)
    text = re.sub("）",")",text)
    return text
'''

#连续实体合并（弃用）
def union_ner(list_ner):
    result_list = []
    union_index = []
    union_index_set = set()
    for i in range(len(list_ner)-1):
        if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2:
            if list_ner[i][1]-list_ner[i+1][0]==1:
                union_index_set.add(i)
                union_index_set.add(i+1)
                union_index.append((i,i+1))
    for i in range(len(list_ner)):
        if i not in union_index_set:
            result_list.append(list_ner[i])
    for item in union_index:
        #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))
        result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3])))
    return result_list
                
                
def getTokensAndNers(sentences,MAXAREA = 10000,useselffool=False):
    '''
    @param: sentences:句子数
    @return 限流执行后的分词和实体识别list
    '''
    def getData(tokens,ners,process_data):
        process_sentences = [item[1] for item in process_data]
        
        token_ = selffool.cut(process_sentences)
        if useselffool:
            ner_ = selffool.self_ner(process_sentences)
        else:
            ner_ = selffool.ner(process_sentences)
        for i in range(len(token_)):
            the_index = process_data[i][0]
            tokens[the_index] = token_[i]
            ners[the_index] = ner_[i]
    sents = []
    for i in range(len(sentences)):
        sents.append([i,sentences[i]])
    sents.sort(key=lambda x:len(x[1]),reverse=True)
    index_ = 0
    tokens = [[]for i in range(len(sentences))]
    ners = [[]for i in range(len(sentences))]
    
    while(True):
        width = len(sents[index_][1])
        height = MAXAREA//width+1
        if height>len(sents)-index_:
            height = len(sents)-index_
        process_data = sents[index_:index_+height]
        getData(tokens, ners, process_data)
        index_ += height
        if index_>=len(sents):
            break
    return tokens,ners

def getTokens(sentences,MAXAREA = 10000,useselffool=True):
    '''
     @param: sentences:句子数
     @return 限流执行后的分词list
     '''
    def getData(tokens,process_data):
        process_sentences = [item[1] for item in process_data]

        token_ = selffool.cut(process_sentences)
        for i in range(len(token_)):
            the_index = process_data[i][0]
            tokens[the_index] = token_[i]
    sents = []
    for i in range(len(sentences)):
        sents.append([i,sentences[i]])
    sents.sort(key=lambda x:len(x[1]),reverse=True)
    index_ = 0
    tokens = [[]for i in range(len(sentences))]

    while(True):
        width = len(sents[index_][1])
        height = MAXAREA//width+1
        if height>len(sents)-index_:
            height = len(sents)-index_
        process_data = sents[index_:index_+height]
        getData(tokens, process_data)
        index_ += height
        if index_>=len(sents):
            break
    return tokens

def getNers(sentences,MAXAREA = 10000,useselffool=False):
    '''
    @param: sentences:句子数
    @return 限流执行后的实体识别list
    '''
    def getData(ners,process_data):
        process_sentences = [item[1] for item in process_data]

        if useselffool:
            ner_ = selffool.self_ner(process_sentences)
        else:
            ner_ = selffool.ner(process_sentences)
        for i in range(len(ner_)):
            the_index = process_data[i][0]
            ners[the_index] = ner_[i]
    sents = []
    for i in range(len(sentences)):
        sents.append([i,sentences[i]])
    sents.sort(key=lambda x:len(x[1]),reverse=True)
    index_ = 0
    ners = [[]for i in range(len(sentences))]

    while(True):
        width = len(sents[index_][1])
        height = MAXAREA//width+1
        if height>len(sents)-index_:
            height = len(sents)-index_
        process_data = sents[index_:index_+height]
        getData( ners, process_data)
        index_ += height
        if index_>=len(sents):
            break
    return ners
    

# def get_preprocessed(articles,useselffool=False):
#     '''
#     @summary:预处理步骤，NLP处理、实体识别
#     @param:
#         articles:待处理的文章list [[id,source,jointime,doc_id,title]]
#     @return:list of articles,list of each article of sentences,list of each article of entitys
#     '''
#     list_articles = []
#     list_sentences = []
#     list_entitys = []
#     cost_time = dict()
#     for article in articles:
#         list_sentences_temp = []
#         list_entitys_temp = []
#         doc_id = article[0]
#         sourceContent = article[1]
#         _send_doc_id = article[3]
#         _title = article[4]
#         #表格处理
#         key_preprocess = "tableToText"
#         start_time = time.time()
#         article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
#
#         # log(article_processed)
#
#         if key_preprocess not in cost_time:
#             cost_time[key_preprocess] = 0
#         cost_time[key_preprocess] += time.time()-start_time
#
#         #article_processed = article[1]
#         list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
#         #nlp处理
#         if article_processed is not None and len(article_processed)!=0:
#             split_patten = "。"
#             sentences = []
#             _begin = 0
#             for _iter in re.finditer(split_patten,article_processed):
#                 sentences.append(article_processed[_begin:_iter.span()[1]])
#                 _begin = _iter.span()[1]
#             sentences.append(article_processed[_begin:])
#
#             lemmas = []
#             doc_offsets = []
#             dep_types = []
#             dep_tokens = []
#
#             time1 = time.time()
#
#             '''
#             tokens_all = fool.cut(sentences)
#             #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
#             #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
#             ner_entitys_all = fool.ner(sentences)
#             '''
#             #限流执行
#             key_nerToken = "nerToken"
#             start_time = time.time()
#             tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool)
#             if key_nerToken not in cost_time:
#                 cost_time[key_nerToken] = 0
#             cost_time[key_nerToken] += time.time()-start_time
#
#
#             for sentence_index in range(len(sentences)):
#
#
#
#                 list_sentence_entitys = []
#                 sentence_text = sentences[sentence_index]
#                 tokens = tokens_all[sentence_index]
#
#                 list_tokenbegin = []
#                 begin = 0
#                 for i in range(0,len(tokens)):
#                     list_tokenbegin.append(begin)
#                     begin += len(str(tokens[i]))
#                 list_tokenbegin.append(begin+1)
#                 #pos_tag = pos_all[sentence_index]
#                 pos_tag = ""
#
#                 ner_entitys = ner_entitys_all[sentence_index]
#
#                 list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))
#
#                 #识别package
#
#
#                 #识别实体
#                 for ner_entity in ner_entitys:
#                     begin_index_temp = ner_entity[0]
#                     end_index_temp = ner_entity[1]
#                     entity_type = ner_entity[2]
#                     entity_text = ner_entity[3]
#
#                     for j in range(len(list_tokenbegin)):
#                         if list_tokenbegin[j]==begin_index_temp:
#                             begin_index = j
#                             break
#                         elif list_tokenbegin[j]>begin_index_temp:
#                             begin_index = j-1
#                             break
#                     begin_index_temp += len(str(entity_text))
#                     for j in range(begin_index,len(list_tokenbegin)):
#                         if list_tokenbegin[j]>=begin_index_temp:
#                             end_index = j-1
#                             break
#                     entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#                     #去掉标点符号
#                     entity_text = re.sub("[,，。：]","",entity_text)
#                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))
#
#
#                 #使用正则识别金额
#                 entity_type = "money"
#
#                 #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"
#
#                 list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*",
#                                       "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*",
#                                       "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"}
#
#                 set_begin = set()
#                 for pattern_key in list_money_pattern.keys():
#                     pattern = re.compile(list_money_pattern[pattern_key])
#                     all_match = re.findall(pattern, sentence_text)
#                     index = 0
#                     for i in range(len(all_match)):
#                         if len(all_match[i][0])>0:
#                             # print("===",all_match[i])
#                             #print(all_match[i][0])
#                             unit = ""
#                             entity_text = all_match[i][3]
#                             if pattern_key in ["key_word","front_m"]:
#                                 unit = all_match[i][1]
#                             else:
#                                 unit = all_match[i][4]
#                             if entity_text.find("元")>=0:
#                                 unit = ""
#
#                             index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
#
#                             begin_index_temp = index
#                             for j in range(len(list_tokenbegin)):
#                                 if list_tokenbegin[j]==index:
#                                     begin_index = j
#                                     break
#                                 elif list_tokenbegin[j]>index:
#                                     begin_index = j-1
#                                     break
#                             index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
#                             end_index_temp = index
#                             #index += len(str(all_match[i][0]))
#                             for j in range(begin_index,len(list_tokenbegin)):
#                                 if list_tokenbegin[j]>=index:
#                                     end_index = j-1
#                                     break
#                             entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
#
#
#                             entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
#                             if len(unit)>0:
#                                 entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
#                             else:
#                                 entity_text = str(getUnifyMoney(entity_text))
#
#                             _exists = False
#                             for item in list_sentence_entitys:
#                                 if item.entity_id==entity_id and item.entity_type==entity_type:
#                                     _exists = True
#                             if not _exists:
#                                 if float(entity_text)>10:
#                                     list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
#
#                         else:
#                             index += 1
#
#                 list_sentence_entitys.sort(key=lambda x:x.begin_index)
#                 list_entitys_temp = list_entitys_temp+list_sentence_entitys
#         list_sentences.append(list_sentences_temp)
#         list_entitys.append(list_entitys_temp)
#     return list_articles,list_sentences,list_entitys,cost_time

def get_preprocessed(articles,useselffool=False):
    '''
    @summary:预处理步骤，NLP处理、实体识别
    @param:
        articles:待处理的文章list [[id,source,jointime,doc_id,title]]
    @return:list of articles,list of each article of sentences,list of each article of entitys
    '''
    cost_time = dict()
    list_articles = get_preprocessed_article(articles,cost_time)
    list_sentences = get_preprocessed_sentences(list_articles,True,cost_time)
    list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)



    return list_articles,list_sentences,list_entitys,cost_time
    

def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
    '''
    :param articles: 待处理的article source html
    :param useselffool: 是否使用selffool
    :return: list_articles
    '''
    list_articles = []
    for article in articles:
        doc_id = article[0]
        sourceContent = article[1]
        _send_doc_id = article[3]
        _title = article[4]
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))

        # log(article_processed)

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += time.time()-start_time

        #article_processed = article[1]
        list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title))
    return list_articles

def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()):
    '''

    :param list_articles: 经过预处理的article text
    :return: list_sentences
    '''
    list_sentences = []
    for article in list_articles:
        list_sentences_temp = []
        list_entitys_temp = []
        doc_id = article.id
        _send_doc_id = article.doc_id
        _title = article.title
        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        article_processed = article.content


        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += time.time()-start_time

        #nlp处理
        if article_processed is not None and len(article_processed)!=0:
            split_patten = "。"
            sentences = []
            _begin = 0
            sentences_set = set()
            for _iter in re.finditer(split_patten,article_processed):
                _sen = article_processed[_begin:_iter.span()[1]]
                if len(_sen)>0 and _sen not in sentences_set:
                    sentences.append(_sen)
                    sentences_set.add(_sen)
                _begin = _iter.span()[1]
            _sen = article_processed[_begin:]
            if len(_sen)>0 and _sen not in sentences_set:
                sentences.append(_sen)
                sentences_set.add(_sen)
            article.content = "".join(sentences)
            # sentences.append(article_processed[_begin:])

            lemmas = []
            doc_offsets = []
            dep_types = []
            dep_tokens = []

            time1 = time.time()

            '''
            tokens_all = fool.cut(sentences)
            #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
            #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
            ner_entitys_all = fool.ner(sentences)
            '''
            #限流执行
            key_nerToken = "nerToken"
            start_time = time.time()
            tokens_all = getTokens(sentences,useselffool=useselffool)
            if key_nerToken not in cost_time:
                cost_time[key_nerToken] = 0
            cost_time[key_nerToken] += time.time()-start_time


            for sentence_index in range(len(sentences)):

                sentence_text = sentences[sentence_index]
                tokens = tokens_all[sentence_index]

                #pos_tag = pos_all[sentence_index]
                pos_tag = ""

                ner_entitys = ""

                list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys))

        if len(list_sentences_temp)==0:
            list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags=""))
        list_sentences.append(list_sentences_temp)
    return list_sentences

def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()):
    '''

    :param list_sentences:分局情况
    :param cost_time:
    :return: list_entitys
    '''

    list_entitys = []
    for list_sentence in list_sentences:
        sentences = []
        list_entitys_temp = []
        for _sentence in list_sentence:
            sentences.append(_sentence.sentence_text)

        lemmas = []
        doc_offsets = []
        dep_types = []
        dep_tokens = []

        time1 = time.time()

        '''
        tokens_all = fool.cut(sentences)
        #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
        #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
        ner_entitys_all = fool.ner(sentences)
        '''
        #限流执行
        key_nerToken = "nerToken"
        start_time = time.time()
        ner_entitys_all = getNers(sentences,useselffool=useselffool)
        if key_nerToken not in cost_time:
            cost_time[key_nerToken] = 0
        cost_time[key_nerToken] += time.time()-start_time


        for sentence_index in range(len(list_sentence)):
            list_sentence_entitys = []

            sentence_text = list_sentence[sentence_index].sentence_text
            tokens = list_sentence[sentence_index].tokens
            doc_id = list_sentence[sentence_index].doc_id
            list_tokenbegin = []
            begin = 0
            for i in range(0,len(tokens)):
                list_tokenbegin.append(begin)
                begin += len(str(tokens[i]))
            list_tokenbegin.append(begin+1)
            #pos_tag = pos_all[sentence_index]
            pos_tag = ""

            ner_entitys = ner_entitys_all[sentence_index]


            #识别package

            #识别实体
            for ner_entity in ner_entitys:
                begin_index_temp = ner_entity[0]
                end_index_temp = ner_entity[1]
                entity_type = ner_entity[2]
                entity_text = ner_entity[3]


                for j in range(len(list_tokenbegin)):
                    if list_tokenbegin[j]==begin_index_temp:
                        begin_index = j
                        break
                    elif list_tokenbegin[j]>begin_index_temp:
                        begin_index = j-1
                        break
                begin_index_temp += len(str(entity_text))
                for j in range(begin_index,len(list_tokenbegin)):
                    if list_tokenbegin[j]>=begin_index_temp:
                        end_index = j-1
                        break
                entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                #去掉标点符号
                entity_text = re.sub("[,，。：]","",entity_text)
                list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1))


            #使用正则识别金额
            entity_type = "money"
            #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*"

            list_money_pattern = {"cn":"(()()(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
                                  "key_word":"((?P<text_key_word>(?:[￥¥]+，?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果：)(?:[,（\(]*\s*(?P<unit_key_word_before>[万元]*(?P<filter_unit2>[台个只]*))\s*[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P<money_key_word>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?(?P<filter_>[%])*\s*(?P<unit_key_word_behind>[万元]*(?P<filter_unit1>[台个只]*))\s*[）\)]?))",
                                  "front_m":"((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万元]+)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P<money_front_m>[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)())",
                                  "behind_m":"(()()(?P<money_behind_m>[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?(?P<unit_behind_m>[万元]+(?P<filter_unit3>[台个只]*))[\)）]?)"}
            pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"]))
            set_begin = set()
            # for pattern_key in list_money_pattern.keys():
            # for pattern_key in ["cn","key_word","behind_m","front_m"]:
            #     # pattern = re.compile(list_money_pattern[pattern_key])
            #     pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?\s*([万元]*)\s*[）\)]?))*|(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*|((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*")
            #     all_match = re.findall(pattern, sentence_text)
                # index = 0
                # for i in range(len(all_match)):
                #     if len(all_match[i][0])>0:
                #         print("===",all_match[i])
                #         #print(all_match[i][0])
                #         unit = ""
                #         entity_text = all_match[i][3]
                #         if pattern_key in ["key_word","front_m"]:
                #             unit = all_match[i][1]
                #             if pattern_key=="key_word":
                #                 if all_match[i][1]=="" and all_match[i][4]!="":
                #                     unit = all_match[i][4]
                #         else:
                #             unit = all_match[i][4]
                #         if entity_text.find("元")>=0:
                #             unit = ""
                #
                #         index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征
                #         begin_index_temp = index
                #         for j in range(len(list_tokenbegin)):
                #             if list_tokenbegin[j]==index:
                #                 begin_index = j
                #                 break
                #             elif list_tokenbegin[j]>index:
                #                 begin_index = j-1
                #                 break
                #         index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体
                #         end_index_temp = index
                #         #index += len(str(all_match[i][0]))
                #         for j in range(begin_index,len(list_tokenbegin)):
                #             if list_tokenbegin[j]>=index:
                #                 end_index = j-1
                #                 break
                #         entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)
                #
                #         entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text)
                #         if len(unit)>0:
                #             entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                #         else:
                #             entity_text = str(getUnifyMoney(entity_text))
                #
                #         _exists = False
                #         for item in list_sentence_entitys:
                #             if item.entity_id==entity_id and item.entity_type==entity_type:
                #                 _exists = True
                #         if not _exists:
                #             if float(entity_text)>1:
                #                 list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))
                #
                #     else:
                #         index += 1
            all_match = re.finditer(pattern_money, sentence_text)
            index = 0
            for _match in all_match:
                if len(_match.group())>0:
                    # print("===",_match.group())
                    # print(_match.groupdict())
                    unit = ""
                    entity_text = ""
                    text_beforeMoney = ""
                    filter = ""
                    filter_unit = False
                    for k,v in _match.groupdict().items():
                        if v!="" and v is not None:
                            if k.split("_")[0]=="money":
                                entity_text = v
                            if k.split("_")[0]=="unit":
                                unit = v
                            if k.split("_")[0]=="text":
                                text_beforeMoney = v
                            if k.split("_")[0]=="filter":
                                filter = v
                            if re.search("filter_unit",k) is not None:
                                filter_unit = True
                    if entity_text.find("元")>=0:
                        unit = ""
                    else:
                        if filter_unit:
                            continue
                    if filter!="":
                        continue

                    index = _match.span()[0]+len(text_beforeMoney)
                    begin_index_temp = index
                    for j in range(len(list_tokenbegin)):
                        if list_tokenbegin[j]==index:
                            begin_index = j
                            break
                        elif list_tokenbegin[j]>index:
                            begin_index = j-1
                            break
                    index = _match.span()[1]
                    end_index_temp = index
                    #index += len(str(all_match[i][0]))
                    for j in range(begin_index,len(list_tokenbegin)):
                        if list_tokenbegin[j]>=index:
                            end_index = j-1
                            break
                    entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index)

                    entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text)
                    if len(unit)>0:
                        entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0]))
                    else:
                        entity_text = str(getUnifyMoney(entity_text))

                    if float(entity_text)<100 or float(entity_text)>100000000000:
                        continue

                    _exists = False
                    for item in list_sentence_entitys:
                        if item.entity_id==entity_id and item.entity_type==entity_type:
                            _exists = True
                    if not _exists:
                        if float(entity_text)>1:
                            list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp))

                else:
                    index += 1


            list_sentence_entitys.sort(key=lambda x:x.begin_index)
            list_entitys_temp = list_entitys_temp+list_sentence_entitys
        list_entitys.append(list_entitys_temp)
    return list_entitys
    
def union_result(codeName,prem):
    '''
    @summary:模型的结果拼成字典
    @param:
        codeName:编号名称模型的结果字典
        prem:拿到属性的角色的字典
    @return:拼接起来的字典
    '''
    result = []
    assert len(codeName)==len(prem)
    for item_code,item_prem in zip(codeName,prem):
        if item_code[0]==item_prem[0]:
            result.append([item_code[0],dict(item_code[1],**item_prem[1])])
    return result

def persistenceData(data):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for item_index in range(len(data)):
        item = data[item_index]
        doc_id = item[0]
        dic = item[1]
        code = dic['code']
        name = dic['name']
        prem = dic['prem']
        if len(code)==0:
            code_insert = ""
        else:
            code_insert = ";".join(code)
        prem_insert = ""
        for item in prem:
            for x in item:
                if isinstance(x, list):
                    if len(x)>0:
                        for x1 in x:
                            prem_insert+="/".join(x1)+","
                    prem_insert+="$"
                else:
                    prem_insert+=str(x)+"$"
            prem_insert+=";"
        sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')"
        cursor.execute(sql)
    conn.commit()
    conn.close()
   
def persistenceData1(list_entitys,list_sentences):
    '''
    @summary:将中间结果保存到数据库-线上生产的时候不需要执行
    '''
    import psycopg2
    conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")
    cursor = conn.cursor()
    for list_entity in list_entitys:
        for entity in list_entity:
            if entity.values is not None:
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")"
            else:    
                sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")"
            cursor.execute(sql)
    for list_sentence in list_sentences:
        for sentence in list_sentence:
            str_tokens = "["
            for item in sentence.tokens:
                str_tokens += "'"
                if item=="'":
                    str_tokens += "''"
                else:
                    str_tokens += item
                str_tokens += "',"
            str_tokens = str_tokens[:-1]+"]"
            sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")"
            cursor.execute(sql)
    conn.commit()
    conn.close()


def _handle(item,result_queue):
    dochtml = item["dochtml"]
    docid = item["docid"]
    list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml"))
    flag = False
    if list_innerTable:
        flag = True
        for table in list_innerTable:
            result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)})

def getPredictTable():
    filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv"
    import pandas as pd
    import json
    from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue
    df = pd.read_csv(filename)
    df_data = {"json_table":[],"docid":[]}
    _count = 0
    _sum = len(df["docid"])
    task_queue = Queue()
    result_queue = Queue()
    _index = 0
    for dochtml,docid in zip(df["dochtmlcon"],df["docid"]):
        task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None})
        _index += 1

    mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1)
    mh.run()
    while True:
        try:
            item = result_queue.get(block=True,timeout=1)
            df_data["docid"].append(item["docid"])
            df_data["json_table"].append(item["json_table"])
        except Exception as e:
            print(e)
            break
    df_1 = pd.DataFrame(df_data)
    df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"])

if __name__=="__main__":
    '''
    import glob
    for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"):
        file_txt = str(file).replace("html","txt")
        with codecs.open(file_txt,"a+",encoding="utf8") as f:
            f.write("\n================\n")
            content = codecs.open(file,"r",encoding="utf8").read()
            f.write(segment(tableToText(BeautifulSoup(content,"lxml"))))
    '''        
    # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # print(segment(tableToText(BeautifulSoup(content,"lxml"))))
    getPredictTable()