from bs4 import BeautifulSoup, Comment import copy import sys import os import time import codecs sys.setrecursionlimit(1000000) sys.path.append(os.path.abspath("../..")) sys.path.append(os.path.abspath("..")) from BiddingKG.dl.common.Utils import * from BiddingKG.dl.interface.Entitys import * from BiddingKG.dl.interface.predictor import * from BiddingKG.dl.foolnltk import selffool from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource from BiddingKG.dl.time.re_servicetime import extract_servicetime from BiddingKG.dl.bidway.re_bidway import extract_bidway # def tableToText(soup): ''' @param: soup:网页html的soup @return:处理完表格信息的网页text ''' def getTrs(tbody): #获取所有的tr trs = [] objs = tbody.find_all(recursive=False) for obj in objs: if obj.name=="tr": trs.append(obj) if obj.name=="tbody": for tr in obj.find_all("tr",recursive=False): trs.append(tr) return trs def fixSpan(tbody): # 处理colspan, rowspan信息补全问题 #trs = tbody.findChildren('tr', recursive=False) trs = getTrs(tbody) ths_len = 0 ths = list() trs_set = set() #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱 # 遍历每一个tr for indtr, tr in enumerate(trs): ths_tmp = tr.findChildren('th', recursive=False) #不补全含有表格的tr if len(tr.findChildren('table'))>0: continue if len(ths_tmp) > 0: ths_len = ths_len + len(ths_tmp) for th in ths_tmp: ths.append(th) trs_set.add(tr) # 遍历每行中的element tds = tr.findChildren(recursive=False) for indtd, td in enumerate(tds): # 若有colspan 则补全同一行下一个位置 if 'colspan' in td.attrs: if str(re.sub("[^0-9]","",str(td['colspan'])))!="": col = int(re.sub("[^0-9]","",str(td['colspan']))) if col<100 and len(td.get_text())<1000: td['colspan'] = 1 for i in range(1, col, 1): td.insert_after(copy.copy(td)) for indtr, tr in enumerate(trs): ths_tmp = tr.findChildren('th', recursive=False) #不补全含有表格的tr if len(tr.findChildren('table'))>0: continue if len(ths_tmp) > 0: ths_len = ths_len + len(ths_tmp) for th in ths_tmp: ths.append(th) trs_set.add(tr) # 遍历每行中的element tds = tr.findChildren(recursive=False) for indtd, td in enumerate(tds): # 若有rowspan 则补全下一行同样位置 if 'rowspan' in td.attrs: if str(re.sub("[^0-9]","",str(td['rowspan'])))!="": row = int(re.sub("[^0-9]","",str(td['rowspan']))) td['rowspan'] = 1 for i in range(1, row, 1): # 获取下一行的所有td， 在对应的位置插入 if indtr+i= (indtd) and len(tds1)>0: if indtd > 0: tds1[indtd - 1].insert_after(copy.copy(td)) else: tds1[0].insert_before(copy.copy(td)) def getTable(tbody): #trs = tbody.findChildren('tr', recursive=False) trs = getTrs(tbody) inner_table = [] for tr in trs: tr_line = [] tds = tr.findChildren(['td','th'], recursive=False) for td in tds: tr_line.append([re.sub('\xa0','',segment(td,final=False)),0]) #tr_line.append([td.get_text(),0]) inner_table.append(tr_line) return inner_table #处理表格不对齐的问题 def fixTable(inner_table,fix_value="~~"): maxWidth = 0 for item in inner_table: if len(item)>maxWidth: maxWidth = len(item) for i in range(len(inner_table)): if len(inner_table[i])0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3: return True return False def getsimilarity(line,line1): same_count = 0 for item,item1 in zip(line,line1): if item[1]==item1[1]: same_count += 1 return same_count/len(line) def selfrepair(inner_table,index,dye_set,key_set): ''' @summary: 计算每个节点受到的挤压度来判断是否需要染色 ''' #print("B",inner_table[index]) min_presure = 3 list_dye = [] first = None count = 0 temp_set = set() _index = 0 for item in inner_table[index]: if first is None: first = item[1] if item[0] not in temp_set: count += 1 temp_set.add(item[0]) else: if first == item[1]: if item[0] not in temp_set: temp_set.add(item[0]) count += 1 else: list_dye.append([first,count,_index]) first = item[1] temp_set.add(item[0]) count = 1 _index += 1 list_dye.append([first,count,_index]) if len(list_dye)>1: begin = 0 end = 0 for i in range(len(list_dye)): end = list_dye[i][2] dye_flag = False #首尾要求压力减一 if i==0: if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1: dye_flag = True dye_type = list_dye[i+1][0] elif i==len(list_dye)-1: if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1: dye_flag = True dye_type = list_dye[i-1][0] else: if list_dye[i][1]>1: if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i+1][0] if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i-1][0] else: if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i+1][0] if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i-1][0] if dye_flag: for h in range(begin,end): inner_table[index][h][1] = dye_type dye_set.add((inner_table[index][h][0],dye_type)) key_set.add(inner_table[index][h][0]) begin = end #print("E",inner_table[index]) def otherrepair(inner_table,index,dye_set,key_set): list_provide_repair = [] if index==0 and len(inner_table)>1: list_provide_repair.append(index+1) elif index==len(inner_table)-1: list_provide_repair.append(index-1) else: list_provide_repair.append(index+1) list_provide_repair.append(index-1) for provide_index in list_provide_repair: if not repairNeeded(inner_table[provide_index]): same_prob = getsimilarity(inner_table[index], inner_table[provide_index]) if same_prob>=0.8: for i in range(len(inner_table[provide_index])): if inner_table[index][i][1]!=inner_table[provide_index][i][1]: dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1])) key_set.add(inner_table[index][i][0]) inner_table[index][i][1] = inner_table[provide_index][i][1] elif same_prob<=0.2: for i in range(len(inner_table[provide_index])): if inner_table[index][i][1]==inner_table[provide_index][i][1]: dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1])) key_set.add(inner_table[index][i][0]) inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1 len_dye_set = len(dye_set) height = len(inner_table) for i in range(height): if repairNeeded(inner_table[i]): selfrepair(inner_table,i,dye_set,key_set) #otherrepair(inner_table,i,dye_set,key_set) for h in range(len(inner_table)): for w in range(len(inner_table[0])): if inner_table[h][w][0] in key_set: for item in dye_set: if inner_table[h][w][0]==item[0]: inner_table[h][w][1] = item[1] #如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代 if len(dye_set)!=len(key_set): for i in range(height): if repairNeeded(inner_table[i]): selfrepair(inner_table,i,dye_set,key_set) #otherrepair(inner_table,i,dye_set,key_set) return if len(dye_set)==len_dye_set: ''' for i in range(height): if repairNeeded(inner_table[i]): otherrepair(inner_table,i,dye_set,key_set) ''' return repairTable(inner_table, dye_set, key_set) def sliceTable(inner_table,fix_value="~~"): #进行分块 height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) last_head = None last_is_same_value = False; for h in range(height): is_all_key = True#是否是全表头行 is_all_value = True#是否是全属性值 is_same_with_lastHead = True#和上一行的结构是否相同 is_same_value=True#一行的item都一样 #is_same_first_item = True#与上一行的第一项是否相同 same_value = inner_table[h][0][0] for w in range(width): if last_head is not None: if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0: is_all_key = False if inner_table[h][w][0]==1: is_all_value = False if inner_table[h][w][1]!= inner_table[h-1][w][1]: is_same_with_lastHead = False if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value: is_same_value = False else: if re.search("\d+",same_value) is not None: is_same_value = False if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]: is_same_first_item = False last_head = h if last_is_same_value: last_is_same_value = is_same_value continue if is_same_value: head_list.append(h) last_is_same_value = is_same_value continue if not is_all_key: if not is_same_with_lastHead: head_list.append(h) head_list.append(height) return head_list def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5): set_item = set() height = len(inner_table) width = len(inner_table[0]) for i in range(height): for j in range(width): item = inner_table[i][j][0] set_item.add(item) list_item = list(set_item) x = [] for item in list_item: x.append(getPredictor("form").encode(item)) predict_y = getPredictor("form").predict(np.array(x),type="item") _dict = dict() for item,values in zip(list_item,list(predict_y)): _dict[item] = values[1] # print("##",item,values) #print(_dict) for i in range(height): for j in range(width): item = inner_table[i][j][0] inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0) # print("=====") # for item in inner_table: # print(item) # print("======") repairTable(inner_table) head_list = sliceTable(inner_table) return inner_table,head_list def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5): data_x,data_position = getPredictor("form").getModel("context").encode(inner_table) predict_y = getPredictor("form").getModel("context").predict(data_x) for _position,_y in zip(data_position,predict_y): _w = _position[0] _h = _position[1] if _y[1]>prob_min: inner_table[_h][_w][1] = 1 else: inner_table[_h][_w][1] = 0 # print("=====") # for item in inner_table: # print(item) # print("======") repairTable(inner_table) head_list = sliceTable(inner_table) return inner_table,head_list #设置表头 def setHead_inline(inner_table,prob_min=0.64): pad_row = "@@" pad_col = "##" removePadding(inner_table, pad_row, pad_col) pad_pattern = re.compile(pad_row+"|"+pad_col) height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) #行表头 is_head_last = False for i in range(height): is_head = False is_long_value = False #判断是否是全padding值 is_same_value = True same_value = inner_table[i][0][0] for j in range(width): if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row: is_same_value = False break #predict is head or not with model temp_item = "" for j in range(width): temp_item += inner_table[i][j][0]+"|" temp_item = re.sub(pad_pattern,"",temp_item) form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line") if form_prob is not None: if form_prob[0][1]>prob_min: is_head = True else: is_head = False #print(temp_item,form_prob) if len(inner_table[i][0][0])>40: is_long_value = True if is_head or is_long_value or is_same_value: #不把连续表头分开 if not is_head_last: head_list.append(i) if is_long_value or is_same_value: head_list.append(i+1) if is_head: for j in range(width): inner_table[i][j][1] = 1 is_head_last = is_head head_list.append(height) #列表头 for i in range(len(head_list)-1): head_begin = head_list[i] head_end = head_list[i+1] #最后一列不设置为列表头 for i in range(width-1): is_head = False #predict is head or not with model temp_item = "" for j in range(head_begin,head_end): temp_item += inner_table[j][i][0]+"|" temp_item = re.sub(pad_pattern,"",temp_item) form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line") if form_prob is not None: if form_prob[0][1]>prob_min: is_head = True else: is_head = False if is_head: for j in range(head_begin,head_end): inner_table[j][i][1] = 2 addPadding(inner_table, pad_row, pad_col) return inner_table,head_list #设置表头 def setHead_withRule(inner_table,pattern,pat_value,count): height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) #行表头 is_head_last = False for i in range(height): set_match = set() is_head = False is_long_value = False is_same_value = True same_value = inner_table[i][0][0] for j in range(width): if inner_table[i][j][0]!=same_value: is_same_value = False break for j in range(width): if re.search(pat_value,inner_table[i][j][0]) is not None: is_head = False break str_find = re.findall(pattern,inner_table[i][j][0]) if len(str_find)>0: set_match.add(inner_table[i][j][0]) if len(set_match)>=count: is_head = True if len(inner_table[i][0][0])>40: is_long_value = True if is_head or is_long_value or is_same_value: if not is_head_last: head_list.append(i) if is_head: for j in range(width): inner_table[i][j][1] = 1 is_head_last = is_head head_list.append(height) #列表头 for i in range(len(head_list)-1): head_begin = head_list[i] head_end = head_list[i+1] #最后一列不设置为列表头 for i in range(width-1): set_match = set() is_head = False for j in range(head_begin,head_end): if re.search(pat_value,inner_table[j][i][0]) is not None: is_head = False break str_find = re.findall(pattern,inner_table[j][i][0]) if len(str_find)>0: set_match.add(inner_table[j][i][0]) if len(set_match)>=count: is_head = True if is_head: for j in range(head_begin,head_end): inner_table[j][i][1] = 2 return inner_table,head_list #取得表格的处理方向 def getDirect(inner_table,begin,end): ''' column_head = set() row_head = set() widths = len(inner_table[0]) for height in range(begin,end): for width in range(widths): if inner_table[height][width][1] ==1: row_head.add(height) if inner_table[height][width][1] ==2: column_head.add(width) company_pattern = re.compile("公司") if 0 in column_head and begin not in row_head: return "column" if 0 in column_head and begin in row_head: for height in range(begin,end): count = 0 count_flag = True for width_index in range(width): if inner_table[height][width_index][1]==0: if re.search(company_pattern,inner_table[height][width_index][0]) is not None: count += 1 else: count_flag = False if count_flag and count>=2: return "column" return "row" ''' count_row_keys = 0 count_column_keys = 0 width = len(inner_table[0]) if begin=2: return "column" if count_column_keys>count_row_keys: return "column" return "row" #根据表格处理方向生成句子， def getTableText(inner_table,head_list,key_direct=False): # packPattern = "(标包|[标包][号段名])" packPattern = "(标包|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则，补充采购类包名 rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标)" # 2020/11/23 大网站规则，添加序号为排序 entityPattern = "(候选|([中投]标|报价)|单位名称|供应商|金额)" height = len(inner_table) width = len(inner_table[0]) text = "" for head_i in range(len(head_list)-1): head_begin = head_list[head_i] head_end = head_list[head_i+1] direct = getDirect(inner_table, head_begin, head_end) #若只有一行，则直接按行读取 if head_end-head_begin==1: text_line = "" for i in range(head_begin,head_end): for w in range(len(inner_table[i])): if inner_table[i][w][1]==1: _punctuation = ":" else: _punctuation = "," if w>0: if inner_table[i][w][0]!= inner_table[i][w-1][0]: text_line += inner_table[i][w][0]+_punctuation else: text_line += inner_table[i][w][0]+_punctuation text_line = text_line+"。" if text_line!="" else text_line text += text_line else: #构建一个共现矩阵 table_occurence = [] for i in range(head_begin,head_end): line_oc = [] for j in range(width): cell = inner_table[i][j] line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":""}) table_occurence.append(line_oc) occu_height = len(table_occurence) occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0 #为每个属性值寻找表头 for i in range(occu_height): for j in range(occu_width): cell = table_occurence[i][j] #是属性值 if cell["type"]==0 and cell["text"]!="": left_head = "" top_head = "" find_flag = False temp_head = "" for loop_i in range(1,i+1): if not key_direct: key_values = [1,2] else: key_values = [1] if table_occurence[i-loop_i][j]["type"] in key_values: if find_flag: if table_occurence[i-loop_i][j]["text"]!=temp_head: top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head else: top_head = table_occurence[i-loop_i][j]["text"]+":"+top_head find_flag = True temp_head = table_occurence[i-loop_i][j]["text"] table_occurence[i-loop_i][j]["occu_count"] += 1 else: #找到表头后遇到属性值就返回 if find_flag: break cell["top_head"] += top_head find_flag = False temp_head = "" for loop_j in range(1,j+1): if not key_direct: key_values = [1,2] else: key_values = [2] if table_occurence[i][j-loop_j]["type"] in key_values: if find_flag: if table_occurence[i][j-loop_j]["text"]!=temp_head: left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head else: left_head = table_occurence[i][j-loop_j]["text"]+":"+left_head find_flag = True temp_head = table_occurence[i][j-loop_j]["text"] table_occurence[i][j-loop_j]["occu_count"] += 1 else: if find_flag: break cell["left_head"] += left_head if direct=="row": for i in range(occu_height): pack_text = "" rank_text = "" entity_text = "" text_line = "" #在同一句话中重复的可以去掉 text_set = set() for j in range(width): cell = table_occurence[i][j] if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0): cell = table_occurence[i][j] head = (cell["top_head"]+":") if len(cell["top_head"])>0 else "" head += cell["left_head"] if str(head+cell["text"]) in text_set: continue if re.search(packPattern,head) is not None: pack_text += head+cell["text"]+"，" elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif #排名替换为同一种表达 rank_text += head+cell["text"]+"，" #print(rank_text) elif re.search(entityPattern,head) is not None: entity_text += head+cell["text"]+"，" #print(entity_text) else: text_line += head+cell["text"]+"，" text_set.add(str(head+cell["text"])) text += pack_text+rank_text+entity_text+text_line text = text[:-1]+"。" if len(text)>0 else text else: for j in range(occu_width): pack_text = "" rank_text = "" entity_text = "" text_line = "" text_set = set() for i in range(occu_height): cell = table_occurence[i][j] if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0): cell = table_occurence[i][j] head = (cell["left_head"]+"") if len(cell["left_head"])>0 else "" head += cell["top_head"] if str(head+cell["text"]) in text_set: continue if re.search(packPattern,head) is not None: pack_text += head+cell["text"]+"，" elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif #排名替换为同一种表达 rank_text += head+cell["text"]+"，" #print(rank_text) elif re.search(entityPattern,head) is not None: entity_text += head+cell["text"]+"，" #print(entity_text) else: text_line += head+cell["text"]+"，" text_set.add(str(head+cell["text"])) text += pack_text+rank_text+entity_text+text_line text = text[:-1]+"。" if len(text)>0 else text # if direct=="row": # for i in range(head_begin,head_end): # pack_text = "" # rank_text = "" # entity_text = "" # text_line = "" # #在同一句话中重复的可以去掉 # text_set = set() # for j in range(width): # cell = inner_table[i][j] # #是属性值 # if cell[1]==0 and cell[0]!="": # head = "" # # find_flag = False # temp_head = "" # for loop_i in range(0,i+1-head_begin): # if not key_direct: # key_values = [1,2] # else: # key_values = [1] # if inner_table[i-loop_i][j][1] in key_values: # if find_flag: # if inner_table[i-loop_i][j][0]!=temp_head: # head = inner_table[i-loop_i][j][0]+":"+head # else: # head = inner_table[i-loop_i][j][0]+":"+head # find_flag = True # temp_head = inner_table[i-loop_i][j][0] # else: # #找到表头后遇到属性值就返回 # if find_flag: # break # # find_flag = False # temp_head = "" # # # # for loop_j in range(1,j+1): # if not key_direct: # key_values = [1,2] # else: # key_values = [2] # if inner_table[i][j-loop_j][1] in key_values: # if find_flag: # if inner_table[i][j-loop_j][0]!=temp_head: # head = inner_table[i][j-loop_j][0]+":"+head # else: # head = inner_table[i][j-loop_j][0]+":"+head # find_flag = True # temp_head = inner_table[i][j-loop_j][0] # else: # if find_flag: # break # # if str(head+inner_table[i][j][0]) in text_set: # continue # if re.search(packPattern,head) is not None: # pack_text += head+inner_table[i][j][0]+"，" # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif # #排名替换为同一种表达 # rank_text += head+inner_table[i][j][0]+"，" # #print(rank_text) # elif re.search(entityPattern,head) is not None: # entity_text += head+inner_table[i][j][0]+"，" # #print(entity_text) # else: # text_line += head+inner_table[i][j][0]+"，" # text_set.add(str(head+inner_table[i][j][0])) # text += pack_text+rank_text+entity_text+text_line # text = text[:-1]+"。" if len(text)>0 else text # else: # for j in range(width): # # rank_text = "" # entity_text = "" # text_line = "" # text_set = set() # for i in range(head_begin,head_end): # cell = inner_table[i][j] # #是属性值 # if cell[1]==0 and cell[0]!="": # find_flag = False # head = "" # temp_head = "" # # for loop_j in range(1,j+1): # if not key_direct: # key_values = [1,2] # else: # key_values = [2] # if inner_table[i][j-loop_j][1] in key_values: # if find_flag: # if inner_table[i][j-loop_j][0]!=temp_head: # head = inner_table[i][j-loop_j][0]+":"+head # else: # head = inner_table[i][j-loop_j][0]+":"+head # find_flag = True # temp_head = inner_table[i][j-loop_j][0] # else: # if find_flag: # break # find_flag = False # temp_head = "" # for loop_i in range(0,i+1-head_begin): # if not key_direct: # key_values = [1,2] # else: # key_values = [1] # if inner_table[i-loop_i][j][1] in key_values: # if find_flag: # if inner_table[i-loop_i][j][0]!=temp_head: # head = inner_table[i-loop_i][j][0]+":"+head # else: # head = inner_table[i-loop_i][j][0]+":"+head # find_flag = True # temp_head = inner_table[i-loop_i][j][0] # else: # if find_flag: # break # if str(head+inner_table[i][j][0]) in text_set: # continue # if re.search(rankPattern,head) is not None: # rank_text += head+inner_table[i][j][0]+"，" # #print(rank_text) # elif re.search(entityPattern,head) is not None: # entity_text += head+inner_table[i][j][0]+"，" # #print(entity_text) # else: # text_line += head+inner_table[i][j][0]+"，" # text_set.add(str(head+inner_table[i][j][0])) # text += rank_text+entity_text+text_line # text = text[:-1]+"。" if len(text)>0 else text return text def removeFix(inner_table,fix_value="~~"): height = len(inner_table) width = len(inner_table[0]) for h in range(height): for w in range(width): if inner_table[h][w][0]==fix_value: inner_table[h][w][0] = "" def trunTable(tbody): fixSpan(tbody) inner_table = getTable(tbody) inner_table = fixTable(inner_table) if len(inner_table)>0 and len(inner_table[0])>0: #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3) #inner_table,head_list = setHead_inline(inner_table) # inner_table,head_list = setHead_initem(inner_table,pat_head) inner_table,head_list = setHead_incontext(inner_table,pat_head) # print(inner_table) # for begin in range(len(head_list[:-1])): # for item in inner_table[head_list[begin]:head_list[begin+1]]: # print(item) # print("====") removeFix(inner_table) # print("----") # print(head_list) # for item in inner_table: # print(item) tbody.string = getTableText(inner_table,head_list) #print(tbody.string) tbody.name = "turntable" return inner_table return None pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标)$') #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)') pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)") list_innerTable = [] tbodies = soup.find_all('table') # 遍历表格中的每个tbody #逆序处理嵌套表格 for tbody_index in range(1,len(tbodies)+1): tbody = tbodies[len(tbodies)-tbody_index] inner_table = trunTable(tbody) list_innerTable.append(inner_table) tbodies = soup.find_all('tbody') # 遍历表格中的每个tbody #逆序处理嵌套表格 for tbody_index in range(1,len(tbodies)+1): tbody = tbodies[len(tbodies)-tbody_index] inner_table = trunTable(tbody) list_innerTable.append(inner_table) return soup # return list_innerTable #数据清洗 def segment(soup,final=True): # print("==") # print(soup) # print("====") #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"] subspaceList = ["td",'a',"span","p"] if soup.name in subspaceList: #判断有值叶子节点数 _count = 0 for child in soup.find_all(recursive=True): if child.get_text().strip()!="" and len(child.find_all())==0: _count += 1 if _count<=1: text = soup.get_text() # 2020/11/24 大网站规则添加 if 'title' in soup.attrs: if '...' in soup.get_text() and (soup.get_text()[:-3]).strip() in soup.attrs['title']: text = soup.attrs['title'] # _list = [] # for x in re.split("\s+",text): # if x.strip()!="": # _list.append(len(x)) # if len(_list)>0: # _minLength = min(_list) # if _minLength>2: # _substr = "，" # else: # _substr = "" # else: # _substr = "" text = text.replace("\r\n","，").replace("\n","，") text = re.sub("\s+","##space##",text) return text segList = ["title"] commaList = ["div","br","td","p"] #commaList = [] spaceList = ["span"] tbodies = soup.find_all('tbody') if len(tbodies) == 0: tbodies = soup.find_all('table') # 递归遍历所有节点,插入符号 for child in soup.find_all(recursive=True): if child.name in segList: child.insert_after("。") if child.name in commaList: child.insert_after("，") # if child.name in subspaceList: # child.insert_before("#subs"+str(child.name)+"#") # child.insert_after("#sube"+str(child.name)+"#") # if child.name in spaceList: # child.insert_after(" ") text = str(soup.get_text()) #替换英文冒号为中文冒号 text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text) #替换为中文逗号 text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text) #替换为中文分号 text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text) #替换"""为"“",否则导入deepdive出错 text = text.replace('"',"“").replace("\r","").replace("\n","，") text = re.sub("\s{4,}","，",text) #替换标点 #替换连续的标点 if final: text = re.sub("##space##"," ",text) punc_pattern = "(?P[。，；：:,\s]+)" list_punc = re.findall(punc_pattern,text) list_punc.sort(key=lambda x:len(x),reverse=True) for punc_del in list_punc: if len(punc_del)>1: text = re.sub(punc_del,punc_del[-1],text) #将连续的中文句号替换为一个 text_split = text.split("。") text_split = [x for x in text_split if len(x)>0] text = "。".join(text_split) # #删除标签中的所有空格 # for subs in subspaceList: # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#" # while(True): # oneMatch = re.search(re.compile(patten),text) # if oneMatch is not None: # _match = oneMatch.group(1) # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match) # else: # break # text过大报错 LOOP_LEN = 10000 LOOP_BEGIN = 0 _text = "" if len(text)<10000000: while(LOOP_BEGIN3 and len(child_text) <50: # 先判断是否字数少于50，成立加逗号，否则加句号 child.insert_after("，") elif len(child_text) >=50: child.insert_after("。") #if child.name in spaceList: #child.insert_after(" ") text = str(soup.get_text()) text = re.sub("\s{5,}","，",text) text = text.replace('"',"“").replace("\r","").replace("\n","，") #替换"""为"“",否则导入deepdive出错 text = text.replace('"',"“") #text = text.replace('"',"“").replace("\r","").replace("\n","") #删除所有空格 text = re.sub("\s+","#nbsp#",text) text_list = text.split('#nbsp#') new_text = '' for i in range(len(text_list)-1): if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']: new_text += text_list[i] elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []: new_text += text_list[i] + '。' elif re.findall('([0-9]、)', text_list[i+1][:4]) != []: new_text += text_list[i] + '；' elif text_list[i].isdigit() and text_list[i+1].isdigit(): new_text += text_list[i] + ' ' elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']: new_text += text_list[i] elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3: new_text += text_list[i] + '，' else: new_text += text_list[i] new_text += text_list[-1] text = new_text #替换英文冒号为中文冒号 text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text) #替换为中文逗号 text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text) #替换为中文分号 text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text) #替换标点 while(True): #替换连续的标点 punc = re.search("，(?P：|。|，|；)\s*",text) if punc is not None: text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text) punc = re.search("(?P：|。|，|；)\s*，",text) if punc is not None: text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text) else: #替换标点之后的空格 punc = re.search("(?P：|。|，|；)\s+",text) if punc is not None: text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text) else: break #将连续的中文句号替换为一个 text_split = text.split("。") text_split = [x for x in text_split if len(x)>0] text = "。".join(text_split) #替换中文括号为英文括号 text = re.sub("（","(",text) text = re.sub("）",")",text) return text ''' #连续实体合并（弃用） def union_ner(list_ner): result_list = [] union_index = [] union_index_set = set() for i in range(len(list_ner)-1): if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2: if list_ner[i][1]-list_ner[i+1][0]==1: union_index_set.add(i) union_index_set.add(i+1) union_index.append((i,i+1)) for i in range(len(list_ner)): if i not in union_index_set: result_list.append(list_ner[i]) for item in union_index: #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3])) result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))) return result_list def getTokensAndNers(sentences,MAXAREA = 10000,useselffool=False): ''' @param: sentences:句子数 @return 限流执行后的分词和实体识别list ''' def getData(tokens,ners,process_data): process_sentences = [item[1] for item in process_data] token_ = selffool.cut(process_sentences) if useselffool: ner_ = selffool.self_ner(process_sentences) else: ner_ = selffool.ner(process_sentences) for i in range(len(token_)): the_index = process_data[i][0] tokens[the_index] = token_[i] ners[the_index] = ner_[i] sents = [] for i in range(len(sentences)): sents.append([i,sentences[i]]) sents.sort(key=lambda x:len(x[1]),reverse=True) index_ = 0 tokens = [[]for i in range(len(sentences))] ners = [[]for i in range(len(sentences))] while(True): width = len(sents[index_][1]) height = MAXAREA//width+1 if height>len(sents)-index_: height = len(sents)-index_ process_data = sents[index_:index_+height] getData(tokens, ners, process_data) index_ += height if index_>=len(sents): break return tokens,ners def getTokens(sentences,MAXAREA = 10000,useselffool=True): ''' @param: sentences:句子数 @return 限流执行后的分词list ''' def getData(tokens,process_data): process_sentences = [item[1] for item in process_data] token_ = selffool.cut(process_sentences) for i in range(len(token_)): the_index = process_data[i][0] tokens[the_index] = token_[i] sents = [] for i in range(len(sentences)): sents.append([i,sentences[i]]) sents.sort(key=lambda x:len(x[1]),reverse=True) index_ = 0 tokens = [[]for i in range(len(sentences))] while(True): width = len(sents[index_][1]) height = MAXAREA//width+1 if height>len(sents)-index_: height = len(sents)-index_ process_data = sents[index_:index_+height] getData(tokens, process_data) index_ += height if index_>=len(sents): break return tokens def getNers(sentences,MAXAREA = 10000,useselffool=False): ''' @param: sentences:句子数 @return 限流执行后的实体识别list ''' def getData(ners,process_data): process_sentences = [item[1] for item in process_data] if useselffool: ner_ = selffool.self_ner(process_sentences) else: ner_ = selffool.ner(process_sentences) for i in range(len(ner_)): the_index = process_data[i][0] ners[the_index] = ner_[i] sents = [] for i in range(len(sentences)): sents.append([i,sentences[i]]) sents.sort(key=lambda x:len(x[1]),reverse=True) index_ = 0 ners = [[]for i in range(len(sentences))] while(True): width = len(sents[index_][1]) height = MAXAREA//width+1 if height>len(sents)-index_: height = len(sents)-index_ process_data = sents[index_:index_+height] getData( ners, process_data) index_ += height if index_>=len(sents): break return ners # def get_preprocessed(articles,useselffool=False): # ''' # @summary:预处理步骤，NLP处理、实体识别 # @param: # articles:待处理的文章list [[id,source,jointime,doc_id,title]] # @return:list of articles,list of each article of sentences,list of each article of entitys # ''' # list_articles = [] # list_sentences = [] # list_entitys = [] # cost_time = dict() # for article in articles: # list_sentences_temp = [] # list_entitys_temp = [] # doc_id = article[0] # sourceContent = article[1] # _send_doc_id = article[3] # _title = article[4] # #表格处理 # key_preprocess = "tableToText" # start_time = time.time() # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml"))) # # # log(article_processed) # # if key_preprocess not in cost_time: # cost_time[key_preprocess] = 0 # cost_time[key_preprocess] += time.time()-start_time # # #article_processed = article[1] # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title)) # #nlp处理 # if article_processed is not None and len(article_processed)!=0: # split_patten = "。" # sentences = [] # _begin = 0 # for _iter in re.finditer(split_patten,article_processed): # sentences.append(article_processed[_begin:_iter.span()[1]]) # _begin = _iter.span()[1] # sentences.append(article_processed[_begin:]) # # lemmas = [] # doc_offsets = [] # dep_types = [] # dep_tokens = [] # # time1 = time.time() # # ''' # tokens_all = fool.cut(sentences) # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all) # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all) # ner_entitys_all = fool.ner(sentences) # ''' # #限流执行 # key_nerToken = "nerToken" # start_time = time.time() # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool) # if key_nerToken not in cost_time: # cost_time[key_nerToken] = 0 # cost_time[key_nerToken] += time.time()-start_time # # # for sentence_index in range(len(sentences)): # # # # list_sentence_entitys = [] # sentence_text = sentences[sentence_index] # tokens = tokens_all[sentence_index] # # list_tokenbegin = [] # begin = 0 # for i in range(0,len(tokens)): # list_tokenbegin.append(begin) # begin += len(str(tokens[i])) # list_tokenbegin.append(begin+1) # #pos_tag = pos_all[sentence_index] # pos_tag = "" # # ner_entitys = ner_entitys_all[sentence_index] # # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys)) # # #识别package # # # #识别实体 # for ner_entity in ner_entitys: # begin_index_temp = ner_entity[0] # end_index_temp = ner_entity[1] # entity_type = ner_entity[2] # entity_text = ner_entity[3] # # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==begin_index_temp: # begin_index = j # break # elif list_tokenbegin[j]>begin_index_temp: # begin_index = j-1 # break # begin_index_temp += len(str(entity_text)) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=begin_index_temp: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # #去掉标点符号 # entity_text = re.sub("[,，。：]","",entity_text) # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1)) # # # #使用正则识别金额 # entity_type = "money" # # #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*" # # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*", # "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*", # "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*", # "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"} # # set_begin = set() # for pattern_key in list_money_pattern.keys(): # pattern = re.compile(list_money_pattern[pattern_key]) # all_match = re.findall(pattern, sentence_text) # index = 0 # for i in range(len(all_match)): # if len(all_match[i][0])>0: # # print("===",all_match[i]) # #print(all_match[i][0]) # unit = "" # entity_text = all_match[i][3] # if pattern_key in ["key_word","front_m"]: # unit = all_match[i][1] # else: # unit = all_match[i][4] # if entity_text.find("元")>=0: # unit = "" # # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征 # # begin_index_temp = index # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==index: # begin_index = j # break # elif list_tokenbegin[j]>index: # begin_index = j-1 # break # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体 # end_index_temp = index # #index += len(str(all_match[i][0])) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=index: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text) # if len(unit)>0: # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) # else: # entity_text = str(getUnifyMoney(entity_text)) # # _exists = False # for item in list_sentence_entitys: # if item.entity_id==entity_id and item.entity_type==entity_type: # _exists = True # if not _exists: # if float(entity_text)>10: # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp)) # # else: # index += 1 # # list_sentence_entitys.sort(key=lambda x:x.begin_index) # list_entitys_temp = list_entitys_temp+list_sentence_entitys # list_sentences.append(list_sentences_temp) # list_entitys.append(list_entitys_temp) # return list_articles,list_sentences,list_entitys,cost_time def get_preprocessed(articles,useselffool=False): ''' @summary:预处理步骤，NLP处理、实体识别 @param: articles:待处理的文章list [[id,source,jointime,doc_id,title]] @return:list of articles,list of each article of sentences,list of each article of entitys ''' cost_time = dict() list_articles = get_preprocessed_article(articles,cost_time) list_sentences = get_preprocessed_sentences(list_articles,True,cost_time) list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time) return list_articles,list_sentences,list_entitys,cost_time def get_preprocessed_article(articles,cost_time = dict(),useselffool=True): ''' :param articles: 待处理的article source html :param useselffool: 是否使用selffool :return: list_articles ''' list_articles = [] for article in articles: doc_id = article[0] sourceContent = article[1] _send_doc_id = article[3] _title = article[4] #表格处理 key_preprocess = "tableToText" start_time = time.time() article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml"))) # log(article_processed) if key_preprocess not in cost_time: cost_time[key_preprocess] = 0 cost_time[key_preprocess] += time.time()-start_time #article_processed = article[1] list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title)) return list_articles def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()): ''' :param list_articles: 经过预处理的article text :return: list_sentences ''' list_sentences = [] for article in list_articles: list_sentences_temp = [] list_entitys_temp = [] doc_id = article.id _send_doc_id = article.doc_id _title = article.title #表格处理 key_preprocess = "tableToText" start_time = time.time() article_processed = article.content if key_preprocess not in cost_time: cost_time[key_preprocess] = 0 cost_time[key_preprocess] += time.time()-start_time #nlp处理 if article_processed is not None and len(article_processed)!=0: split_patten = "。" sentences = [] _begin = 0 sentences_set = set() for _iter in re.finditer(split_patten,article_processed): _sen = article_processed[_begin:_iter.span()[1]] if len(_sen)>0 and _sen not in sentences_set: sentences.append(_sen) sentences_set.add(_sen) _begin = _iter.span()[1] _sen = article_processed[_begin:] if len(_sen)>0 and _sen not in sentences_set: sentences.append(_sen) sentences_set.add(_sen) article.content = "".join(sentences) # sentences.append(article_processed[_begin:]) lemmas = [] doc_offsets = [] dep_types = [] dep_tokens = [] time1 = time.time() ''' tokens_all = fool.cut(sentences) #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all) #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all) ner_entitys_all = fool.ner(sentences) ''' #限流执行 key_nerToken = "nerToken" start_time = time.time() tokens_all = getTokens(sentences,useselffool=useselffool) if key_nerToken not in cost_time: cost_time[key_nerToken] = 0 cost_time[key_nerToken] += time.time()-start_time for sentence_index in range(len(sentences)): sentence_text = sentences[sentence_index] tokens = tokens_all[sentence_index] #pos_tag = pos_all[sentence_index] pos_tag = "" ner_entitys = "" list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys)) if len(list_sentences_temp)==0: list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags="")) list_sentences.append(list_sentences_temp) return list_sentences def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()): ''' :param list_sentences:分局情况 :param cost_time: :return: list_entitys ''' list_entitys = [] for list_sentence in list_sentences: sentences = [] list_entitys_temp = [] for _sentence in list_sentence: sentences.append(_sentence.sentence_text) lemmas = [] doc_offsets = [] dep_types = [] dep_tokens = [] time1 = time.time() ''' tokens_all = fool.cut(sentences) #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all) #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all) ner_entitys_all = fool.ner(sentences) ''' #限流执行 key_nerToken = "nerToken" start_time = time.time() ner_entitys_all = getNers(sentences,useselffool=useselffool) if key_nerToken not in cost_time: cost_time[key_nerToken] = 0 cost_time[key_nerToken] += time.time()-start_time for sentence_index in range(len(list_sentence)): list_sentence_entitys = [] sentence_text = list_sentence[sentence_index].sentence_text tokens = list_sentence[sentence_index].tokens doc_id = list_sentence[sentence_index].doc_id list_tokenbegin = [] begin = 0 for i in range(0,len(tokens)): list_tokenbegin.append(begin) begin += len(str(tokens[i])) list_tokenbegin.append(begin+1) #pos_tag = pos_all[sentence_index] pos_tag = "" ner_entitys = ner_entitys_all[sentence_index] #识别package #识别实体 for ner_entity in ner_entitys: begin_index_temp = ner_entity[0] end_index_temp = ner_entity[1] entity_type = ner_entity[2] entity_text = ner_entity[3] for j in range(len(list_tokenbegin)): if list_tokenbegin[j]==begin_index_temp: begin_index = j break elif list_tokenbegin[j]>begin_index_temp: begin_index = j-1 break begin_index_temp += len(str(entity_text)) for j in range(begin_index,len(list_tokenbegin)): if list_tokenbegin[j]>=begin_index_temp: end_index = j-1 break entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) #去掉标点符号 entity_text = re.sub("[,，。：]","",entity_text) list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1)) #使用正则识别金额 entity_type = "money" #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*" list_money_pattern = {"cn":"(()()(?P[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())", "key_word":"((?P(?:[￥¥]+，?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果：)(?:[,（\(]*\s*(?P[万元]*(?P[台个只]*))\s*[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?(?P[%])*\s*(?P[万元]*(?P[台个只]*))\s*[）\)]?))", "front_m":"((?P(?:[（\(]?\s*(?P[万元]+)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)())", "behind_m":"(()()(?P[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?(?P[万元]+(?P[台个只]*))[\)）]?)"} pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"])) set_begin = set() # for pattern_key in list_money_pattern.keys(): # for pattern_key in ["cn","key_word","behind_m","front_m"]: # # pattern = re.compile(list_money_pattern[pattern_key]) # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?\s*([万元]*)\s*[）\)]?))*|(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*|((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*") # all_match = re.findall(pattern, sentence_text) # index = 0 # for i in range(len(all_match)): # if len(all_match[i][0])>0: # print("===",all_match[i]) # #print(all_match[i][0]) # unit = "" # entity_text = all_match[i][3] # if pattern_key in ["key_word","front_m"]: # unit = all_match[i][1] # if pattern_key=="key_word": # if all_match[i][1]=="" and all_match[i][4]!="": # unit = all_match[i][4] # else: # unit = all_match[i][4] # if entity_text.find("元")>=0: # unit = "" # # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征 # begin_index_temp = index # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==index: # begin_index = j # break # elif list_tokenbegin[j]>index: # begin_index = j-1 # break # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体 # end_index_temp = index # #index += len(str(all_match[i][0])) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=index: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text) # if len(unit)>0: # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) # else: # entity_text = str(getUnifyMoney(entity_text)) # # _exists = False # for item in list_sentence_entitys: # if item.entity_id==entity_id and item.entity_type==entity_type: # _exists = True # if not _exists: # if float(entity_text)>1: # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp)) # # else: # index += 1 all_match = re.finditer(pattern_money, sentence_text) index = 0 for _match in all_match: if len(_match.group())>0: # print("===",_match.group()) # print(_match.groupdict()) unit = "" entity_text = "" text_beforeMoney = "" filter = "" filter_unit = False for k,v in _match.groupdict().items(): if v!="" and v is not None: if k.split("_")[0]=="money": entity_text = v if k.split("_")[0]=="unit": unit = v if k.split("_")[0]=="text": text_beforeMoney = v if k.split("_")[0]=="filter": filter = v if re.search("filter_unit",k) is not None: filter_unit = True if entity_text.find("元")>=0: unit = "" else: if filter_unit: continue if filter!="": continue index = _match.span()[0]+len(text_beforeMoney) begin_index_temp = index for j in range(len(list_tokenbegin)): if list_tokenbegin[j]==index: begin_index = j break elif list_tokenbegin[j]>index: begin_index = j-1 break index = _match.span()[1] end_index_temp = index #index += len(str(all_match[i][0])) for j in range(begin_index,len(list_tokenbegin)): if list_tokenbegin[j]>=index: end_index = j-1 break entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text) if len(unit)>0: entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) else: entity_text = str(getUnifyMoney(entity_text)) if float(entity_text)<100 or float(entity_text)>100000000000: continue _exists = False for item in list_sentence_entitys: if item.entity_id==entity_id and item.entity_type==entity_type: _exists = True if not _exists: if float(entity_text)>1: list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp)) else: index += 1 # 资金来源提取 2020/12/30 新增 list_moneySource = extract_moneySource(sentence_text) entity_type = "moneysource" for moneySource in list_moneySource: begin_index_temp = moneySource['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = moneySource['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = moneySource['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp)) # 服务期限提取 2020/12/30 新增 list_servicetime = extract_servicetime(sentence_text) entity_type = "serviceTime" for servicetime in list_servicetime: begin_index_temp = servicetime['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = servicetime['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = servicetime['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp)) # 招标方式提取 2020/12/30 新增 list_bidway = extract_bidway(sentence_text) entity_type = "bidway" for bidway in list_bidway: begin_index_temp = bidway['begin_index'] end_index_temp = bidway['end_index'] begin_index = changeIndexFromWordToWords(tokens, begin_index_temp) end_index = changeIndexFromWordToWords(tokens, end_index_temp) entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = bidway['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp)) list_sentence_entitys.sort(key=lambda x:x.begin_index) list_entitys_temp = list_entitys_temp+list_sentence_entitys list_entitys.append(list_entitys_temp) return list_entitys def union_result(codeName,prem): ''' @summary:模型的结果拼成字典 @param: codeName:编号名称模型的结果字典 prem:拿到属性的角色的字典 @return:拼接起来的字典 ''' result = [] assert len(codeName)==len(prem) for item_code,item_prem in zip(codeName,prem): result.append(dict(item_code,**item_prem)) return result def persistenceData(data): ''' @summary:将中间结果保存到数据库-线上生产的时候不需要执行 ''' import psycopg2 conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101") cursor = conn.cursor() for item_index in range(len(data)): item = data[item_index] doc_id = item[0] dic = item[1] code = dic['code'] name = dic['name'] prem = dic['prem'] if len(code)==0: code_insert = "" else: code_insert = ";".join(code) prem_insert = "" for item in prem: for x in item: if isinstance(x, list): if len(x)>0: for x1 in x: prem_insert+="/".join(x1)+"," prem_insert+="$" else: prem_insert+=str(x)+"$" prem_insert+=";" sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')" cursor.execute(sql) conn.commit() conn.close() def persistenceData1(list_entitys,list_sentences): ''' @summary:将中间结果保存到数据库-线上生产的时候不需要执行 ''' import psycopg2 conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101") cursor = conn.cursor() for list_entity in list_entitys: for entity in list_entity: if entity.values is not None: sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")" else: sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")" cursor.execute(sql) for list_sentence in list_sentences: for sentence in list_sentence: str_tokens = "[" for item in sentence.tokens: str_tokens += "'" if item=="'": str_tokens += "''" else: str_tokens += item str_tokens += "'," str_tokens = str_tokens[:-1]+"]" sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")" cursor.execute(sql) conn.commit() conn.close() def _handle(item,result_queue): dochtml = item["dochtml"] docid = item["docid"] list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml")) flag = False if list_innerTable: flag = True for table in list_innerTable: result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)}) def getPredictTable(): filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv" import pandas as pd import json from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue df = pd.read_csv(filename) df_data = {"json_table":[],"docid":[]} _count = 0 _sum = len(df["docid"]) task_queue = Queue() result_queue = Queue() _index = 0 for dochtml,docid in zip(df["dochtmlcon"],df["docid"]): task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None}) _index += 1 mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1) mh.run() while True: try: item = result_queue.get(block=True,timeout=1) df_data["docid"].append(item["docid"]) df_data["json_table"].append(item["json_table"]) except Exception as e: print(e) break df_1 = pd.DataFrame(df_data) df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"]) if __name__=="__main__": ''' import glob for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"): file_txt = str(file).replace("html","txt") with codecs.open(file_txt,"a+",encoding="utf8") as f: f.write("\n================\n") content = codecs.open(file,"r",encoding="utf8").read() f.write(segment(tableToText(BeautifulSoup(content,"lxml")))) ''' # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read() # print(segment(tableToText(BeautifulSoup(content,"lxml")))) getPredictTable()