# -*- coding: utf-8 -*- from bs4 import BeautifulSoup, Comment import copy import sys import os import time import codecs from BiddingKG.dl.ratio.re_ratio import extract_ratio from BiddingKG.dl.table_head.predict import predict sys.setrecursionlimit(1000000) sys.path.append(os.path.abspath("../..")) sys.path.append(os.path.abspath("..")) from BiddingKG.dl.common.Utils import * from BiddingKG.dl.interface.Entitys import * from BiddingKG.dl.interface.predictor import getPredictor from BiddingKG.dl.common.nerUtils import * from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource from BiddingKG.dl.time.re_servicetime import extract_servicetime from BiddingKG.dl.relation_extraction.re_email import extract_email from BiddingKG.dl.bidway.re_bidway import extract_bidway,bidway_integrate from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint from BiddingKG.dl.entityLink.entityLink import * # def tableToText(soup): ''' @param: soup:网页html的soup @return:处理完表格信息的网页text ''' def getTrs(tbody): #获取所有的tr trs = [] objs = tbody.find_all(recursive=False) for obj in objs: if obj.name=="tr": trs.append(obj) if obj.name=="tbody": for tr in obj.find_all("tr",recursive=False): trs.append(tr) return trs def fixSpan(tbody): # 处理colspan, rowspan信息补全问题 #trs = tbody.findChildren('tr', recursive=False) trs = getTrs(tbody) ths_len = 0 ths = list() trs_set = set() #修改为先进行列补全再进行行补全，否则可能会出现表格解析混乱 # 遍历每一个tr for indtr, tr in enumerate(trs): ths_tmp = tr.findChildren('th', recursive=False) #不补全含有表格的tr if len(tr.findChildren('table'))>0: continue if len(ths_tmp) > 0: ths_len = ths_len + len(ths_tmp) for th in ths_tmp: ths.append(th) trs_set.add(tr) # 遍历每行中的element tds = tr.findChildren(recursive=False) for indtd, td in enumerate(tds): # 若有colspan 则补全同一行下一个位置 if 'colspan' in td.attrs: if str(re.sub("[^0-9]","",str(td['colspan'])))!="": col = int(re.sub("[^0-9]","",str(td['colspan']))) if col<100 and len(td.get_text())<1000: td['colspan'] = 1 for i in range(1, col, 1): td.insert_after(copy.copy(td)) for indtr, tr in enumerate(trs): ths_tmp = tr.findChildren('th', recursive=False) #不补全含有表格的tr if len(tr.findChildren('table'))>0: continue if len(ths_tmp) > 0: ths_len = ths_len + len(ths_tmp) for th in ths_tmp: ths.append(th) trs_set.add(tr) # 遍历每行中的element tds = tr.findChildren(recursive=False) for indtd, td in enumerate(tds): # 若有rowspan 则补全下一行同样位置 if 'rowspan' in td.attrs: if str(re.sub("[^0-9]","",str(td['rowspan'])))!="": row = int(re.sub("[^0-9]","",str(td['rowspan']))) td['rowspan'] = 1 for i in range(1, row, 1): # 获取下一行的所有td， 在对应的位置插入 if indtr+i= (indtd) and len(tds1)>0: if indtd > 0: tds1[indtd - 1].insert_after(copy.copy(td)) else: tds1[0].insert_before(copy.copy(td)) elif indtd-2>0 and len(tds1) > 0 and len(tds1) == indtd - 1: # 修正某些表格最后一列没补全 tds1[indtd-2].insert_after(copy.copy(td)) def getTable(tbody): #trs = tbody.findChildren('tr', recursive=False) trs = getTrs(tbody) inner_table = [] for tr in trs: tr_line = [] tds = tr.findChildren(['td','th'], recursive=False) if len(tds)==0: tr_line.append([re.sub('\xa0','',segment(tr,final=False)),0]) # 2021/12/21 修复部分表格没有td 造成数据丢失 for td in tds: tr_line.append([re.sub('\xa0','',segment(td,final=False)),0]) #tr_line.append([td.get_text(),0]) inner_table.append(tr_line) return inner_table #处理表格不对齐的问题 def fixTable(inner_table,fix_value="~~"): maxWidth = 0 for item in inner_table: if len(item)>maxWidth: maxWidth = len(item) for i in range(len(inner_table)): if len(inner_table[i])0 or last_0-len(line)+1<0 or last_1==len(line)-1 or count_1-count_0>=3: return True return False def getsimilarity(line,line1): same_count = 0 for item,item1 in zip(line,line1): if item[1]==item1[1]: same_count += 1 return same_count/len(line) def selfrepair(inner_table,index,dye_set,key_set): ''' @summary: 计算每个节点受到的挤压度来判断是否需要染色 ''' #print("B",inner_table[index]) min_presure = 3 list_dye = [] first = None count = 0 temp_set = set() _index = 0 for item in inner_table[index]: if first is None: first = item[1] if item[0] not in temp_set: count += 1 temp_set.add(item[0]) else: if first == item[1]: if item[0] not in temp_set: temp_set.add(item[0]) count += 1 else: list_dye.append([first,count,_index]) first = item[1] temp_set.add(item[0]) count = 1 _index += 1 list_dye.append([first,count,_index]) if len(list_dye)>1: begin = 0 end = 0 for i in range(len(list_dye)): end = list_dye[i][2] dye_flag = False #首尾要求压力减一 if i==0: if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure-1: dye_flag = True dye_type = list_dye[i+1][0] elif i==len(list_dye)-1: if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure-1: dye_flag = True dye_type = list_dye[i-1][0] else: if list_dye[i][1]>1: if list_dye[i+1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i+1][0] if list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i-1][0] else: if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i+1][0] if list_dye[i+1][1]+list_dye[i-1][1]-list_dye[i][1]+1>=min_presure: dye_flag = True dye_type = list_dye[i-1][0] if dye_flag: for h in range(begin,end): inner_table[index][h][1] = dye_type dye_set.add((inner_table[index][h][0],dye_type)) key_set.add(inner_table[index][h][0]) begin = end #print("E",inner_table[index]) def otherrepair(inner_table,index,dye_set,key_set): list_provide_repair = [] if index==0 and len(inner_table)>1: list_provide_repair.append(index+1) elif index==len(inner_table)-1: list_provide_repair.append(index-1) else: list_provide_repair.append(index+1) list_provide_repair.append(index-1) for provide_index in list_provide_repair: if not repairNeeded(inner_table[provide_index]): same_prob = getsimilarity(inner_table[index], inner_table[provide_index]) if same_prob>=0.8: for i in range(len(inner_table[provide_index])): if inner_table[index][i][1]!=inner_table[provide_index][i][1]: dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1])) key_set.add(inner_table[index][i][0]) inner_table[index][i][1] = inner_table[provide_index][i][1] elif same_prob<=0.2: for i in range(len(inner_table[provide_index])): if inner_table[index][i][1]==inner_table[provide_index][i][1]: dye_set.add((inner_table[index][i][0],inner_table[provide_index][i][1])) key_set.add(inner_table[index][i][0]) inner_table[index][i][1] = 0 if inner_table[provide_index][i][1] ==1 else 1 len_dye_set = len(dye_set) height = len(inner_table) for i in range(height): if repairNeeded(inner_table[i]): selfrepair(inner_table,i,dye_set,key_set) #otherrepair(inner_table,i,dye_set,key_set) for h in range(len(inner_table)): for w in range(len(inner_table[0])): if inner_table[h][w][0] in key_set: for item in dye_set: if inner_table[h][w][0]==item[0]: inner_table[h][w][1] = item[1] #如果两个set长度不相同，则有同一个key被反复染色，将导致无限迭代 if len(dye_set)!=len(key_set): for i in range(height): if repairNeeded(inner_table[i]): selfrepair(inner_table,i,dye_set,key_set) #otherrepair(inner_table,i,dye_set,key_set) return if len(dye_set)==len_dye_set: ''' for i in range(height): if repairNeeded(inner_table[i]): otherrepair(inner_table,i,dye_set,key_set) ''' return repairTable(inner_table, dye_set, key_set) def sliceTable(inner_table,fix_value="~~"): #进行分块 height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) last_head = None last_is_same_value = False for h in range(height): is_all_key = True#是否是全表头行 is_all_value = True#是否是全属性值 is_same_with_lastHead = True#和上一行的结构是否相同 is_same_value=True#一行的item都一样 #is_same_first_item = True#与上一行的第一项是否相同 same_value = inner_table[h][0][0] for w in range(width): if last_head is not None: if inner_table[h-1][w][0]!=fix_value and inner_table[h-1][w][1] == 0: is_all_key = False if inner_table[h][w][0]==1: is_all_value = False if inner_table[h][w][1]!= inner_table[h-1][w][1]: is_same_with_lastHead = False if inner_table[h][w][0]!=fix_value and inner_table[h][w][0]!=same_value: is_same_value = False else: if re.search("\d+",same_value) is not None: is_same_value = False if h>0 and inner_table[h][0][0]!=inner_table[h-1][0][0]: is_same_first_item = False last_head = h if last_is_same_value: last_is_same_value = is_same_value continue if is_same_value: head_list.append(h) last_is_same_value = is_same_value continue if not is_all_key: if not is_same_with_lastHead: head_list.append(h) head_list.append(height) return head_list def setHead_initem(inner_table,pat_head,fix_value="~~",prob_min=0.5): set_item = set() height = len(inner_table) width = len(inner_table[0]) empty_set = set() for i in range(height): for j in range(width): item = inner_table[i][j][0] if item.strip()=="": empty_set.add(item) else: set_item.add(item) list_item = list(set_item) if list_item: x = [] for item in list_item: x.append(getPredictor("form").encode(item)) predict_y = getPredictor("form").predict(np.array(x),type="item") _dict = dict() for item,values in zip(list_item,list(predict_y)): _dict[item] = values[1] # print("##",item,values) #print(_dict) for i in range(height): for j in range(width): item = inner_table[i][j][0] if item not in empty_set: inner_table[i][j][1] = 1 if _dict[item]>prob_min else (1 if re.search(pat_head,item) is not None and len(item)<8 else 0) # print("=====") # for item in inner_table: # print(item) # print("======") repairTable(inner_table) head_list = sliceTable(inner_table) return inner_table,head_list def set_head_model(inner_table): for i in range(len(inner_table)): for j in range(len(inner_table[i])): # 删掉单格前后符号，以免影响表头预测 col = inner_table[i][j][0] col = re.sub("^[^\u4e00-\u9fa5a-zA-Z0-9]+", "", col) col = re.sub("[^\u4e00-\u9fa5a-zA-Z0-9]+$", "", col) inner_table[i][j] = col # 模型预测表头 predict_list = predict(inner_table) # 组合结果 for i in range(len(inner_table)): for j in range(len(inner_table[i])): inner_table[i][j] = [inner_table[i][j], int(predict_list[i][j])] head_list = sliceTable(inner_table) return inner_table, head_list def setHead_incontext(inner_table,pat_head,fix_value="~~",prob_min=0.5): data_x,data_position = getPredictor("form").getModel("context").encode(inner_table) predict_y = getPredictor("form").getModel("context").predict(data_x) for _position,_y in zip(data_position,predict_y): _w = _position[0] _h = _position[1] if _y[1]>prob_min: inner_table[_h][_w][1] = 1 else: inner_table[_h][_w][1] = 0 _item = inner_table[_h][_w][0] if re.search(pat_head,_item) is not None and len(_item)<8: inner_table[_h][_w][1] = 1 # print("=====") # for item in inner_table: # print(item) # print("======") height = len(inner_table) width = len(inner_table[0]) for i in range(height): for j in range(width): if re.search("[:：]$", inner_table[i][j][0]) and len(inner_table[i][j][0])<8: inner_table[i][j][1] = 1 repairTable(inner_table) head_list = sliceTable(inner_table) # print("inner_table:",inner_table) return inner_table,head_list #设置表头 def setHead_inline(inner_table,prob_min=0.64): pad_row = "@@" pad_col = "##" removePadding(inner_table, pad_row, pad_col) pad_pattern = re.compile(pad_row+"|"+pad_col) height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) #行表头 is_head_last = False for i in range(height): is_head = False is_long_value = False #判断是否是全padding值 is_same_value = True same_value = inner_table[i][0][0] for j in range(width): if inner_table[i][j][0]!=same_value and inner_table[i][j][0]!=pad_row: is_same_value = False break #predict is head or not with model temp_item = "" for j in range(width): temp_item += inner_table[i][j][0]+"|" temp_item = re.sub(pad_pattern,"",temp_item) form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line") if form_prob is not None: if form_prob[0][1]>prob_min: is_head = True else: is_head = False #print(temp_item,form_prob) if len(inner_table[i][0][0])>40: is_long_value = True if is_head or is_long_value or is_same_value: #不把连续表头分开 if not is_head_last: head_list.append(i) if is_long_value or is_same_value: head_list.append(i+1) if is_head: for j in range(width): inner_table[i][j][1] = 1 is_head_last = is_head head_list.append(height) #列表头 for i in range(len(head_list)-1): head_begin = head_list[i] head_end = head_list[i+1] #最后一列不设置为列表头 for i in range(width-1): is_head = False #predict is head or not with model temp_item = "" for j in range(head_begin,head_end): temp_item += inner_table[j][i][0]+"|" temp_item = re.sub(pad_pattern,"",temp_item) form_prob = getPredictor("form").predict(formEncoding(temp_item,expand=True),type="line") if form_prob is not None: if form_prob[0][1]>prob_min: is_head = True else: is_head = False if is_head: for j in range(head_begin,head_end): inner_table[j][i][1] = 2 addPadding(inner_table, pad_row, pad_col) return inner_table,head_list #设置表头 def setHead_withRule(inner_table,pattern,pat_value,count): height = len(inner_table) width = len(inner_table[0]) head_list = [] head_list.append(0) #行表头 is_head_last = False for i in range(height): set_match = set() is_head = False is_long_value = False is_same_value = True same_value = inner_table[i][0][0] for j in range(width): if inner_table[i][j][0]!=same_value: is_same_value = False break for j in range(width): if re.search(pat_value,inner_table[i][j][0]) is not None: is_head = False break str_find = re.findall(pattern,inner_table[i][j][0]) if len(str_find)>0: set_match.add(inner_table[i][j][0]) if len(set_match)>=count: is_head = True if len(inner_table[i][0][0])>40: is_long_value = True if is_head or is_long_value or is_same_value: if not is_head_last: head_list.append(i) if is_head: for j in range(width): inner_table[i][j][1] = 1 is_head_last = is_head head_list.append(height) #列表头 for i in range(len(head_list)-1): head_begin = head_list[i] head_end = head_list[i+1] #最后一列不设置为列表头 for i in range(width-1): set_match = set() is_head = False for j in range(head_begin,head_end): if re.search(pat_value,inner_table[j][i][0]) is not None: is_head = False break str_find = re.findall(pattern,inner_table[j][i][0]) if len(str_find)>0: set_match.add(inner_table[j][i][0]) if len(set_match)>=count: is_head = True if is_head: for j in range(head_begin,head_end): inner_table[j][i][1] = 2 return inner_table,head_list #取得表格的处理方向 def getDirect(inner_table,begin,end): ''' column_head = set() row_head = set() widths = len(inner_table[0]) for height in range(begin,end): for width in range(widths): if inner_table[height][width][1] ==1: row_head.add(height) if inner_table[height][width][1] ==2: column_head.add(width) company_pattern = re.compile("公司") if 0 in column_head and begin not in row_head: return "column" if 0 in column_head and begin in row_head: for height in range(begin,end): count = 0 count_flag = True for width_index in range(width): if inner_table[height][width_index][1]==0: if re.search(company_pattern,inner_table[height][width_index][0]) is not None: count += 1 else: count_flag = False if count_flag and count>=2: return "column" return "row" ''' count_row_keys = 0 count_column_keys = 0 width = len(inner_table[0]) if begin=2: return "column" # if count_column_keys>count_row_keys: #2022/2/15 此项不够严谨，造成很多错误，故取消 # return "column" return "row" #根据表格处理方向生成句子， def getTableText(inner_table,head_list,key_direct=False): # packPattern = "(标包|[标包][号段名])" packPattern = "(标包|标的|[标包][号段名]|((项目|物资|设备|场次|标段|标的|产品)(名称)))" # 2020/11/23 大网站规则，补充采购类包名 rankPattern = "(排名|排序|名次|序号|评标结果|评审结果|是否中标|推荐意见)" # 2020/11/23 大网站规则，添加序号为排序 entityPattern = "((候选|([中投]标|报价))(单位|公司|人|供应商))" moneyPattern = "([中投]标|报价)(金额|价)" height = len(inner_table) width = len(inner_table[0]) text = "" for head_i in range(len(head_list)-1): head_begin = head_list[head_i] head_end = head_list[head_i+1] direct = getDirect(inner_table, head_begin, head_end) #若只有一行，则直接按行读取 if head_end-head_begin==1: text_line = "" for i in range(head_begin,head_end): for w in range(len(inner_table[i])): if inner_table[i][w][1]==1: _punctuation = "：" else: _punctuation = "，" #2021/12/15 统一为中文标点，避免 206893924 国际F座1108,1,009,197.49元 if w>0: if inner_table[i][w][0]!= inner_table[i][w-1][0]: text_line += inner_table[i][w][0]+_punctuation else: text_line += inner_table[i][w][0]+_punctuation text_line = text_line+"。" if text_line!="" else text_line text += text_line else: #构建一个共现矩阵 table_occurence = [] for i in range(head_begin,head_end): line_oc = [] for j in range(width): cell = inner_table[i][j] line_oc.append({"text":cell[0],"type":cell[1],"occu_count":0,"left_head":"","top_head":"","left_dis":0,"top_dis":0}) table_occurence.append(line_oc) occu_height = len(table_occurence) occu_width = len(table_occurence[0]) if len(table_occurence)>0 else 0 #为每个属性值寻找表头 for i in range(occu_height): for j in range(occu_width): cell = table_occurence[i][j] #是属性值 if cell["type"]==0 and cell["text"]!="": left_head = "" top_head = "" find_flag = False temp_head = "" for loop_i in range(1,i+1): if not key_direct: key_values = [1,2] else: key_values = [1] if table_occurence[i-loop_i][j]["type"] in key_values: if find_flag: if table_occurence[i-loop_i][j]["text"]!=temp_head: top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head else: top_head = table_occurence[i-loop_i][j]["text"]+"："+top_head find_flag = True temp_head = table_occurence[i-loop_i][j]["text"] table_occurence[i-loop_i][j]["occu_count"] += 1 else: #找到表头后遇到属性值就返回 if find_flag: break cell["top_head"] += top_head find_flag = False temp_head = "" for loop_j in range(1,j+1): if not key_direct: key_values = [1,2] else: key_values = [2] if table_occurence[i][j-loop_j]["type"] in key_values: if find_flag: if table_occurence[i][j-loop_j]["text"]!=temp_head: left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head else: left_head = table_occurence[i][j-loop_j]["text"]+"："+left_head find_flag = True temp_head = table_occurence[i][j-loop_j]["text"] table_occurence[i][j-loop_j]["occu_count"] += 1 else: if find_flag: break cell["left_head"] += left_head if direct=="row": for i in range(occu_height): pack_text = "" rank_text = "" entity_text = "" text_line = "" money_text = "" #在同一句话中重复的可以去掉 text_set = set() for j in range(width): cell = table_occurence[i][j] if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0): cell = table_occurence[i][j] head = (cell["top_head"]+"：") if len(cell["top_head"])>0 else "" if re.search("单报标限总]价|金额|成交报?价|报价", head): head = cell["left_head"] + head else: head += cell["left_head"] if str(head+cell["text"]) in text_set: continue if re.search(packPattern,head) is not None: pack_text += head+cell["text"]+"，" elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif #排名替换为同一种表达 rank_text += head+cell["text"]+"，" #print(rank_text) elif re.search(entityPattern,head) is not None: entity_text += head+cell["text"]+"，" #print(entity_text) else: if re.search(moneyPattern,head) is not None and entity_text!="": money_text += head+cell["text"]+"," else: text_line += head+cell["text"]+"，" text_set.add(str(head+cell["text"])) text += pack_text+rank_text+entity_text+money_text+text_line text = text[:-1]+"。" if len(text)>0 else text else: for j in range(occu_width): pack_text = "" rank_text = "" entity_text = "" text_line = "" text_set = set() for i in range(occu_height): cell = table_occurence[i][j] if cell["type"]==0 or (cell["type"]==1 and cell["occu_count"]==0): cell = table_occurence[i][j] head = (cell["left_head"]+"") if len(cell["left_head"])>0 else "" if re.search("单报标限总]价|金额|成交报?价|报价", head): head = cell["top_head"] + head else: head += cell["top_head"] if str(head+cell["text"]) in text_set: continue if re.search(packPattern,head) is not None: pack_text += head+cell["text"]+"，" elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif #排名替换为同一种表达 rank_text += head+cell["text"]+"，" #print(rank_text) elif re.search(entityPattern,head) is not None and \ re.search('业绩|资格|条件',head)==None and re.search('业绩',cell["text"])==None : #2021/10/19 解决包含业绩的行调到前面问题 entity_text += head+cell["text"]+"，" #print(entity_text) else: text_line += head+cell["text"]+"，" text_set.add(str(head+cell["text"])) text += pack_text+rank_text+entity_text+text_line text = text[:-1]+"。" if len(text)>0 else text # if direct=="row": # for i in range(head_begin,head_end): # pack_text = "" # rank_text = "" # entity_text = "" # text_line = "" # #在同一句话中重复的可以去掉 # text_set = set() # for j in range(width): # cell = inner_table[i][j] # #是属性值 # if cell[1]==0 and cell[0]!="": # head = "" # # find_flag = False # temp_head = "" # for loop_i in range(0,i+1-head_begin): # if not key_direct: # key_values = [1,2] # else: # key_values = [1] # if inner_table[i-loop_i][j][1] in key_values: # if find_flag: # if inner_table[i-loop_i][j][0]!=temp_head: # head = inner_table[i-loop_i][j][0]+":"+head # else: # head = inner_table[i-loop_i][j][0]+":"+head # find_flag = True # temp_head = inner_table[i-loop_i][j][0] # else: # #找到表头后遇到属性值就返回 # if find_flag: # break # # find_flag = False # temp_head = "" # # # # for loop_j in range(1,j+1): # if not key_direct: # key_values = [1,2] # else: # key_values = [2] # if inner_table[i][j-loop_j][1] in key_values: # if find_flag: # if inner_table[i][j-loop_j][0]!=temp_head: # head = inner_table[i][j-loop_j][0]+":"+head # else: # head = inner_table[i][j-loop_j][0]+":"+head # find_flag = True # temp_head = inner_table[i][j-loop_j][0] # else: # if find_flag: # break # # if str(head+inner_table[i][j][0]) in text_set: # continue # if re.search(packPattern,head) is not None: # pack_text += head+inner_table[i][j][0]+"，" # elif re.search(rankPattern,head) is not None: # 2020/11/23 大网站规则发现问题，if 改elif # #排名替换为同一种表达 # rank_text += head+inner_table[i][j][0]+"，" # #print(rank_text) # elif re.search(entityPattern,head) is not None: # entity_text += head+inner_table[i][j][0]+"，" # #print(entity_text) # else: # text_line += head+inner_table[i][j][0]+"，" # text_set.add(str(head+inner_table[i][j][0])) # text += pack_text+rank_text+entity_text+text_line # text = text[:-1]+"。" if len(text)>0 else text # else: # for j in range(width): # # rank_text = "" # entity_text = "" # text_line = "" # text_set = set() # for i in range(head_begin,head_end): # cell = inner_table[i][j] # #是属性值 # if cell[1]==0 and cell[0]!="": # find_flag = False # head = "" # temp_head = "" # # for loop_j in range(1,j+1): # if not key_direct: # key_values = [1,2] # else: # key_values = [2] # if inner_table[i][j-loop_j][1] in key_values: # if find_flag: # if inner_table[i][j-loop_j][0]!=temp_head: # head = inner_table[i][j-loop_j][0]+":"+head # else: # head = inner_table[i][j-loop_j][0]+":"+head # find_flag = True # temp_head = inner_table[i][j-loop_j][0] # else: # if find_flag: # break # find_flag = False # temp_head = "" # for loop_i in range(0,i+1-head_begin): # if not key_direct: # key_values = [1,2] # else: # key_values = [1] # if inner_table[i-loop_i][j][1] in key_values: # if find_flag: # if inner_table[i-loop_i][j][0]!=temp_head: # head = inner_table[i-loop_i][j][0]+":"+head # else: # head = inner_table[i-loop_i][j][0]+":"+head # find_flag = True # temp_head = inner_table[i-loop_i][j][0] # else: # if find_flag: # break # if str(head+inner_table[i][j][0]) in text_set: # continue # if re.search(rankPattern,head) is not None: # rank_text += head+inner_table[i][j][0]+"，" # #print(rank_text) # elif re.search(entityPattern,head) is not None: # entity_text += head+inner_table[i][j][0]+"，" # #print(entity_text) # else: # text_line += head+inner_table[i][j][0]+"，" # text_set.add(str(head+inner_table[i][j][0])) # text += rank_text+entity_text+text_line # text = text[:-1]+"。" if len(text)>0 else text return text def removeFix(inner_table,fix_value="~~"): height = len(inner_table) width = len(inner_table[0]) for h in range(height): for w in range(width): if inner_table[h][w][0]==fix_value: inner_table[h][w][0] = "" def trunTable(tbody,in_attachment): # print(tbody.find('tbody')) # 附件中的表格，排除异常错乱的表格 if in_attachment: if tbody.name=='table': _tbody = tbody.find('tbody') if _tbody is None: _tbody = tbody else: _tbody = tbody _td_len_list = [] for _tr in _tbody.find_all(recursive=False): len_td = len(_tr.find_all(recursive=False)) _td_len_list.append(len_td) if len(list(set(_td_len_list)))>8: return None fixSpan(tbody) inner_table = getTable(tbody) inner_table = fixTable(inner_table) if len(inner_table)>0 and len(inner_table[0])>0: #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3) #inner_table,head_list = setHead_inline(inner_table) # inner_table, head_list = setHead_initem(inner_table,pat_head) inner_table, head_list = set_head_model(inner_table) # inner_table,head_list = setHead_incontext(inner_table,pat_head) # print("table_head", inner_table) # for begin in range(len(head_list[:-1])): # for item in inner_table[head_list[begin]:head_list[begin+1]]: # print(item) # print("====") removeFix(inner_table) # print("----") # print(head_list) # for item in inner_table: # print(item) tbody.string = getTableText(inner_table,head_list) table_max_len = 30000 tbody.string = tbody.string[:table_max_len] #print(tbody.string) tbody.name = "turntable" return inner_table return None pat_head = re.compile('^(名称|序号|项目|标项|工程|品目[一二三四1234]|第[一二三四1234](标段|名|候选人|中标)|包段|标包|分包|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理|制造|企业资质|质量目标|工期目标|(需求|服务|项目|施工|采购|招租|出租|转让|出让|业主|询价|委托|权属|招标|竞得|抽取|承建)(人|方|单位)(名称)?|(供应商|供货商|服务商)(名称)?)$') #pat_head = re.compile('(名称|序号|项目|工程|品目[一二三四1234]|第[一二三四1234](标段|候选人|中标)|包段|包号|货物|单位|数量|价格|报价|金额|总价|单价|[招投中]标|供应商|候选|编号|得分|评委|评分|名次|排名|排序|科室|方式|工期|时间|产品|开始|结束|联系|日期|面积|姓名|证号|备注|级别|地[点址]|类型|代理)') pat_value = re.compile("(\d{2,}.\d{1}|\d+年\d+月|\d{8,}|\d{3,}-\d{6,}|有限[责任]*公司|^\d+$)") list_innerTable = [] # 2022/2/9 删除干扰标签 for tag in soup.find_all('option'): #例子： 216661412 if 'selected' not in tag.attrs: tag.extract() for ul in soup.find_all('ul'): #例子 156439663 多个不同channel 类别的标题 if ul.find_all('li') == ul.findChildren(recursive=False) and len(set(re.findall( '招标公告|中标结果公示|中标候选人公示|招标答疑|开标评标|合同履?约?公示|开标评标|资格评审', ul.get_text(), re.S)))>3: ul.extract() # tbodies = soup.find_all('table') # 遍历表格中的每个tbody tbodies = [] in_attachment = False for _part in soup.find_all(): if _part.name=='table': tbodies.append((_part,in_attachment)) elif _part.name=='div': if 'class' in _part.attrs and "richTextFetch" in _part['class']: in_attachment = True #逆序处理嵌套表格 for tbody_index in range(1,len(tbodies)+1): tbody,_in_attachment = tbodies[len(tbodies)-tbody_index] inner_table = trunTable(tbody,_in_attachment) list_innerTable.append(inner_table) # tbodies = soup.find_all('tbody') # 遍历表格中的每个tbody tbodies = [] in_attachment = False for _part in soup.find_all(): if _part.name == 'tbody': tbodies.append((_part, in_attachment)) elif _part.name == 'div': if 'class' in _part.attrs and "richTextFetch" in _part['class']: in_attachment = True #逆序处理嵌套表格 for tbody_index in range(1,len(tbodies)+1): tbody,_in_attachment = tbodies[len(tbodies)-tbody_index] inner_table = trunTable(tbody,_in_attachment) list_innerTable.append(inner_table) return soup # return list_innerTable re_num = re.compile("[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十]") num_dict = { "一": 1, "二": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9, "十": 10} # 一百以内的中文大写转换为数字 def change2num(text): result_num = -1 # text = text[:6] match = re_num.search(text) if match: _num = match.group() if num_dict.get(_num): return num_dict.get(_num) else: tenths = 1 the_unit = 0 num_split = _num.split("十") if num_dict.get(num_split[0]): tenths = num_dict.get(num_split[0]) if num_dict.get(num_split[1]): the_unit = num_dict.get(num_split[1]) result_num = tenths * 10 + the_unit elif re.search("\d{1,2}",text): _num = re.search("\d{1,2}",text).group() result_num = int(_num) return result_num #大纲分段处理 def get_preprocessed_outline(soup): pattern_0 = re.compile("^(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[、．\.]") pattern_1 = re.compile("^[\(（]?(?:[二三四五六七八九]十[一二三四五六七八九]?|十[一二三四五六七八九]|[一二三四五六七八九十])[\)）]") pattern_2 = re.compile("^\d{1,2}[、．\.](?=[^\d]{1,2}|$)") pattern_3 = re.compile("^[\(（]?\d{1,2}[\)）]") pattern_list = [pattern_0, pattern_1, pattern_2, pattern_3] body = soup.find("body") body_child = body.find_all(recursive=False) deal_part = body # print(body_child[0]['id']) if 'id' in body_child[0].attrs: if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent': deal_part = body_child[0] if len(deal_part.find_all(recursive=False))>2: deal_part = deal_part.parent skip_tag = ['turntable', 'tbody', 'th', 'tr', 'td', 'table','thead','tfoot'] for part in deal_part.find_all(recursive=False): # 查找解析文本的主干部分 is_main_text = False through_text_num = 0 while (not is_main_text and part.find_all(recursive=False)): while len(part.find_all(recursive=False)) == 1 and part.get_text(strip=True) == \ part.find_all(recursive=False)[0].get_text(strip=True): part = part.find_all(recursive=False)[0] max_len = len(part.get_text(strip=True)) is_main_text = True for t_part in part.find_all(recursive=False): if t_part.name not in skip_tag and t_part.get_text(strip=True)!="": through_text_num += 1 if t_part.get_text(strip=True)!="" and len(t_part.get_text(strip=True))/max_len>=0.65: if t_part.name not in skip_tag: is_main_text = False part = t_part break else: while len(t_part.find_all(recursive=False)) == 1 and t_part.get_text(strip=True) == \ t_part.find_all(recursive=False)[0].get_text(strip=True): t_part = t_part.find_all(recursive=False)[0] if through_text_num>2: is_table = True for _t_part in t_part.find_all(recursive=False): if _t_part.name not in skip_tag: is_table = False break if not is_table: is_main_text = False part = t_part break else: is_main_text = False part = t_part break is_find = False for _pattern in pattern_list: last_index = 0 handle_list = [] for _part in part.find_all(recursive=False): if _part.name not in skip_tag and _part.get_text(strip=True) != "": # print('text:', _part.get_text(strip=True)) re_match = re.search(_pattern, _part.get_text(strip=True)) if re_match: outline_index = change2num(re_match.group()) if last_index < outline_index: # _part.insert_before("##split##") handle_list.append(_part) last_index = outline_index if len(handle_list)>1: is_find = True for _part in handle_list: _part.insert_before("##split##") if is_find: break # print(soup) return soup #数据清洗 def segment(soup,final=True): # print("==") # print(soup) # print("====") #segList = ["tr","div","h1", "h2", "h3", "h4", "h5", "h6", "header"] subspaceList = ["td",'a',"span","p"] if soup.name in subspaceList: #判断有值叶子节点数 _count = 0 for child in soup.find_all(recursive=True): if child.get_text().strip()!="" and len(child.find_all())==0: _count += 1 if _count<=1: text = soup.get_text() # 2020/11/24 大网站规则添加 if 'title' in soup.attrs: if '...' in soup.get_text() and soup.get_text().strip()[:-3] in soup.attrs['title']: text = soup.attrs['title'] _list = [] for x in re.split("\s+",text): if x.strip()!="": _list.append(len(x)) if len(_list)>0: _minLength = min(_list) if _minLength>2: _substr = "，" else: _substr = "" else: _substr = "" text = text.replace("\r\n","，").replace("\n","，") text = re.sub("\s+",_substr,text) # text = re.sub("\s+","##space##",text) return text segList = ["title"] commaList = ["div","br","td","p","li"] #commaList = [] spaceList = ["span"] tbodies = soup.find_all('tbody') if len(tbodies) == 0: tbodies = soup.find_all('table') # 递归遍历所有节点,插入符号 for child in soup.find_all(recursive=True): # print(child.name,child.get_text()) if child.name in segList: child.insert_after("。") if child.name in commaList: child.insert_after("，") # if child.name == 'div' and 'class' in child.attrs: # # 添加附件"attachment"标识 # if "richTextFetch" in child['class']: # child.insert_before("##attachment##") # print(child.parent) # if child.name in subspaceList: # child.insert_before("#subs"+str(child.name)+"#") # child.insert_after("#sube"+str(child.name)+"#") # if child.name in spaceList: # child.insert_after(" ") text = str(soup.get_text()) #替换英文冒号为中文冒号 text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text) #替换为中文逗号 text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text) #替换为中文分号 text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text) # 感叹号替换为中文句号 text = re.sub("(?<=[\u4e00-\u9fa5])[!！]|[!！](?=[\u4e00-\u9fa5])","。",text) #替换格式未识别的问号为" " ,update:2021/7/20 text = re.sub("[？\?]{2,}"," ",text) #替换"""为"“",否则导入deepdive出错 # text = text.replace('"',"“").replace("\r","").replace("\n","，") text = text.replace('"',"“").replace("\r","").replace("\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司 # print('==1',text) # text = re.sub("\s{4,}","，",text) # 解决公告中的" "空格替换问题 if re.search("\s{4,}",text): _text = "" for _sent in re.split("。+",text): for _sent2 in re.split('，+',_sent): for _sent3 in re.split("：+",_sent2): for _t in re.split("\s{4,}",_sent3): if len(_t)<3: _text += _t else: _text += "，"+_t _text += "：" _text = _text[:-1] _text += "，" _text = _text[:-1] _text += "。" _text = _text[:-1] text = _text # print('==2',text) #替换标点 #替换连续的标点 if final: text = re.sub("##space##"," ",text) punc_pattern = "(?P[。，；：:,\s]+)" list_punc = re.findall(punc_pattern,text) list_punc.sort(key=lambda x:len(x),reverse=True) for punc_del in list_punc: if len(punc_del)>1: if len(punc_del.strip())>0: if "：" in punc_del.strip(): if "。" in punc_del.strip(): text = re.sub(punc_del, "：。", text) else: text = re.sub(punc_del,"：",text) else: text = re.sub(punc_del,punc_del.strip()[0],text) #2021/12/09 修正由于某些标签后插入符号把原来符号替换 else: text = re.sub(punc_del,"",text) #将连续的中文句号替换为一个 text_split = text.split("。") text_split = [x for x in text_split if len(x)>0] text = "。".join(text_split) # #删除标签中的所有空格 # for subs in subspaceList: # patten = "#subs"+str(subs)+"#(.*?)#sube"+str(subs)+"#" # while(True): # oneMatch = re.search(re.compile(patten),text) # if oneMatch is not None: # _match = oneMatch.group(1) # text = text.replace("#subs"+str(subs)+"#"+_match+"#sube"+str(subs)+"#",_match) # else: # break # text过大报错 LOOP_LEN = 10000 LOOP_BEGIN = 0 _text = "" if len(text)<10000000: while(LOOP_BEGIN3 and len(child_text) <50: # 先判断是否字数少于50，成立加逗号，否则加句号 child.insert_after("，") elif len(child_text) >=50: child.insert_after("。") #if child.name in spaceList: #child.insert_after(" ") text = str(soup.get_text()) text = re.sub("\s{5,}","，",text) text = text.replace('"',"“").replace("\r","").replace("\n","，") #替换"""为"“",否则导入deepdive出错 text = text.replace('"',"“") #text = text.replace('"',"“").replace("\r","").replace("\n","") #删除所有空格 text = re.sub("\s+","#nbsp#",text) text_list = text.split('#nbsp#') new_text = '' for i in range(len(text_list)-1): if text_list[i] == '' or text_list[i][-1] in ['，','。','；','：']: new_text += text_list[i] elif re.findall('([一二三四五六七八九]、)', text_list[i+1][:4]) != []: new_text += text_list[i] + '。' elif re.findall('([0-9]、)', text_list[i+1][:4]) != []: new_text += text_list[i] + '；' elif text_list[i].isdigit() and text_list[i+1].isdigit(): new_text += text_list[i] + ' ' elif text_list[i][-1] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','￥'] or text_list[i+1][0] in ['-',':','(',')','/','（','）','——','年','月','日','时','分','元','万元']: new_text += text_list[i] elif len(text_list[i]) >= 3 and len(text_list[i+1]) >= 3: new_text += text_list[i] + '，' else: new_text += text_list[i] new_text += text_list[-1] text = new_text #替换英文冒号为中文冒号 text = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])","：",text) #替换为中文逗号 text = re.sub("(?<=[\u4e00-\u9fa5]),|,(?=[\u4e00-\u9fa5])","，",text) #替换为中文分号 text = re.sub("(?<=[\u4e00-\u9fa5]);|;(?=[\u4e00-\u9fa5])","；",text) #替换标点 while(True): #替换连续的标点 punc = re.search("，(?P：|。|，|；)\s*",text) if punc is not None: text = re.sub("，"+punc.group("punc")+"\s*",punc.group("punc"),text) punc = re.search("(?P：|。|，|；)\s*，",text) if punc is not None: text = re.sub(punc.group("punc")+"\s*，",punc.group("punc"),text) else: #替换标点之后的空格 punc = re.search("(?P：|。|，|；)\s+",text) if punc is not None: text = re.sub(punc.group("punc")+"\s+",punc.group("punc"),text) else: break #将连续的中文句号替换为一个 text_split = text.split("。") text_split = [x for x in text_split if len(x)>0] text = "。".join(text_split) #替换中文括号为英文括号 text = re.sub("（","(",text) text = re.sub("）",")",text) return text ''' #连续实体合并（弃用） def union_ner(list_ner): result_list = [] union_index = [] union_index_set = set() for i in range(len(list_ner)-1): if len(set([str(list_ner[i][2]),str(list_ner[i+1][2])])&set(["org","company"]))==2: if list_ner[i][1]-list_ner[i+1][0]==1: union_index_set.add(i) union_index_set.add(i+1) union_index.append((i,i+1)) for i in range(len(list_ner)): if i not in union_index_set: result_list.append(list_ner[i]) for item in union_index: #print(str(list_ner[item[0]][3])+str(list_ner[item[1]][3])) result_list.append((list_ner[item[0]][0],list_ner[item[1]][1],'company',str(list_ner[item[0]][3])+str(list_ner[item[1]][3]))) return result_list # def get_preprocessed(articles,useselffool=False): # ''' # @summary:预处理步骤，NLP处理、实体识别 # @param: # articles:待处理的文章list [[id,source,jointime,doc_id,title]] # @return:list of articles,list of each article of sentences,list of each article of entitys # ''' # list_articles = [] # list_sentences = [] # list_entitys = [] # cost_time = dict() # for article in articles: # list_sentences_temp = [] # list_entitys_temp = [] # doc_id = article[0] # sourceContent = article[1] # _send_doc_id = article[3] # _title = article[4] # #表格处理 # key_preprocess = "tableToText" # start_time = time.time() # article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml"))) # # # log(article_processed) # # if key_preprocess not in cost_time: # cost_time[key_preprocess] = 0 # cost_time[key_preprocess] += time.time()-start_time # # #article_processed = article[1] # list_articles.append(Article(doc_id,article_processed,sourceContent,_send_doc_id,_title)) # #nlp处理 # if article_processed is not None and len(article_processed)!=0: # split_patten = "。" # sentences = [] # _begin = 0 # for _iter in re.finditer(split_patten,article_processed): # sentences.append(article_processed[_begin:_iter.span()[1]]) # _begin = _iter.span()[1] # sentences.append(article_processed[_begin:]) # # lemmas = [] # doc_offsets = [] # dep_types = [] # dep_tokens = [] # # time1 = time.time() # # ''' # tokens_all = fool.cut(sentences) # #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all) # #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all) # ner_entitys_all = fool.ner(sentences) # ''' # #限流执行 # key_nerToken = "nerToken" # start_time = time.time() # tokens_all,ner_entitys_all = getTokensAndNers(sentences,useselffool=useselffool) # if key_nerToken not in cost_time: # cost_time[key_nerToken] = 0 # cost_time[key_nerToken] += time.time()-start_time # # # for sentence_index in range(len(sentences)): # # # # list_sentence_entitys = [] # sentence_text = sentences[sentence_index] # tokens = tokens_all[sentence_index] # # list_tokenbegin = [] # begin = 0 # for i in range(0,len(tokens)): # list_tokenbegin.append(begin) # begin += len(str(tokens[i])) # list_tokenbegin.append(begin+1) # #pos_tag = pos_all[sentence_index] # pos_tag = "" # # ner_entitys = ner_entitys_all[sentence_index] # # list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys)) # # #识别package # # # #识别实体 # for ner_entity in ner_entitys: # begin_index_temp = ner_entity[0] # end_index_temp = ner_entity[1] # entity_type = ner_entity[2] # entity_text = ner_entity[3] # # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==begin_index_temp: # begin_index = j # break # elif list_tokenbegin[j]>begin_index_temp: # begin_index = j-1 # break # begin_index_temp += len(str(entity_text)) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=begin_index_temp: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # #去掉标点符号 # entity_text = re.sub("[,，。：]","",entity_text) # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1]-1)) # # # #使用正则识别金额 # entity_type = "money" # # #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*" # # list_money_pattern = {"cn":"(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*", # "key_word":"((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*", # "front_m":"((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*", # "behind_m":"(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*"} # # set_begin = set() # for pattern_key in list_money_pattern.keys(): # pattern = re.compile(list_money_pattern[pattern_key]) # all_match = re.findall(pattern, sentence_text) # index = 0 # for i in range(len(all_match)): # if len(all_match[i][0])>0: # # print("===",all_match[i]) # #print(all_match[i][0]) # unit = "" # entity_text = all_match[i][3] # if pattern_key in ["key_word","front_m"]: # unit = all_match[i][1] # else: # unit = all_match[i][4] # if entity_text.find("元")>=0: # unit = "" # # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征 # # begin_index_temp = index # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==index: # begin_index = j # break # elif list_tokenbegin[j]>index: # begin_index = j-1 # break # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体 # end_index_temp = index # #index += len(str(all_match[i][0])) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=index: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text) # if len(unit)>0: # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) # else: # entity_text = str(getUnifyMoney(entity_text)) # # _exists = False # for item in list_sentence_entitys: # if item.entity_id==entity_id and item.entity_type==entity_type: # _exists = True # if not _exists: # if float(entity_text)>10: # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp)) # # else: # index += 1 # # list_sentence_entitys.sort(key=lambda x:x.begin_index) # list_entitys_temp = list_entitys_temp+list_sentence_entitys # list_sentences.append(list_sentences_temp) # list_entitys.append(list_entitys_temp) # return list_articles,list_sentences,list_entitys,cost_time def get_preprocessed(articles, useselffool=False): ''' @summary:预处理步骤，NLP处理、实体识别 @param: articles:待处理的文章list [[id,source,jointime,doc_id,title]] @return:list of articles,list of each article of sentences,list of each article of entitys ''' cost_time = dict() list_articles = get_preprocessed_article(articles,cost_time) list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time) list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time) calibrateEnterprise(list_articles,list_sentences,list_entitys) return list_articles,list_sentences,list_entitys,list_outlines,cost_time def special_treatment(sourceContent, web_source_no): try: if web_source_no == 'DX000202-1': ser = re.search('中标供应商及中标金额：【((\w{5,20}-[\d,.]+,)+)】', sourceContent) if ser: new = "" l = ser.group(1).split(',') for i in range(len(l)): it = l[i] if '-' in it: role, money = it.split('-') new += '标段%d, 中标供应商: ' % (i + 1) + role + '，中标金额：' + money + '。' sourceContent = sourceContent.replace(ser.group(0), new, 1) elif web_source_no == '00753-14': body = sourceContent.find("body") body_child = body.find_all(recursive=False) pcontent = body if 'id' in body_child[0].attrs: if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent': pcontent = body_child[0] # pcontent = sourceContent.find("div", id="pcontent") pcontent = pcontent.find_all(recursive=False)[0] first_table = None for idx in range(len(pcontent.find_all(recursive=False))): t_part = pcontent.find_all(recursive=False)[idx] if t_part.name != "table": break if idx == 0: first_table = t_part else: for _tr in t_part.find("tbody").find_all(recursive=False): first_table.find("tbody").append(_tr) t_part.clear() elif web_source_no == 'DX008357-11': body = sourceContent.find("body") body_child = body.find_all(recursive=False) pcontent = body if 'id' in body_child[0].attrs: if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent': pcontent = body_child[0] # pcontent = sourceContent.find("div", id="pcontent") pcontent = pcontent.find_all(recursive=False)[0] error_table = [] is_error_table = False for part in pcontent.find_all(recursive=False): if is_error_table: if part.name == "table": error_table.append(part) else: break if part.name == "div" and part.get_text(strip=True) == "中标候选单位：": is_error_table = True first_table = None for idx in range(len(error_table)): t_part = error_table[idx] # if t_part.name != "table": # break if idx == 0: for _tr in t_part.find("tbody").find_all(recursive=False): if _tr.get_text(strip=True) == "": _tr.decompose() first_table = t_part else: for _tr in t_part.find("tbody").find_all(recursive=False): if _tr.get_text(strip=True) != "": first_table.find("tbody").append(_tr) t_part.clear() elif web_source_no == '18021-2': body = sourceContent.find("body") body_child = body.find_all(recursive=False) pcontent = body if 'id' in body_child[0].attrs: if len(body_child) <= 2 and body_child[0]['id'] == 'pcontent': pcontent = body_child[0] # pcontent = sourceContent.find("div", id="pcontent") td = pcontent.find_all("td") for _td in td: if str(_td.string).strip() == "报价金额": _td.string = "单价" elif web_source_no == '13740-2': # “xxx成为成交供应商” re_match = re.search("[^，。]+成为[^，。]*成交供应商", sourceContent) if re_match: sourceContent = sourceContent.replace(re_match.group(), "成交人：" + re_match.group()) elif web_source_no == '03786-10': ser1 = re.search('中标价：([\d,.]+)', sourceContent) ser2 = re.search('合同金额[(（]万元[)）]：([\d,.]+)', sourceContent) if ser1 and ser2: m1 = ser1.group(1).replace(',', '') m2 = ser2.group(1).replace(',', '') if float(m1) < 100000 and (m1.split('.')[0] == m2.split('.')[0] or m2 == '0'): new = '中标价(万元)：' + m1 sourceContent = sourceContent.replace(ser1.group(0), new, 1) elif web_source_no=='00076-4': ser = re.search('主要标的数量：([0-9一]+)\w{,3}，主要标的单价：([\d,.]+)元?，合同金额：(.00)，', sourceContent) if ser: num = ser.group(1).replace('一', '1') try: num = 1 if num == '0' else num unit_price = ser.group(2).replace(',', '') total_price = str(int(num) * float(unit_price)) new = '合同金额：' + total_price sourceContent = sourceContent.replace('合同金额：.00', new, 1) except Exception as e: log('preprocessing.py special_treatment exception') elif web_source_no=='DX000105-2': if re.search("成交公示", sourceContent) and re.search('，投标人：', sourceContent) and re.search('，成交人：', sourceContent)==None: sourceContent = sourceContent.replace('，投标人：', '，成交人：') elif web_source_no in ['04080-3', '04080-4']: ser = re.search('合同金额：([0-9,]+.[0-9]{3,})(.{,4})', sourceContent) if ser and '万' not in ser.group(2): sourceContent = sourceContent.replace('合同金额：', '合同金额(万元)：') elif web_source_no=='03761-3': ser = re.search('中标价，([0-9]+)[.0-9]*%', sourceContent) if ser and int(ser.group(1))>100: sourceContent = sourceContent.replace(ser.group(0), ser.group(0)[:-1]+'元') elif web_source_no=='00695-7': ser = re.search('支付金额：', sourceContent) if ser: sourceContent = sourceContent.replace('支付金额：', '合同金额：') return sourceContent except Exception as e: log('特殊数据源: %s 预处理特别修改抛出异常: %s'%(web_source_no, e)) return sourceContent def article_limit(soup,limit_words=30000): sub_space = re.compile("\s+") def soup_limit(_soup,_count,max_count=30000,max_gap=500): """ :param _soup: soup :param _count: 当前字数 :param max_count: 字数最大限制 :param max_gap: 超过限制后的最大误差 :return: """ _gap = _count - max_count _is_skip = False next_soup = None while len(_soup.find_all(recursive=False)) == 1 and \ _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True): _soup = _soup.find_all(recursive=False)[0] if len(_soup.find_all(recursive=False)) == 0: _soup.string = str(_soup.get_text())[:max_count-_count] _count += len(re.sub(sub_space, "", _soup.string)) _gap = _count - max_count next_soup = None else: for _soup_part in _soup.find_all(recursive=False): if not _is_skip: _count += len(re.sub(sub_space, "", _soup_part.get_text())) if _count >= max_count: _gap = _count - max_count if _gap <= max_gap: _is_skip = True else: _is_skip = True next_soup = _soup_part _count -= len(re.sub(sub_space, "", _soup_part.get_text())) continue else: _soup_part.decompose() return _count,_gap,next_soup text_count = 0 have_attachment = False attachment_part = None for child in soup.find_all(recursive=True): if child.name == 'div' and 'class' in child.attrs: if "richTextFetch" in child['class']: child.insert_before("##attachment##。") # 句号分开，避免项目名称等提取 attachment_part = child have_attachment = True break if not have_attachment: # 无附件 if len(re.sub(sub_space, "", soup.get_text())) > limit_words: text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500) while n_soup: text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500) else: # 有附件 _text = re.sub(sub_space, "", soup.get_text()) _text_split = _text.split("##attachment##") if len(_text_split[0])>limit_words: main_soup = attachment_part.parent main_text = main_soup.find_all(recursive=False)[0] text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500) while n_soup: text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500) if len(_text_split[1])>limit_words: # attachment_html纯文本，无子结构 if len(attachment_part.find_all(recursive=False))==0: attachment_part.string = str(attachment_part.get_text())[:limit_words] else: attachment_text_nums = 0 attachment_skip = False for part in attachment_part.find_all(recursive=False): if not attachment_skip: last_attachment_text_nums = attachment_text_nums attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text())) if attachment_text_nums>=limit_words: part.string = str(part.get_text())[:limit_words-last_attachment_text_nums] attachment_skip = True else: part.decompose() return soup def attachment_filelink(soup): have_attachment = False attachment_part = None for child in soup.find_all(recursive=True): if child.name == 'div' and 'class' in child.attrs: if "richTextFetch" in child['class']: attachment_part = child have_attachment = True break if not have_attachment: return soup else: # 附件类型：图片、表格 attachment_type = re.compile("\.(?:png|jpg|jpeg|tif|bmp|xlsx|xls)$") attachment_dict = dict() for _attachment in attachment_part.find_all(recursive=False): if _attachment.name == 'div' and 'filemd5' in _attachment.attrs: # print('filemd5',_attachment['filemd5']) attachment_dict[_attachment['filemd5']] = _attachment # print(attachment_dict) for child in soup.find_all(recursive=True): if child.name == 'div' and 'class' in child.attrs: if "richTextFetch" in child['class']: break if "filelink" in child.attrs and child['filelink'] in attachment_dict: if re.search(attachment_type,str(child.string).strip()) or \ ('original' in child.attrs and re.search(attachment_type,str(child['original']).strip())): # 附件插入正文标识 child.insert_before("。##attachment_begin##") child.insert_after("。##attachment_end##") child.replace_with(attachment_dict[child['filelink']]) # print('格式化输出',soup.prettify()) return soup def get_preprocessed_article(articles,cost_time = dict(),useselffool=True): ''' :param articles: 待处理的article source html :param useselffool: 是否使用selffool :return: list_articles ''' list_articles = [] for article in articles: doc_id = article[0] sourceContent = article[1] sourceContent = re.sub("|||","",sourceContent) sourceContent = sourceContent.replace('
', '
') sourceContent = re.sub("
(\s{0,}
)+","
",sourceContent) # for br_match in re.findall("[^>]+?
",sourceContent): # _new = re.sub("
","",br_match) # #
标签替换为

标签 # if not re.search("^\s+$",_new): # _new = '

'+_new + '

' # # print(br_match,_new) # sourceContent = sourceContent.replace(br_match,_new,1) _send_doc_id = article[3] _title = article[4] page_time = article[5] web_source_no = article[6] '''特别数据源对 html 做特别修改''' if web_source_no in ['DX000202-1']: sourceContent = special_treatment(sourceContent, web_source_no) #表格处理 key_preprocess = "tableToText" start_time = time.time() # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml")) article_processed = BeautifulSoup(sourceContent,"lxml") '''特别数据源对 BeautifulSoup(html) 做特别修改''' if web_source_no in ["00753-14","DX008357-11","18021-2"]: article_processed = special_treatment(article_processed, web_source_no) for _soup in article_processed.descendants: # 识别无标签文本，添加标签 if not _soup.name and not _soup.parent.string and _soup.string.strip()!="": # print(_soup.parent.string,_soup.string.strip()) _soup.wrap(article_processed.new_tag("span")) # print(article_processed) # 正文和附件内容限制字数30000 article_processed = article_limit(article_processed, limit_words=30000) # 把每个附件识别对应的html放回原来出现的位置 article_processed = attachment_filelink(article_processed) article_processed = get_preprocessed_outline(article_processed) # print('article_processed') article_processed = tableToText(article_processed) article_processed = segment(article_processed) article_processed = article_processed.replace('(', '（').replace(')', '）') #2022/8/10 统一为中文括号 article_processed = article_processed.replace('．','.') # 2021/12/01 修正OCR识别PDF小数点错误问题 article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改 article_processed = article_processed.replace('成交工程价款', '成交工程价') # 2021/12/21 修正为中标价 article_processed = re.sub('任务(?=编号[:：])', '项目',article_processed) # 2022/08/10 修正为项目编号 article_processed = article_processed.replace('招标（建设）单位', '招标单位') #2022/8/10 修正预测不到表达 article_processed = re.sub('(招标|采购)人(概况|信息)[，。]', '采购人信息：', article_processed) # 2022/8/10统一表达 # 修复OCR金额中“，”、“。”识别错误 article_processed_list = article_processed.split("##attachment##") if len(article_processed_list)>1: attachment_text = article_processed_list[1] for _match in re.finditer("\d。\d{2}",attachment_text): _match_text = _match.group() attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1) for _match in re.finditer("(\d，\d{3})[,，.]",attachment_text): _match_text = _match.group() attachment_text = attachment_text.replace(_match_text,_match_text.replace("，",","),1) article_processed_list[1] = attachment_text article_processed = "##attachment##".join(article_processed_list) '''特别数据源对 预处理后文本 做特别修改''' if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2"]: article_processed = special_treatment(article_processed, web_source_no) # 提取bidway list_bidway = extract_bidway(article_processed, _title) if list_bidway: bidway = list_bidway[0].get("body") # bidway名称统一规范 bidway = bidway_integrate(bidway) else: bidway = "" # 修正被"，"逗号分隔的时间 repair_time = re.compile("[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d|" "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[:时点]，?[0-6]\d分?|" "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[时点]|" "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]|" "[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d" ) for _time in set(re.findall(repair_time,article_processed)): if re.search("，",_time): _time2 = re.sub("，", "", _time) item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time2) if item: _time2 = _time2.replace(item.group(),item.group() + " ") article_processed = article_processed.replace(_time, _time2) else: item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time) if item: _time2 = _time.replace(item.group(),item.group() + " ") article_processed = article_processed.replace(_time, _time2) # print('re_rtime',re.findall(repair_time,article_processed)) # log(article_processed) if key_preprocess not in cost_time: cost_time[key_preprocess] = 0 cost_time[key_preprocess] += round(time.time()-start_time,2) #article_processed = article[1] _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title, bidway=bidway) _article.fingerprint = getFingerprint(_title+sourceContent) _article.page_time = page_time list_articles.append(_article) return list_articles def get_preprocessed_sentences(list_articles,useselffool=True,cost_time=dict()): ''' :param list_articles: 经过预处理的article text :return: list_sentences ''' list_sentences = [] list_outlines = [] for article in list_articles: list_sentences_temp = [] list_entitys_temp = [] doc_id = article.id _send_doc_id = article.doc_id _title = article.title #表格处理 key_preprocess = "tableToText" start_time = time.time() article_processed = article.content if len(_title)<100 and _title not in article_processed: # 把标题放到正文 article_processed = _title + article_processed attachment_begin_index = -1 if key_preprocess not in cost_time: cost_time[key_preprocess] = 0 cost_time[key_preprocess] += time.time()-start_time #nlp处理 if article_processed is not None and len(article_processed)!=0: split_patten = "。" sentences = [] _begin = 0 sentences_set = set() for _iter in re.finditer(split_patten,article_processed): _sen = article_processed[_begin:_iter.span()[1]] if len(_sen)>0 and _sen not in sentences_set: # 标识在附件里的句子 if re.search("##attachment##",_sen): attachment_begin_index = len(sentences) # _sen = re.sub("##attachment##","",_sen) sentences.append(_sen) sentences_set.add(_sen) _begin = _iter.span()[1] _sen = article_processed[_begin:] if re.search("##attachment##", _sen): # _sen = re.sub("##attachment##", "", _sen) attachment_begin_index = len(sentences) if len(_sen)>0 and _sen not in sentences_set: sentences.append(_sen) sentences_set.add(_sen) # 解析outline大纲分段 outline_list = [] if re.search("##split##",article.content): temp_sentences = [] last_sentence_index = (-1,-1) outline_index = 0 for sentence_index in range(len(sentences)): sentence_text = sentences[sentence_index] for _ in re.findall("##split##", sentence_text): _match = re.search("##split##", sentence_text) if last_sentence_index[0] > -1: sentence_begin_index,wordOffset_begin = last_sentence_index sentence_end_index = sentence_index wordOffset_end = _match.start() if sentence_begin_index=attachment_begin_index: outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,attachment_begin_index-1,wordOffset_begin,len(sentences[attachment_begin_index-1]))) else: outline_list.append(Outline(doc_id,outline_index,'',sentence_begin_index,sentence_end_index,wordOffset_begin,wordOffset_end)) outline_index += 1 sentence_text = re.sub("##split##", "", sentence_text,count=1) last_sentence_index = (sentence_index,_match.start()) temp_sentences.append(sentence_text) if attachment_begin_index>-1 and last_sentence_index[0]= attachment_begin_index and attachment_begin_index!=-1: in_attachment = True tokens = tokens_all[sentence_index] #pos_tag = pos_all[sentence_index] pos_tag = "" ner_entitys = "" list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=sentence_index,sentence_text=sentence_text,tokens=tokens,pos_tags=pos_tag,ner_tags=ner_entitys,in_attachment=in_attachment)) if len(list_sentences_temp)==0: list_sentences_temp.append(Sentences(doc_id=doc_id,sentence_index=0,sentence_text="sentence_text",tokens=[],pos_tags=[],ner_tags="")) list_sentences.append(list_sentences_temp) list_outlines.append(outline_list) return list_sentences,list_outlines def get_preprocessed_entitys(list_sentences,useselffool=True,cost_time=dict()): ''' :param list_sentences:分局情况 :param cost_time: :return: list_entitys ''' list_entitys = [] for list_sentence in list_sentences: sentences = [] list_entitys_temp = [] for _sentence in list_sentence: sentences.append(_sentence.sentence_text) lemmas = [] doc_offsets = [] dep_types = [] dep_tokens = [] time1 = time.time() ''' tokens_all = fool.cut(sentences) #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all) #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all) ner_entitys_all = fool.ner(sentences) ''' #限流执行 key_nerToken = "nerToken" start_time = time.time() found_yeji = 0 # 2021/8/6 增加判断是否正文包含评标结果 及类似业绩判断用于过滤后面的金额 # found_pingbiao = False ner_entitys_all = getNers(sentences,useselffool=useselffool) if key_nerToken not in cost_time: cost_time[key_nerToken] = 0 cost_time[key_nerToken] += round(time.time()-start_time,2) company_dict = set() company_index = dict((i,set()) for i in range(len(list_sentence))) for sentence_index in range(len(list_sentence)): list_sentence_entitys = [] sentence_text = list_sentence[sentence_index].sentence_text tokens = list_sentence[sentence_index].tokens doc_id = list_sentence[sentence_index].doc_id in_attachment = list_sentence[sentence_index].in_attachment list_tokenbegin = [] begin = 0 for i in range(0,len(tokens)): list_tokenbegin.append(begin) begin += len(str(tokens[i])) list_tokenbegin.append(begin+1) #pos_tag = pos_all[sentence_index] pos_tag = "" ner_entitys = ner_entitys_all[sentence_index] '''正则识别角色实体 经营部|经销部|电脑部|服务部|复印部|印刷部|彩印部|装饰部|修理部|汽修部|修理店|零售店|设计店|服务店|家具店|专卖店|分店|文具行|商行|印刷厂|修理厂|维修中心|修配中心|养护中心|服务中心|会馆|文化馆|超市|门市|商场|家具城|印刷社|经销处''' for it in re.finditer( '(?P(((单一来源|中标|中选|中价|成交)(供应商|供货商|服务商|候选人|单位|人))|(供应商|供货商|服务商|候选人))(名称)?[为:：]+)(?P([^，。、；《:：]{5,20})(厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处))[，。]', sentence_text): for k, v in it.groupdict().items(): if k == 'text_key_word': keyword = v if k == 'text': entity = v b = it.start() + len(keyword) e = it.end() - 1 if (b, e, 'location', entity) in ner_entitys: ner_entitys.remove((b, e, 'location', entity)) ner_entitys.append((b, e, 'company', entity)) elif (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys: ner_entitys.append((b, e, 'company', entity)) for it in re.finditer( '(?P((建设|招租|招标|采购)(单位|人)|业主)(名称)?[为:：]+)(?P\w{2,4}[省市县区镇]([^，。、；《]{2,20})(管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园))[，。]', sentence_text): for k, v in it.groupdict().items(): if k == 'text_key_word': keyword = v if k == 'text': entity = v b = it.start() + len(keyword) e = it.end() - 1 if (b, e, 'location', entity) in ner_entitys: ner_entitys.remove((b, e, 'location', entity)) ner_entitys.append((b, e, 'org', entity)) if (b, e, 'org', entity) not in ner_entitys and (b, e, 'company', entity) not in ner_entitys: ner_entitys.append((b, e, 'org', entity)) for ner_entity in ner_entitys: if ner_entity[2] in ['company','org']: company_dict.add((ner_entity[2],ner_entity[3])) company_index[sentence_index].add((ner_entity[0],ner_entity[1])) #识别package #识别实体 for ner_entity in ner_entitys: begin_index_temp = ner_entity[0] end_index_temp = ner_entity[1] entity_type = ner_entity[2] entity_text = ner_entity[3] if entity_type in ["org","company"] and not isLegalEnterprise(entity_text): continue for j in range(len(list_tokenbegin)): if list_tokenbegin[j]==begin_index_temp: begin_index = j break elif list_tokenbegin[j]>begin_index_temp: begin_index = j-1 break begin_index_temp += len(str(entity_text)) for j in range(begin_index,len(list_tokenbegin)): if list_tokenbegin[j]>=begin_index_temp: end_index = j-1 break entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) #去掉标点符号 entity_text = re.sub("[,，。：!&@$\*]","",entity_text) entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment)) # 标记文章末尾的"发布人”、“发布时间”实体 if sentence_index==len(list_sentence)-1: if len(list_sentence_entitys[-2:])>2: second2last = list_sentence_entitys[-2] last = list_sentence_entitys[-1] if (second2last.entity_type in ["company",'org'] and last.entity_type=="time") or ( second2last.entity_type=="time" and last.entity_type in ["company",'org']): if last.wordOffset_begin - second2last.wordOffset_end < 6 and len(sentence_text) - last.wordOffset_end<6: last.is_tail = True second2last.is_tail = True #使用正则识别金额 entity_type = "money" #money_patten_str = "(([1-9][\d,，]*(?:\.\d+)?[百千万亿]?[\(\)（）元整]+)|([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})|(?:[￥¥]+，?|报价|标价)[（\(]?([万])?元?[）\)]?[:：]?.{,7}?([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)|([1-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]?)[\(（]?([万元]{1,2}))*" # list_money_pattern = {"cn":"(()()(?P[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())", # "key_word":"((?P(?:[￥¥]+，?|[单报标限]价|金额|价格|标的基本情况|CNY|成交结果：)(?:[,（\(]*\s*(?P[万元]*(?P[台个只]*))\s*[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,8}?))(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?(?P[%])*\s*(?P[万元]*(?P[台个只]*))\s*[）\)]?))", # "front_m":"((?P(?:[（\(]?\s*(?P[万元]+)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千万亿元]*)())", # "behind_m":"(()()(?P[0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?(?P[万元]+(?P[台个只]*))[\)）]?)"} list_money_pattern = {"cn":"(()()(?P[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())", "key_word": "((?P(?:[￥¥]+，?|[单报标限总]价|金额|成交报?价|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果|成交额|中标额)(?:[,，（\(]*\s*(人民币)?(?P[万亿]?元?(?P[台个只吨]*))\s*(/?费率)?(人民币)?[）\)]?)\s*[，,:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]{,1})(?:[（\(]?(?P[%])*\s*(单位[:：])?(?P[万亿]?元?(?P[台只吨斤棵株页亩方条天]*))\s*[）\)]?))", "front_m":"((?P(?:[（\(]?\s*(?P[万亿]?元)\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元]{,7}?))(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]*)())", "behind_m":"(()()(?P[0-9][\d,]*(?:\.\d+)?(?:，?)[百千]*)(人民币)?[\(（]?(?P[万亿]?元(?P[台个只吨斤棵株页亩方条米]*))[\)）]?)"} # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。 pattern_money = re.compile("%s|%s|%s|%s"%(list_money_pattern["cn"],list_money_pattern["key_word"],list_money_pattern["behind_m"],list_money_pattern["front_m"])) set_begin = set() # for pattern_key in list_money_pattern.keys(): # for pattern_key in ["cn","key_word","behind_m","front_m"]: # # pattern = re.compile(list_money_pattern[pattern_key]) # pattern = re.compile("(()()([零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,})())*|((?:[￥¥]+，?|[报标限]价|金额)(?:[（\(]?\s*([万元]*)\s*[）\)]?)\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)(?:[（\(]?\s*([万元]*)\s*[）\)]?))*|(()()([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿]*)[\(（]?([万元]+)[\)）]?)*|((?:[（\(]?\s*([万元]+)\s*[）\)])\s*[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分]{,7}?)([0-9][\d,，]*(?:\.\d+)?(?:，?)[百千万亿元]*)())*") # all_match = re.findall(pattern, sentence_text) # index = 0 # for i in range(len(all_match)): # if len(all_match[i][0])>0: # print("===",all_match[i]) # #print(all_match[i][0]) # unit = "" # entity_text = all_match[i][3] # if pattern_key in ["key_word","front_m"]: # unit = all_match[i][1] # if pattern_key=="key_word": # if all_match[i][1]=="" and all_match[i][4]!="": # unit = all_match[i][4] # else: # unit = all_match[i][4] # if entity_text.find("元")>=0: # unit = "" # # index += len(all_match[i][0])-len(entity_text)-len(all_match[i][4])#-len(all_match[i][1])-len(all_match[i][2])#整个提出来的作为实体->数字部分作为整体，否则会丢失特征 # begin_index_temp = index # for j in range(len(list_tokenbegin)): # if list_tokenbegin[j]==index: # begin_index = j # break # elif list_tokenbegin[j]>index: # begin_index = j-1 # break # index += len(str(entity_text))+len(all_match[i][4])#+len(all_match[i][2])+len(all_match[i][1])#整个提出来的作为实体 # end_index_temp = index # #index += len(str(all_match[i][0])) # for j in range(begin_index,len(list_tokenbegin)): # if list_tokenbegin[j]>=index: # end_index = j-1 # break # entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) # # entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]","",entity_text) # if len(unit)>0: # entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) # else: # entity_text = str(getUnifyMoney(entity_text)) # # _exists = False # for item in list_sentence_entitys: # if item.entity_id==entity_id and item.entity_type==entity_type: # _exists = True # if not _exists: # if float(entity_text)>1: # list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp)) # # else: # index += 1 # if re.search('评标结果|候选人公示', sentence_text): # found_pingbiao = True if re.search('业绩', sentence_text): found_yeji += 1 if found_yeji >= 2: # 过滤掉业绩后面的所有金额 all_match = [] else: all_match = re.finditer(pattern_money, sentence_text) index = 0 for _match in all_match: if len(_match.group())>0: # print("===",_match.group()) # # print(_match.groupdict()) notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位 unit = "" entity_text = "" text_beforeMoney = "" filter = "" filter_unit = False notSure = False if re.search('业绩', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额 # print('金额在业绩后面: ', _match.group(0)) found_yeji += 1 break for k,v in _match.groupdict().items(): if v!="" and v is not None: if k=='text_key_word': notSure = True if k.split("_")[0]=="money": entity_text = v if k.split("_")[0]=="unit": if v=='万元' or unit=="": # 处理 预算金额(元)：160万元 这种出现前后单位不一致情况 unit = v if k.split("_")[0]=="text": text_beforeMoney = v if k.split("_")[0]=="filter": filter = v if re.search("filter_unit",k) is not None: filter_unit = True # print(_match.group()) # print(entity_text,unit,text_beforeMoney,filter,filter_unit) if re.search('(^\d{2,},\d{4,}万?$)|(^\d{2,},\d{2}万?$)', entity_text.strip()): # 2021/7/19 修正OCR识别小数点为逗号 if re.search('[幢栋号楼层]', sentence_text[max(0, _match.span()[0]-2):_match.span()[0]]): entity_text = re.sub('\d+,', '', entity_text) else: entity_text = entity_text.replace(',', '.') # print(' 修正OCR识别小数点为逗号') if entity_text.find("元")>=0: unit = "" if unit == "": #2021/7/21 有明显金额特征的补充单位，避免被过滤 if ('￥' in text_beforeMoney or '¥' in text_beforeMoney): unit = '元' # print('明显金额特征补充单位 元') elif re.search('[单报标限]价|金额|价格|(监理|设计|勘察)(服务)?费[:：为]+$', text_beforeMoney.strip()) and \ re.search('\d{5,}',entity_text) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}',entity_text)==None: unit = '元' # print('明显金额特征补充单位 元') elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)',entity_text): unit = '元' # print('明显金额特征补充单位 元') if unit.find("万") >= 0 and entity_text.find("万") >= 0: #2021/7/19修改为金额文本有万，不计算单位 # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit) unit = "元" if re.search('.*万元万元', entity_text): #2021/7/19 修正两个万元 # print(' 修正两个万元',entity_text) entity_text = entity_text.replace('万元万元','万元') else: if filter_unit: continue if filter!="": continue index = _match.span()[0]+len(text_beforeMoney) begin_index_temp = index for j in range(len(list_tokenbegin)): if list_tokenbegin[j]==index: begin_index = j break elif list_tokenbegin[j]>index: begin_index = j-1 break index = _match.span()[1] end_index_temp = index #index += len(str(all_match[i][0])) for j in range(begin_index,len(list_tokenbegin)): if list_tokenbegin[j]>=index: end_index = j-1 break entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",entity_text) # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney) if re.search('总投资|投资总额|总预算|总概算|投资规模', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/8/5过滤掉总投资金额 # print('总投资金额: ', _match.group(0)) notes = '总投资' elif re.search('投资', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额 notes = '投资' elif re.search('工程造价', sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额 notes = '工程造价' elif (re.search('保证金', sentence_text[max(0, _match.span()[0] - 5):_match.span()[1]]) or re.search('保证金的?(缴纳)?(金额|金\?|额|\?)?[\(（]*(万?元|为?人民币|大写|调整|变更|已?修改|更改|更正)?[\)）]*[:：为]', sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]) or re.search('保证金由[\d.,]+.{,3}(变更|修改|更改|更正|调整?)为', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]])): notes = '保证金' # print('保证金信息：', sentence_text[max(0, _match.span()[0] - 15):_match.span()[1]]) elif re.search('成本(警戒|预警)(线|价|值)[^0-9元]{,10}', sentence_text[max(0, _match.span()[0] - 10):_match.span()[0]]): notes = '成本警戒线' elif re.search('(监理|设计|勘察)(服务)?费(报价)?[约为：]', sentence_text[_match.span()[0]:_match.span()[1]]): cost_re = re.search('(监理|设计|勘察)(服务)?费', sentence_text[_match.span()[0]:_match.span()[1]]) notes = cost_re.group(1) elif re.search('单价|总金额', sentence_text[_match.span()[0]:_match.span()[1]]): notes = '单价' elif re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆]', entity_text) != None: notes = '大写' if entity_text[0] == "拾": # 2021/12/16 修正大写金额省略了数字转换错误问题 entity_text = "壹"+entity_text # print("补充备注：notes = 大写") if len(unit)>0: if unit.find('万')>=0 and len(entity_text.split('.')[0])>=8: # 2021/7/19 修正万元金额过大的情况 # print('修正单位万元金额过大的情况 金额：', entity_text, '单位:', unit) entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(unit[0])/10000) unit = '元' # 修正金额后单位 重置为元 else: # print('str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])):') entity_text = str(getUnifyMoney(entity_text)*getMultipleFactor(unit[0])) else: if entity_text.find('万')>=0 and entity_text.split('.')[0].isdigit() and len(entity_text.split('.')[0])>=8: entity_text = str(getUnifyMoney(entity_text)/10000) # print('修正金额字段含万 过大的情况') else: entity_text = str(getUnifyMoney(entity_text)) if float(entity_text)>100000000000 or float(entity_text)<100: # float(entity_text)<100 or 2022/3/4 取消最小金额限制 # print('过滤掉金额：float(entity_text)<100 or float(entity_text)>100000000000', entity_text, unit) continue if notSure and unit=="" and float(entity_text)>100*10000: # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit) continue _exists = False for item in list_sentence_entitys: if item.entity_id==entity_id and item.entity_type==entity_type: _exists = True if (begin_index >=item.begin_index and begin_index<=item.end_index) or (end_index>=item.begin_index and end_index<=item.end_index): _exists = True if not _exists: if float(entity_text)>1: list_sentence_entitys.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,begin_index_temp,end_index_temp,in_attachment=in_attachment)) list_sentence_entitys[-1].notes = notes # 2021/7/20 新增金额备注 list_sentence_entitys[-1].money_unit = unit # 2021/7/20 新增金额备注 # print('预处理中的 金额:%s, 单位：%s'%(entity_text,unit)) # print(entity_text,unit,notes) else: index += 1 # "联系人"正则补充提取 2021/11/15 新增 list_person_text = [entity.entity_text for entity in list_sentence_entitys if entity.entity_type=='person'] error_text = ['交易','机构','教育','项目','公司','中标','开标','截标','监督','政府','国家','中国','技术','投标','传真','网址','电子邮', '联系','联系电','联系地','采购代','邮政编','邮政','电话','手机','手机号','联系人','地址','地点','邮箱','邮编','联系方','招标','招标人','代理', '代理人','采购','附件','注意','登录','报名','踏勘',"测试"] list_person_text = set(list_person_text + error_text) re_person = re.compile("联系人[:：]([\u4e00-\u9fa5]工)|" "联系人[:：]([\u4e00-\u9fa5]{2,3})(?=联系)|" "联系人[:：]([\u4e00-\u9fa5]{2,3})") list_person = [] if not in_attachment: for match_result in re_person.finditer(sentence_text): match_text = match_result.group() entity_text = match_text[4:] wordOffset_begin = match_result.start() + 4 wordOffset_end = match_result.end() # print(text[wordOffset_begin:wordOffset_end]) # 排除一些不为人名的实体 if re.search("^[\u4e00-\u9fa5]{7,}([，。]|$)",sentence_text[wordOffset_begin:wordOffset_begin+20]): continue if entity_text not in list_person_text and entity_text[:2] not in list_person_text: _person = dict() _person['body'] = entity_text _person['begin_index'] = wordOffset_begin _person['end_index'] = wordOffset_end list_person.append(_person) entity_type = "person" for person in list_person: begin_index_temp = person['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = person['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = person['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp,in_attachment=in_attachment)) # 资金来源提取 2020/12/30 新增 list_moneySource = extract_moneySource(sentence_text) entity_type = "moneysource" for moneySource in list_moneySource: begin_index_temp = moneySource['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = moneySource['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = moneySource['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp,in_attachment=in_attachment)) # 电子邮箱提取 2021/11/04 新增 list_email = extract_email(sentence_text) entity_type = "email" # 电子邮箱 for email in list_email: begin_index_temp = email['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = email['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = email['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp,in_attachment=in_attachment)) # 服务期限提取 2020/12/30 新增 list_servicetime = extract_servicetime(sentence_text) entity_type = "serviceTime" for servicetime in list_servicetime: begin_index_temp = servicetime['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = servicetime['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = servicetime['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp,in_attachment=in_attachment, prob=servicetime["prob"])) # 招标方式提取 2020/12/30 新增 # list_bidway = extract_bidway(sentence_text, ) # entity_type = "bidway" # for bidway in list_bidway: # begin_index_temp = bidway['begin_index'] # end_index_temp = bidway['end_index'] # begin_index = changeIndexFromWordToWords(tokens, begin_index_temp) # end_index = changeIndexFromWordToWords(tokens, end_index_temp) # if begin_index is None or end_index is None: # continue # print(begin_index_temp,end_index_temp,begin_index,end_index) # entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) # entity_text = bidway['body'] # list_sentence_entitys.append( # Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, # begin_index_temp, end_index_temp)) # 2021/12/29 新增比率提取 list_ratio = extract_ratio(sentence_text) entity_type = "ratio" for ratio in list_ratio: # print("ratio", ratio) begin_index_temp = ratio['begin_index'] for j in range(len(list_tokenbegin)): if list_tokenbegin[j] == begin_index_temp: begin_index = j break elif list_tokenbegin[j] > begin_index_temp: begin_index = j - 1 break index = ratio['end_index'] end_index_temp = index for j in range(begin_index, len(list_tokenbegin)): if list_tokenbegin[j] >= index: end_index = j - 1 break entity_id = "%s_%d_%d_%d" % (doc_id, sentence_index, begin_index, end_index) entity_text = ratio['body'] list_sentence_entitys.append( Entity(doc_id, entity_id, entity_text, entity_type, sentence_index, begin_index, end_index, begin_index_temp, end_index_temp,in_attachment=in_attachment)) list_sentence_entitys.sort(key=lambda x:x.begin_index) list_entitys_temp = list_entitys_temp+list_sentence_entitys # 补充ner模型未识别全的company/org实体 for sentence_index in range(len(list_sentence)): sentence_text = list_sentence[sentence_index].sentence_text tokens = list_sentence[sentence_index].tokens doc_id = list_sentence[sentence_index].doc_id in_attachment = list_sentence[sentence_index].in_attachment list_tokenbegin = [] begin = 0 for i in range(0, len(tokens)): list_tokenbegin.append(begin) begin += len(str(tokens[i])) list_tokenbegin.append(begin + 1) add_sentence_entitys = [] company_dict = sorted(list(company_dict),key=lambda x:len(x[1]),reverse=True) for company_type,company_text in company_dict: begin_index_list = findAllIndex(company_text,sentence_text) for begin_index in begin_index_list: is_continue = False for t_begin,t_end in list(company_index[sentence_index]): if begin_index>=t_begin and begin_index+len(company_text)<=t_end: is_continue = True break if not is_continue: add_sentence_entitys.append((begin_index,begin_index+len(company_text),company_type,company_text)) company_index[sentence_index].add((begin_index,begin_index+len(company_text))) else: continue for ner_entity in add_sentence_entitys: begin_index_temp = ner_entity[0] end_index_temp = ner_entity[1] entity_type = ner_entity[2] entity_text = ner_entity[3] if entity_type in ["org","company"] and not isLegalEnterprise(entity_text): continue for j in range(len(list_tokenbegin)): if list_tokenbegin[j]==begin_index_temp: begin_index = j break elif list_tokenbegin[j]>begin_index_temp: begin_index = j-1 break begin_index_temp += len(str(entity_text)) for j in range(begin_index,len(list_tokenbegin)): if list_tokenbegin[j]>=begin_index_temp: end_index = j-1 break entity_id = "%s_%d_%d_%d"%(doc_id,sentence_index,begin_index,end_index) #去掉标点符号 entity_text = re.sub("[,，。：!&@$\*]","",entity_text) entity_text = entity_text.replace("(","（").replace(")","）") if isinstance(entity_text,str) else entity_text list_entitys_temp.append(Entity(doc_id,entity_id,entity_text,entity_type,sentence_index,begin_index,end_index,ner_entity[0],ner_entity[1],in_attachment=in_attachment)) list_entitys_temp.sort(key=lambda x:(x.sentence_index,x.begin_index)) list_entitys.append(list_entitys_temp) return list_entitys def union_result(codeName,prem): ''' @summary:模型的结果拼成字典 @param: codeName:编号名称模型的结果字典 prem:拿到属性的角色的字典 @return:拼接起来的字典 ''' result = [] assert len(codeName)==len(prem) for item_code,item_prem in zip(codeName,prem): result.append(dict(item_code,**item_prem)) return result def persistenceData(data): ''' @summary:将中间结果保存到数据库-线上生产的时候不需要执行 ''' import psycopg2 conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101") cursor = conn.cursor() for item_index in range(len(data)): item = data[item_index] doc_id = item[0] dic = item[1] code = dic['code'] name = dic['name'] prem = dic['prem'] if len(code)==0: code_insert = "" else: code_insert = ";".join(code) prem_insert = "" for item in prem: for x in item: if isinstance(x, list): if len(x)>0: for x1 in x: prem_insert+="/".join(x1)+"," prem_insert+="$" else: prem_insert+=str(x)+"$" prem_insert+=";" sql = " insert into predict_validation(doc_id,code,name,prem) values('"+doc_id+"','"+code_insert+"','"+name+"','"+prem_insert+"')" cursor.execute(sql) conn.commit() conn.close() def persistenceData1(list_entitys,list_sentences): ''' @summary:将中间结果保存到数据库-线上生产的时候不需要执行 ''' import psycopg2 conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101") cursor = conn.cursor() for list_entity in list_entitys: for entity in list_entity: if entity.values is not None: sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index,label,values) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+","+str(entity.label)+",array"+str(entity.values)+")" else: sql = " insert into predict_entity(entity_id,entity_text,entity_type,doc_id,sentence_index,begin_index,end_index) values('"+str(entity.entity_id)+"','"+str(entity.entity_text)+"','"+str(entity.entity_type)+"','"+str(entity.doc_id)+"',"+str(entity.sentence_index)+","+str(entity.begin_index)+","+str(entity.end_index)+")" cursor.execute(sql) for list_sentence in list_sentences: for sentence in list_sentence: str_tokens = "[" for item in sentence.tokens: str_tokens += "'" if item=="'": str_tokens += "''" else: str_tokens += item str_tokens += "'," str_tokens = str_tokens[:-1]+"]" sql = " insert into predict_sentences(doc_id,sentence_index,tokens) values('"+sentence.doc_id+"',"+str(sentence.sentence_index)+",array"+str_tokens+")" cursor.execute(sql) conn.commit() conn.close() def _handle(item,result_queue): dochtml = item["dochtml"] docid = item["docid"] list_innerTable = tableToText(BeautifulSoup(dochtml,"lxml")) flag = False if list_innerTable: flag = True for table in list_innerTable: result_queue.put({"docid":docid,"json_table":json.dumps(table,ensure_ascii=False)}) def getPredictTable(): filename = "D:\Workspace2016\DataExport\data\websouce_doc.csv" import pandas as pd import json from BiddingKG.dl.common.MultiHandler import MultiHandler,Queue df = pd.read_csv(filename) df_data = {"json_table":[],"docid":[]} _count = 0 _sum = len(df["docid"]) task_queue = Queue() result_queue = Queue() _index = 0 for dochtml,docid in zip(df["dochtmlcon"],df["docid"]): task_queue.put({"docid":docid,"dochtml":dochtml,"json_table":None}) _index += 1 mh = MultiHandler(task_queue=task_queue,task_handler=_handle,result_queue=result_queue,process_count=5,thread_count=1) mh.run() while True: try: item = result_queue.get(block=True,timeout=1) df_data["docid"].append(item["docid"]) df_data["json_table"].append(item["json_table"]) except Exception as e: print(e) break df_1 = pd.DataFrame(df_data) df_1.to_csv("../form/websource_67000_table.csv",columns=["docid","json_table"]) if __name__=="__main__": ''' import glob for file in glob.glob("C:\\Users\\User\\Desktop\\test\\*.html"): file_txt = str(file).replace("html","txt") with codecs.open(file_txt,"a+",encoding="utf8") as f: f.write("\n================\n") content = codecs.open(file,"r",encoding="utf8").read() f.write(segment(tableToText(BeautifulSoup(content,"lxml")))) ''' # content = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read() # print(segment(tableToText(BeautifulSoup(content,"lxml")))) # getPredictTable() with open('D:/138786703.html', 'r', encoding='utf-8') as f: sourceContent = f.read() # article_processed = segment(tableToText(BeautifulSoup(sourceContent, "lxml"))) # print(article_processed) list_articles, list_sentences, list_entitys, _cost_time = get_preprocessed([['doc_id', sourceContent, "", "", '', '2021-02-01']], useselffool=True) for entity in list_entitys[0]: print(entity.entity_type, entity.entity_text)