# -*- coding: utf-8 -*- """``CodeNamePredict`` — 编号名称模型。 Phase 5 从 ``interface/predictor.py``（约 217-794 行）迁出。 原 ``from common.Utils import *`` / ``from interface.modelFactory import *`` 已替换为显式 import；``os.path.dirname(__file__)`` 路径引用替换为 ``predictors._common.INTERFACE_DIR``。 """ from __future__ import absolute_import import os import re import json import numpy as np import tensorflow as tf import requests from BiddingKG.dl.common.logging import log from BiddingKG.dl.common.Utils import load from BiddingKG.dl.common.context_utils import find_index from BiddingKG.dl.model_runtime.viterbi import viterbi_decode from BiddingKG.dl.model_runtime.embed import getLazyLoad, embedding_word from BiddingKG.dl.interface.Entitys import Entity from BiddingKG.dl.services.external_inference.paieas import ( USE_API, API_URL, USE_PAI_EAS, tf_predict_pb2, vpc_requests, codeclasses_url, codeclasses_authorization, limitRun, ) from BiddingKG.dl.predictors._common import INTERFACE_DIR from keras.preprocessing.sequence import pad_sequences __all__ = ["CodeNamePredict"] # 编号名称模型 class CodeNamePredict(): def __init__(self,EMBED_DIM=None,BiRNN_UNITS=None,lazyLoad=getLazyLoad(),config=None): self.model = None self.MAX_LEN = None self.model_code = None if EMBED_DIM is None: self.EMBED_DIM = 60 else: self.EMBED_DIM = EMBED_DIM if BiRNN_UNITS is None: self.BiRNN_UNITS = 200 else: self.BiRNN_UNITS = BiRNN_UNITS self.filepath = os.path.join(INTERFACE_DIR, "..", "projectCode", "models", "model_project_"+str(self.EMBED_DIM)+"_"+str(self.BiRNN_UNITS)+".hdf5") #self.filepath = "../projectCode/models/model_project_60_200_200ep017-loss6.456-val_loss7.852-val_acc0.969.hdf5" self.filepath_code = os.path.join(INTERFACE_DIR, "..", "projectCode", "models", "model_code.hdf5") vocabpath = os.path.join(INTERFACE_DIR, "codename_vocab.pk") classlabelspath = os.path.join(INTERFACE_DIR, "codename_classlabels.pk") self.vocab = load(vocabpath) self.class_labels = load(classlabelspath) #生成提取编号和名称的正则 id_PC_B = self.class_labels.index("PC_B") id_PC_M = self.class_labels.index("PC_M") id_PC_E = self.class_labels.index("PC_E") id_PN_B = self.class_labels.index("PN_B") id_PN_M = self.class_labels.index("PN_M") id_PN_E = self.class_labels.index("PN_E") self.PC_pattern = re.compile(str(id_PC_B)+str(id_PC_M)+"*"+str(id_PC_E)) self.PN_pattern = re.compile(str(id_PN_B)+str(id_PN_M)+"*"+str(id_PN_E)) # print("pc",self.PC_pattern) # print("pn",self.PN_pattern) self.word2index = dict((w,i) for i,w in enumerate(np.array(self.vocab))) self.inputs = None self.outputs = None self.sess_codename = tf.Session(graph=tf.Graph(),config=config) self.sess_codesplit = tf.Session(graph=tf.Graph(),config=config) self.inputs_code = None self.outputs_code = None if not lazyLoad: self.getModel() self.getModel_code() def getModel(self): ''' @summary: 取得编号和名称模型 ''' if self.inputs is None: log("get model of codename") with self.sess_codename.as_default(): with self.sess_codename.graph.as_default(): meta_graph_def = tf.saved_model.loader.load(self.sess_codename, ["serve"], export_dir=os.path.join(INTERFACE_DIR, "codename_savedmodel_tf")) signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY signature_def = meta_graph_def.signature_def self.inputs = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name) self.inputs_length = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs_length"].name) self.keepprob = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["keepprob"].name) self.logits = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["logits"].name) self.trans = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["trans"].name) return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans else: return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans ''' if self.model is None: self.model = self.getBiLSTMCRFModel(self.MAX_LEN, self.vocab, self.EMBED_DIM, self.BiRNN_UNITS, self.class_labels,weights=None) self.model.load_weights(self.filepath) return self.model ''' def getModel_code(self): if self.inputs_code is None: log("get model of code") with self.sess_codesplit.as_default(): with self.sess_codesplit.graph.as_default(): meta_graph_def = tf.saved_model.loader.load(self.sess_codesplit, ["serve"], export_dir=os.path.join(INTERFACE_DIR, "codesplit_savedmodel")) signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY signature_def = meta_graph_def.signature_def self.inputs_code = [] self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)) self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)) self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)) self.outputs_code = self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name) self.sess_codesplit.graph.finalize() return self.inputs_code,self.outputs_code else: return self.inputs_code,self.outputs_code ''' if self.model_code is None: log("get model of model_code") with self.sess_codesplit.as_default(): with self.sess_codesplit.graph.as_default(): self.model_code = models.load_model(self.filepath_code, custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score}) return self.model_code ''' def getBiLSTMCRFModel(self,MAX_LEN,vocab,EMBED_DIM,BiRNN_UNITS,chunk_tags,weights): ''' model = models.Sequential() model.add(layers.Embedding(len(vocab), EMBED_DIM, mask_zero=True)) # Random embedding model.add(layers.Bidirectional(layers.LSTM(BiRNN_UNITS // 2, return_sequences=True))) crf = CRF(len(chunk_tags), sparse_target=True) model.add(crf) model.summary() model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy]) return model ''' input = layers.Input(shape=(None,)) if weights is not None: embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True,weights=[weights],trainable=True)(input) else: embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True)(input) bilstm = layers.Bidirectional(layers.LSTM(BiRNN_UNITS//2,return_sequences=True))(embedding) bilstm_dense = layers.TimeDistributed(layers.Dense(len(chunk_tags)))(bilstm) crf = CRF(len(chunk_tags),sparse_target=True) crf_out = crf(bilstm_dense) model = models.Model(input=[input],output = [crf_out]) model.summary() model.compile(optimizer = 'adam', loss = crf.loss_function, metrics = [crf.accuracy]) return model #根据规则补全编号或名称两边的符号 def fitDataByRule(self,data): symbol_dict = {"(":")", "（":"）", "[":"]", "【":"】", ")":"(", "）":"（", "]":"[", "】":"【"} leftSymbol_pattern = re.compile("[\(（\[【]") rightSymbol_pattern = re.compile("[\)）\]】]") leftfinds = re.findall(leftSymbol_pattern,data) rightfinds = re.findall(rightSymbol_pattern,data) result = data if len(leftfinds)+len(rightfinds)==0: return data elif len(leftfinds)==len(rightfinds): return data elif abs(len(leftfinds)-len(rightfinds))==1: if len(leftfinds)>len(rightfinds): if symbol_dict.get(data[0]) is not None: result = data[1:] else: #print(symbol_dict.get(leftfinds[0])) result = data+symbol_dict.get(leftfinds[0]) else: if symbol_dict.get(data[-1]) is not None: result = data[:-1] else: result = symbol_dict.get(rightfinds[0])+data return result def decode(self,logits, trans, sequence_lengths, tag_num): viterbi_sequences = [] for logit, length in zip(logits, sequence_lengths): score = logit[:length] viterbi_seq, viterbi_score = viterbi_decode(score, trans) viterbi_sequences.append(viterbi_seq) return viterbi_sequences def predict(self,list_sentences,list_entitys=None,MAX_AREA = 5000, doctitle = ""): #@summary: 获取每篇文章的code和name # pattern_score = re.compile("工程|服务|采购|施工|项目|系统|招标|中标|公告|学校|[大中小]学校?|医院|公司|分公司|研究院|政府采购中心|学院|中心校?|办公室|政府|财[政务]局|办事处|委员会|[部总支]队|警卫局|幼儿园|党委|党校|银行|分行|解放军|发电厂|供电局|管理所|供电公司|卷烟厂|机务段|研究[院所]|油厂|调查局|调查中心|出版社|电视台|监狱|水厂|服务站|信用合作联社|信用社|交易所|交易中心|交易中心党校|科学院|测绘所|运输厅|管理处|局|中心|机关|部门?|处|科|厂|集团|图书馆|馆|所|厅|楼|区|酒店|场|基地|矿|餐厅|酒店") pattern_score = re.compile('建设项目|服务项目|工程项目|工程施工|建设工程|服务中心|基础设施|物业管理|工程设计|妇幼保健|咨询服务|管理系统|管理中心|改建工程|配套工程|公安局|幼儿园|管理局|使用权|办公楼|教育局|管理处|图书馆|经营权|项目|采购|工程|改造|服务|设备|中心|医院|系统|建设|监理|施工|维修|学院|安装|设计|关于|标段|招标|技术|询价|管理|学校|小学|中学|平台|提升|设施|检测|整治|社区|装修|政府|绿化|物资|租赁|地块|医疗|编制|公开|规划|监控|教育|维护|校区|治理|升级|安置|竞价|购置|评估|勘察|承包|实验|大学|材料|生产|耗材|招租|硬化|维保|用地|消防|审计|拍卖|物业|入围|养护|机关|企业|用房|出让|资产|分局|验收|宣传|处置|校园|研究|咨询|修缮|更换|装饰|劳务|保养|物流|出租|局|院') result = [] index_unk = self.word2index.get("") # index_pad = self.word2index.get("") if list_entitys is None: list_entitys = [[] for _ in range(len(list_sentences))] for list_sentence,list_entity in zip(list_sentences,list_entitys): if len(list_sentence)==0: result.append([{"code":[],"name":""}]) continue doc_id = list_sentence[0].doc_id # sentences = [] # for sentence in list_sentence: # if len(sentence.sentence_text)>MAX_AREA: # for _sentence_comma in re.split("[;；，\n]",sentence): # _comma_index = 0 # while(_comma_indexMAX_AREA: MAX_LEN = MAX_AREA _LEN = MAX_AREA//MAX_LEN #预测 x = [[self.word2index.get(word,index_unk)for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]] # x = [[getIndexOfWord(word) for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]] x_len = [len(_x) if len(_x) < MAX_LEN else MAX_LEN for _x in x] x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post") if USE_API: requests_result = requests.post(API_URL + "/predict_codeName", json={"inouts": x.tolist(), "inouts_len": x_len},verify=True) predict_y = json.loads(requests_result.text)['result'] # print("cost_time:", json.loads(requests_result.text)['cost_time']) # print(MAX_LEN,_LEN,_begin_index) else: with self.sess_codename.as_default(): t_input,t_input_length,t_keepprob,t_logits,t_trans = self.getModel() _logits,_trans = self.sess_codename.run([t_logits,t_trans],feed_dict={t_input:x, t_input_length:x_len, t_keepprob:1.0}) predict_y = self.decode(_logits,_trans,x_len,7) # print('==========',_logits) ''' for item11 in np.argmax(predict_y,-1): print(item11) print(predict_y) ''' # print(predict_y) for sentence,predict in zip(list_sentence[_begin_index:_begin_index+_LEN],np.array(predict_y)): pad_sentence = sentence.sentence_text[:MAX_LEN] join_predict = "".join([str(s) for s in predict]) # print(pad_sentence) # print(join_predict) code_x = [] code_text = [] pre_text = [] temp_entitys = [] for iter in re.finditer(self.PC_pattern,join_predict): get_len = 40 if iter.span()[0]0: code_x = np.transpose(np.array(code_x,dtype=np.float32),(1,0,2,3)) if USE_PAI_EAS: request = tf_predict_pb2.PredictRequest() request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT request.inputs["input0"].array_shape.dim.extend(np.shape(code_x[0])) request.inputs["input0"].float_val.extend(np.array(code_x[0],dtype=np.float64).reshape(-1)) request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT request.inputs["input1"].array_shape.dim.extend(np.shape(code_x[1])) request.inputs["input1"].float_val.extend(np.array(code_x[1],dtype=np.float64).reshape(-1)) request.inputs["input2"].dtype = tf_predict_pb2.DT_FLOAT request.inputs["input2"].array_shape.dim.extend(np.shape(code_x[2])) request.inputs["input2"].float_val.extend(np.array(code_x[2],dtype=np.float64).reshape(-1)) request_data = request.SerializeToString() list_outputs = ["outputs"] _result = vpc_requests(codeclasses_url, codeclasses_authorization, request_data, list_outputs) if _result is not None: predict_code = _result["outputs"] else: with self.sess_codesplit.as_default(): with self.sess_codesplit.graph.as_default(): predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]]) else: with self.sess_codesplit.as_default(): with self.sess_codesplit.graph.as_default(): inputs_code,outputs_code = self.getModel_code() predict_code = limitRun(self.sess_codesplit,[outputs_code],feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]})[0] #predict_code = self.sess_codesplit.run(outputs_code,feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]}) #predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]]) for h in range(len(predict_code)): if predict_code[h][0]>0.5: the_code = self.fitDataByRule(code_text[h]) # print(the_code) #add code to entitys list_entity.append(temp_entitys[h]) in_att = 1 if temp_entitys[h].in_attachment else 0 # 是否在附件 if re.search(',|/|;|、|，', the_code) and len(the_code)>25: for it in re.split(',|/|;|、|，', the_code): if len(it) > 8: if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]): item['code'].append((it, in_att, 0, sentence.sentence_index)) elif re.search('采购(计划)?编[号码]：?$', pre_text[h]): item['code'].append((it, in_att, 1, sentence.sentence_index)) elif re.search('(询价|合同)编[号码]：?$', pre_text[h]): item['code'].append((it, in_att, 2, sentence.sentence_index)) elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]): item['code'].append((it, in_att, 2.5, sentence.sentence_index)) else: item['code'].append((it, in_att, 3, sentence.sentence_index)) elif len(item['code']) > 0: new_it = item['code'][-1][0] + re.search(',|/|;|、|，', the_code).group(0) + it if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]): item['code'][-1] = (new_it, in_att, 0, sentence.sentence_index) elif re.search('采购(计划)?编[号码]：?$', pre_text[h]): item['code'][-1] = (new_it, in_att, 1, sentence.sentence_index) elif re.search('(询价|合同)编[号码]：?$', pre_text[h]): item['code'][-1] = (new_it, in_att, 2, sentence.sentence_index) elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]): item['code'].append((new_it, in_att, 2.5, sentence.sentence_index)) else: item['code'][-1] = (new_it, in_att, 3, sentence.sentence_index) else: if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]): item['code'].append((the_code, in_att, 0, sentence.sentence_index)) elif re.search('采购(计划)?编[号码]：?$', pre_text[h]): item['code'].append((the_code, in_att, 1, sentence.sentence_index)) elif re.search('(询价|合同)编[号码]：?$', pre_text[h]): item['code'].append((the_code, in_att, 2, sentence.sentence_index)) elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]): item['code'].append((the_code, in_att, 2.5, sentence.sentence_index)) else: item['code'].append((the_code, in_att, 3, sentence.sentence_index)) break else: if len(the_code)<5: # 避免510545935 这种把 招标项目编[号码]：2024年第二期 只提取2024 continue # item['code'].append(the_code) if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]): item['code'].append((the_code, in_att, 0, sentence.sentence_index)) elif re.search('采购(计划)?编[号码]：?$', pre_text[h]): item['code'].append((the_code, in_att, 1, sentence.sentence_index)) elif re.search('(询价|合同)编[号码]：?$', pre_text[h]): item['code'].append((the_code, in_att, 2, sentence.sentence_index)) elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]): item['code'].append((the_code, in_att, 2.5, sentence.sentence_index)) else: item['code'].append((the_code, in_att, 3, sentence.sentence_index)) for iter in re.finditer(self.PN_pattern,join_predict): raw_name = pad_sentence[iter.start():iter.end()] _name = self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]) if len(_name)>200: # 避免模型预测类似 202750503 这种很长重复字很多的错误项目名称 continue elif '公司：你单位在' in _name: # 避免类似 339900030 这种作为项目名称，导致中标角色作为招标角色 continue elif _name.endswith('公司') and len(_name)<20: # 修复 456957250 雄县辉茂纸塑包装制品销售有限公司 作为项目名称 continue elif re.search('(工程|项目)$', raw_name) and re.match('(有限)?公司', pad_sentence[iter.end():]) and re.search( '(项目|工程)名称：', pad_sentence[max(0, iter.start()-8):iter.start()])==None: # 修复 150904150 威海经济技术开发区园林绿化工程公司 中的 威海经济技术开发区园林绿化工程 为项目名称 continue elif re.search('恭喜中标', raw_name): # 修复 536826334 type:name,text:浙江德邦印务有限公司恭喜中标标段1 continue #add name to entitys _entity = Entity(doc_id=sentence.doc_id,entity_id="%s_%s_%s_%s"%(sentence.doc_id,sentence.sentence_index,iter.span()[0],iter.span()[1]),entity_text=_name,entity_type="name",sentence_index=sentence.sentence_index,begin_index=0,end_index=0,wordOffset_begin=iter.span()[0],wordOffset_end=iter.span()[1],in_attachment=sentence.in_attachment) list_entity.append(_entity) # w = 1 if re.search('(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]', pad_sentence[iter.span()[0]-10:iter.span()[0]])!=None else 0.5 # w = 1 if re.search('(项目|工程|招标|采购|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题|项目)[:：\s]', pad_sentence[iter.span()[0]-10:iter.span()[0]])!=None else 0.5 # if _name not in dict_name_freq_score: # # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1] # len_name = len(_name) if len(_name) <50 else 100-len(_name) # 2023/03/02 超出50长度的逐渐递减 # dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len_name * 0.05), w] # else: # dict_name_freq_score[_name][0] += 1 # if w > dict_name_freq_score[_name][2]: # dict_name_freq_score[_name][2] = w score = 1 if re.search('项目名称[:：\s]', pad_sentence[iter.span()[0]-8:iter.span()[0]]): score += 1 elif re.search('(工程|招标|采购|合同|标项|标的|计划|询价|询价[单书]|询价通知书|申购)(名称|标题|主题|项目)[:：\s]', pad_sentence[iter.span()[0]-8:iter.span()[0]]): score += 0.5 elif re.search('(招标|采购)(条件|范围|内容)|合同包|合同段|采购包', pad_sentence[iter.span()[0]-20:iter.span()[0]]): score += 0.49 if raw_name in doctitle: score += 0.4 + len(raw_name)/len(doctitle)/10 score += (1-sentence.sentence_index/len(list_sentence))/10 if sentence.in_attachment: score /= 2 name_list.append((_name, score)) ''' for iter in re.finditer(self.PN_pattern,join_predict): print("name-",self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])) if item[1]['name']=="": for iter in re.finditer(self.PN_pattern,join_predict): #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]) item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]) break ''' if _begin_index+_LEN>=len(list_sentence): break _begin_index += _LEN list_name_freq_score = [] # print('模型预测项目名称：', dict_name_freq_score) # 2020/11/23 大网站规则调整 # if len(dict_name_freq_score) == 0: if len(name_list) == 0: # name_re1 = '(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]+([^，。：；]{2,60})[，。]' name_re1 = '(项目|工程|招标|采购(条目)?|合同|标项|标的|计划|询价|询价单|询价通知书|申购单|申购)(名称|标名|标题|主题)[:：\s]+(?P[^，。：；]{2,60})[，。]' name_re2 = '(合同|采购)包\d（(?P[^，。：；]{2,60})）[：，。]' # 20241202 补充合同包 包名表达 558410976 for sentence in list_sentence: # pad_sentence = sentence.sentence_text othername = re.search(name_re1, sentence.sentence_text) if othername == None: othername = re.search(name_re2, sentence.sentence_text) if othername != None: project_name = othername.group('name') if re.search('[\u4e00-\u9fa5]+', project_name) == None: # 没有中文的项目名称去除 # log('没有中文的项目名称去除') continue beg = find_index([project_name], sentence.sentence_text)[0] end = beg + len(project_name) raw_name = sentence.sentence_text[beg:end] _name = self.fitDataByRule(sentence.sentence_text[beg:end]) # print('规则召回项目名称：', _name) # add name to entitys _entity = Entity(doc_id=sentence.doc_id, entity_id="%s_%s_%s_%s" % ( sentence.doc_id, sentence.sentence_index, beg, end), entity_text=_name, entity_type="name", sentence_index=sentence.sentence_index, begin_index=0, end_index=0, wordOffset_begin=beg, wordOffset_end=end,in_attachment=sentence.in_attachment) list_entity.append(_entity) # w = 1 # if _name not in dict_name_freq_score: # # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1] # dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05), w] # else: # dict_name_freq_score[_name][0] += 1 score = 1 if re.search('项目名称[:：\s]', sentence.sentence_text[beg - 8:beg]): score += 1 elif re.search('(工程|招标|采购|合同|标项|标的|计划|询价|询价[单书]|询价通知书|申购)(名称|标题|主题|项目)[:：\s]', sentence.sentence_text[beg - 8:beg]): score += 0.5 elif re.search('(招标|采购)(条件|范围|内容)|合同包|合同段|采购包', sentence.sentence_text[beg - 20:beg]): score += 0.49 if raw_name in doctitle: score += 0.4 + len(raw_name) / len(doctitle) / 10 score += (1 - sentence.sentence_index / len(list_sentence)) / 10 if sentence.in_attachment: score /= 2 name_list.append((_name, score)) # othername = re.search(name_re1, sentence.sentence_text) # if othername != None: # _name = othername.group(3) # if _name not in dict_name_freq_score: # dict_name_freq_score[_name] = [1, len(re.findall(pattern_score, _name)) + len(_name) * 0.1] # else: # dict_name_freq_score[_name][0] += 1 # for _name in dict_name_freq_score.keys(): # list_name_freq_score.append([_name,dict_name_freq_score[_name]]) # # print(list_name_freq_score) # if len(list_name_freq_score)>0: # list_name_freq_score.sort(key=lambda x:x[1][0]*x[1][1]*x[1][2],reverse=True) # item['name'] = list_name_freq_score[0][0] # for it in list_name_freq_score: # print('项目名称及分值：',it[0],it[1], it[1][0]*it[1][1]) # if list_name_freq_score[0][1][0]>1: # item[1]['name'] = list_name_freq_score[0][0] # else: # list_name_freq_score.sort(key=lambda x:x[1][1],reverse=True) # item[1]["name"] = list_name_freq_score[0][0] if name_list: name_list.sort(key=lambda x: x[1], reverse=True) item['name'] = name_list[0][0] #下面代码加上去用正则添加某些识别不到的项目编号 if item['code'] == []: for sentence in list_sentence: # othercode = re.search('(采购计划编号|询价编号)[\)）]?[:：]?([\[\]a-zA-Z0-9\-]{5,30})', sentence.sentence_text) # if othercode != None: # item[1]['code'].append(othercode.group(2)) # 2020/11/23 大网站规则调整 in_att = 1 if sentence.in_attachment else 0 othercode = re.search('(项目|采购|招标|品目|询价|竞价|询价[单书]|磋商|订单|账单|交易|文件|计划|场次|标的|标段|标包|分包|标段\(包\)|招标文件|合同|通知书|公告|工程|寻源|标书|包件|谈判|申购)(单据?号|编号|标号|编码|代码|备案号|号)[:：\s]+(?P[^，。；：、]{6,30}[a-zA-Z0-9\号期])[\)，。\u4e00-\u9fa5]', sentence.sentence_text) if othercode != None: # item['code'].append(othercode.group('code')) if re.search("(项目编[号码]|招标编[号码])：?", othercode.group(0)): item['code'].append((othercode.group('code'), in_att, 0, sentence.sentence_index)) elif re.search('采购(计划)?编[号码]：?', othercode.group(0)): item['code'].append((othercode.group('code'), in_att, 1, sentence.sentence_index)) elif re.search('(询价|合同)编[号码]：?', othercode.group(0)): item['code'].append((othercode.group('code'), in_att, 2, sentence.sentence_index)) elif re.search('(询价|合同|采购|招标|项目)[单标]号：?', othercode.group(0)): item['code'].append((othercode.group('code'), in_att, 2.5, sentence.sentence_index)) else: item['code'].append((othercode.group('code'), in_att, 3, sentence.sentence_index)) # print('规则召回项目编号：', othercode.group('code')) # item['code'] = [code for code in item['code'] if len(code)<500] # item['code'].sort(key=lambda x:len(x),reverse=True) item['code'] = [code for code in item['code'] if len(code[0]) < 500] item['code'].sort(key=lambda x: [x[1],x[2],x[3]]) code_list = [] for it in item['code']: if it[0] not in code_list: code_list.append(it[0]) # item['code'] = [it[0] for it in item['code']] item['code'] = code_list result.append(item) list_sentence.sort(key=lambda x: x.sentence_index,reverse=False) return result ''' #当数据量过大时会报错 def predict(self,articles,MAX_LEN = None): sentences = [] for article in articles: for sentence in article.content.split("。"): sentences.append([sentence,article.id]) if MAX_LEN is None: sent_len = [len(sentence[0]) for sentence in sentences] MAX_LEN = max(sent_len) #print(MAX_LEN) #若为空，则直接返回空 result = [] if MAX_LEN==0: for article in articles: result.append([article.id,{"code":[],"name":""}]) return result index_unk = self.word2index.get("") index_pad = self.word2index.get("") x = [[self.word2index.get(word,index_unk)for word in sentence[0]]for sentence in sentences] x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post") predict_y = self.getModel().predict(x) last_doc_id = "" item = [] for sentence,predict in zip(sentences,np.argmax(predict_y,-1)): pad_sentence = sentence[0][:MAX_LEN] doc_id = sentence[1] join_predict = "".join([str(s) for s in predict]) if doc_id!=last_doc_id: if last_doc_id!="": result.append(item) item = [doc_id,{"code":[],"name":""}] code_set = set() code_x = [] code_text = [] for iter in re.finditer(self.PC_pattern,join_predict): get_len = 40 if iter.span()[0]0: code_x = np.transpose(np.array(code_x),(1,0,2,3)) predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]]) for h in range(len(predict_code)): if predict_code[h][0]>0.5: the_code = self.fitDataByRule(code_text[h]) if the_code not in code_set: code_set.add(the_code) item[1]['code'] = list(code_set) if item[1]['name']=="": for iter in re.finditer(self.PN_pattern,join_predict): #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]) item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]) break last_doc_id = doc_id result.append(item) return result '''