BIDI_EXTRACTION_AI/BiddingKG/dl/predictors/codename.py

# -*- coding: utf-8 -*-
"""``CodeNamePredict`` — 编号名称模型。

Phase 5 从 ``interface/predictor.py``（约 217-794 行）迁出。
原 ``from common.Utils import *`` / ``from interface.modelFactory import *``
已替换为显式 import；``os.path.dirname(__file__)`` 路径引用替换为
``predictors._common.INTERFACE_DIR``。
"""
from __future__ import absolute_import

import os
import re
import json

import numpy as np
import tensorflow as tf
import requests

from BiddingKG.dl.common.logging import log
from BiddingKG.dl.common.Utils import load
from BiddingKG.dl.common.context_utils import find_index
from BiddingKG.dl.model_runtime.viterbi import viterbi_decode
from BiddingKG.dl.model_runtime.embed import getLazyLoad, embedding_word
from BiddingKG.dl.interface.Entitys import Entity
from BiddingKG.dl.services.external_inference.paieas import (
    USE_API,
    API_URL,
    USE_PAI_EAS,
    tf_predict_pb2,
    vpc_requests,
    codeclasses_url,
    codeclasses_authorization,
    limitRun,
)
from BiddingKG.dl.predictors._common import INTERFACE_DIR
from keras.preprocessing.sequence import pad_sequences

__all__ = ["CodeNamePredict"]


# 编号名称模型
class CodeNamePredict():
    
    def __init__(self,EMBED_DIM=None,BiRNN_UNITS=None,lazyLoad=getLazyLoad(),config=None):
        
        self.model = None
        self.MAX_LEN = None
        self.model_code = None
        if EMBED_DIM is None:
            self.EMBED_DIM = 60
        else:
            self.EMBED_DIM = EMBED_DIM
        if BiRNN_UNITS is None:
            self.BiRNN_UNITS = 200
        else:
            self.BiRNN_UNITS = BiRNN_UNITS
        self.filepath = os.path.join(INTERFACE_DIR, "..", "projectCode", "models", "model_project_"+str(self.EMBED_DIM)+"_"+str(self.BiRNN_UNITS)+".hdf5")
        #self.filepath = "../projectCode/models/model_project_60_200_200ep017-loss6.456-val_loss7.852-val_acc0.969.hdf5"
        self.filepath_code = os.path.join(INTERFACE_DIR, "..", "projectCode", "models", "model_code.hdf5")
        vocabpath = os.path.join(INTERFACE_DIR, "codename_vocab.pk")
        classlabelspath = os.path.join(INTERFACE_DIR, "codename_classlabels.pk")
        self.vocab = load(vocabpath)
        self.class_labels = load(classlabelspath)
        
        #生成提取编号和名称的正则
        id_PC_B = self.class_labels.index("PC_B")
        id_PC_M = self.class_labels.index("PC_M")
        id_PC_E = self.class_labels.index("PC_E")
        id_PN_B = self.class_labels.index("PN_B")
        id_PN_M = self.class_labels.index("PN_M")
        id_PN_E = self.class_labels.index("PN_E")
        self.PC_pattern = re.compile(str(id_PC_B)+str(id_PC_M)+"*"+str(id_PC_E))
        self.PN_pattern = re.compile(str(id_PN_B)+str(id_PN_M)+"*"+str(id_PN_E))
        # print("pc",self.PC_pattern)
        # print("pn",self.PN_pattern)
        self.word2index = dict((w,i) for i,w in enumerate(np.array(self.vocab)))
        
        self.inputs = None
        self.outputs = None
        self.sess_codename = tf.Session(graph=tf.Graph(),config=config)
        self.sess_codesplit = tf.Session(graph=tf.Graph(),config=config)
        self.inputs_code = None
        self.outputs_code = None
        if not lazyLoad:
            self.getModel()
            self.getModel_code()
        
        
        
    def getModel(self):
        '''
        @summary: 取得编号和名称模型
        '''
        if self.inputs is None:
            log("get model of codename")
            with self.sess_codename.as_default():
                with self.sess_codename.graph.as_default():
                    meta_graph_def = tf.saved_model.loader.load(self.sess_codename, ["serve"], export_dir=os.path.join(INTERFACE_DIR, "codename_savedmodel_tf"))
                    signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
                    signature_def = meta_graph_def.signature_def
                    self.inputs = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
                    self.inputs_length = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs_length"].name)
                    self.keepprob = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["keepprob"].name)
                    self.logits = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["logits"].name)
                    self.trans = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["trans"].name)

                return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans
        else:
            return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans
        '''    
        if self.model is None:
            self.model = self.getBiLSTMCRFModel(self.MAX_LEN, self.vocab, self.EMBED_DIM, self.BiRNN_UNITS, self.class_labels,weights=None)
            self.model.load_weights(self.filepath)
        return self.model
        '''
    
    def getModel_code(self):
        if self.inputs_code is None:
            log("get model of code")
            with self.sess_codesplit.as_default():
                with self.sess_codesplit.graph.as_default():
                    meta_graph_def = tf.saved_model.loader.load(self.sess_codesplit, ["serve"], export_dir=os.path.join(INTERFACE_DIR, "codesplit_savedmodel"))
                    signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
                    signature_def = meta_graph_def.signature_def
                    self.inputs_code = []
                    self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name))
                    self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name))
                    self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name))
                    self.outputs_code = self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
                    self.sess_codesplit.graph.finalize()
                    return self.inputs_code,self.outputs_code
        else:
            return self.inputs_code,self.outputs_code
        '''
        if self.model_code is None:
            log("get model of model_code")
            with self.sess_codesplit.as_default():
                with self.sess_codesplit.graph.as_default():
                    self.model_code = models.load_model(self.filepath_code, custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
        return self.model_code
        '''
    
    def getBiLSTMCRFModel(self,MAX_LEN,vocab,EMBED_DIM,BiRNN_UNITS,chunk_tags,weights):
        '''
        model = models.Sequential()
        model.add(layers.Embedding(len(vocab), EMBED_DIM, mask_zero=True))  # Random embedding
        model.add(layers.Bidirectional(layers.LSTM(BiRNN_UNITS // 2, return_sequences=True)))
        crf = CRF(len(chunk_tags), sparse_target=True)
        model.add(crf)
        model.summary()
        model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
        return model
        '''
        input = layers.Input(shape=(None,))
        if weights is not None:
            embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True,weights=[weights],trainable=True)(input)
        else:
            embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True)(input)
        bilstm = layers.Bidirectional(layers.LSTM(BiRNN_UNITS//2,return_sequences=True))(embedding)
        bilstm_dense = layers.TimeDistributed(layers.Dense(len(chunk_tags)))(bilstm)
        crf = CRF(len(chunk_tags),sparse_target=True)
        crf_out = crf(bilstm_dense)
        model = models.Model(input=[input],output = [crf_out])
        model.summary()
        model.compile(optimizer = 'adam', loss = crf.loss_function, metrics = [crf.accuracy])
        return model
    
    #根据规则补全编号或名称两边的符号
    def fitDataByRule(self,data):
        symbol_dict = {"(":")",
                       "（":"）",
                       "[":"]",
                       "【":"】",
                       ")":"(",
                       "）":"（",
                       "]":"[",
                       "】":"【"}
        leftSymbol_pattern = re.compile("[\(（\[【]")
        rightSymbol_pattern = re.compile("[\)）\]】]")
        leftfinds = re.findall(leftSymbol_pattern,data)
        rightfinds = re.findall(rightSymbol_pattern,data)
        result = data
        if len(leftfinds)+len(rightfinds)==0:
            return data
        elif len(leftfinds)==len(rightfinds):
            return data
        elif abs(len(leftfinds)-len(rightfinds))==1:
            if len(leftfinds)>len(rightfinds):
                if symbol_dict.get(data[0]) is not None:
                    result = data[1:]
                else:
                    #print(symbol_dict.get(leftfinds[0]))
                    result = data+symbol_dict.get(leftfinds[0])
            else:
                if symbol_dict.get(data[-1]) is not None:
                    result = data[:-1]
                else:
                    result = symbol_dict.get(rightfinds[0])+data
        return  result

    def decode(self,logits, trans, sequence_lengths, tag_num):
        viterbi_sequences = []
        for logit, length in zip(logits, sequence_lengths):
            score = logit[:length]
            viterbi_seq, viterbi_score = viterbi_decode(score, trans)
            viterbi_sequences.append(viterbi_seq)
        return viterbi_sequences
    
    def predict(self,list_sentences,list_entitys=None,MAX_AREA = 5000, doctitle = ""):
        #@summary: 获取每篇文章的code和name
        # pattern_score = re.compile("工程|服务|采购|施工|项目|系统|招标|中标|公告|学校|[大中小]学校?|医院|公司|分公司|研究院|政府采购中心|学院|中心校?|办公室|政府|财[政务]局|办事处|委员会|[部总支]队|警卫局|幼儿园|党委|党校|银行|分行|解放军|发电厂|供电局|管理所|供电公司|卷烟厂|机务段|研究[院所]|油厂|调查局|调查中心|出版社|电视台|监狱|水厂|服务站|信用合作联社|信用社|交易所|交易中心|交易中心党校|科学院|测绘所|运输厅|管理处|局|中心|机关|部门?|处|科|厂|集团|图书馆|馆|所|厅|楼|区|酒店|场|基地|矿|餐厅|酒店")
        pattern_score = re.compile('建设项目|服务项目|工程项目|工程施工|建设工程|服务中心|基础设施|物业管理|工程设计|妇幼保健|咨询服务|管理系统|管理中心|改建工程|配套工程|公安局|幼儿园|管理局|使用权|办公楼|教育局|管理处|图书馆|经营权|项目|采购|工程|改造|服务|设备|中心|医院|系统|建设|监理|施工|维修|学院|安装|设计|关于|标段|招标|技术|询价|管理|学校|小学|中学|平台|提升|设施|检测|整治|社区|装修|政府|绿化|物资|租赁|地块|医疗|编制|公开|规划|监控|教育|维护|校区|治理|升级|安置|竞价|购置|评估|勘察|承包|实验|大学|材料|生产|耗材|招租|硬化|维保|用地|消防|审计|拍卖|物业|入围|养护|机关|企业|用房|出让|资产|分局|验收|宣传|处置|校园|研究|咨询|修缮|更换|装饰|劳务|保养|物流|出租|局|院')
        result = []
        index_unk = self.word2index.get("<unk>")
        # index_pad = self.word2index.get("<pad>")
        if list_entitys is None:
            list_entitys = [[] for _ in range(len(list_sentences))]
        for list_sentence,list_entity in zip(list_sentences,list_entitys):
            if len(list_sentence)==0:
                result.append([{"code":[],"name":""}])
                continue
            doc_id = list_sentence[0].doc_id
            # sentences = []
            # for sentence in list_sentence:
            #     if len(sentence.sentence_text)>MAX_AREA:
            #         for _sentence_comma in re.split("[;；，\n]",sentence):
            #             _comma_index = 0
            #             while(_comma_index<len(_sentence_comma)):
            #                 sentences.append(_sentence_comma[_comma_index:_comma_index+MAX_AREA])
            #                 _comma_index += MAX_AREA
            #     else:
            #         sentences.append(sentence+"。")
            list_sentence.sort(key=lambda x:len(x.sentence_text),reverse=True)
            _begin_index = 0
            
            item = {"code":[],"name":""}
            code_set = set()
            dict_name_freq_score = dict()
            name_list = []
            while(True):
                MAX_LEN = len(list_sentence[_begin_index].sentence_text)
                if MAX_LEN>MAX_AREA:
                    MAX_LEN = MAX_AREA
                _LEN = MAX_AREA//MAX_LEN
                #预测

                x = [[self.word2index.get(word,index_unk)for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]]
                # x = [[getIndexOfWord(word) for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]]
                x_len = [len(_x) if len(_x) < MAX_LEN else MAX_LEN for _x in x]
                x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post")

                if USE_API:
                    requests_result = requests.post(API_URL + "/predict_codeName", json={"inouts": x.tolist(), "inouts_len": x_len},verify=True)
                    predict_y = json.loads(requests_result.text)['result']
                    # print("cost_time:", json.loads(requests_result.text)['cost_time'])
                    # print(MAX_LEN,_LEN,_begin_index)
                else:
                    with self.sess_codename.as_default():
                        t_input,t_input_length,t_keepprob,t_logits,t_trans = self.getModel()
                        _logits,_trans = self.sess_codename.run([t_logits,t_trans],feed_dict={t_input:x,
                                                                                              t_input_length:x_len,
                                                                                              t_keepprob:1.0})
                        predict_y = self.decode(_logits,_trans,x_len,7)
                        # print('==========',_logits)

                        '''
                        for item11 in np.argmax(predict_y,-1):
                            print(item11)
                        print(predict_y)
                        '''
                # print(predict_y)
                for sentence,predict in zip(list_sentence[_begin_index:_begin_index+_LEN],np.array(predict_y)):
                    pad_sentence = sentence.sentence_text[:MAX_LEN]
                    join_predict = "".join([str(s) for s in predict])
                    # print(pad_sentence)
                    # print(join_predict)
                    code_x = []
                    code_text = []
                    pre_text = []
                    temp_entitys = []
                    for iter in re.finditer(self.PC_pattern,join_predict):
                        get_len = 40
                        if iter.span()[0]<get_len:
                            begin = 0
                        else:
                            begin = iter.span()[0]-get_len
                        end = iter.span()[1]+get_len
                        code_x.append(embedding_word([pad_sentence[begin:iter.span()[0]],pad_sentence[iter.span()[0]:iter.span()[1]].replace("，",""),pad_sentence[iter.span()[1]:end]],shape=(3,get_len,60)))
                        code_text.append(pad_sentence[iter.span()[0]:iter.span()[1]].replace("，", ""))
                        pre_text.append(pad_sentence[begin:iter.span()[0]])
                        _entity = Entity(doc_id=sentence.doc_id,entity_id="%s_%s_%s_%s"%(sentence.doc_id,sentence.sentence_index,iter.span()[0],iter.span()[1]),entity_text=pad_sentence[iter.span()[0]:iter.span()[1]].replace("，",""),entity_type="code",sentence_index=sentence.sentence_index,begin_index=0,end_index=0,wordOffset_begin=iter.span()[0],wordOffset_end=iter.span()[1],in_attachment=sentence.in_attachment)
                        temp_entitys.append(_entity)
                    #print("code",code_text)
                    if len(code_x)>0:
                        code_x = np.transpose(np.array(code_x,dtype=np.float32),(1,0,2,3))
                        if USE_PAI_EAS:
                            request = tf_predict_pb2.PredictRequest()
                            request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT
                            request.inputs["input0"].array_shape.dim.extend(np.shape(code_x[0]))
                            request.inputs["input0"].float_val.extend(np.array(code_x[0],dtype=np.float64).reshape(-1))
                            request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT
                            request.inputs["input1"].array_shape.dim.extend(np.shape(code_x[1]))
                            request.inputs["input1"].float_val.extend(np.array(code_x[1],dtype=np.float64).reshape(-1))
                            request.inputs["input2"].dtype = tf_predict_pb2.DT_FLOAT
                            request.inputs["input2"].array_shape.dim.extend(np.shape(code_x[2]))
                            request.inputs["input2"].float_val.extend(np.array(code_x[2],dtype=np.float64).reshape(-1))
                            request_data = request.SerializeToString()
                            list_outputs = ["outputs"]
                            _result = vpc_requests(codeclasses_url, codeclasses_authorization, request_data, list_outputs)
                            if _result is not None:
                                predict_code = _result["outputs"]
                            else:
                                with self.sess_codesplit.as_default():
                                    with self.sess_codesplit.graph.as_default():
                                        predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
                        else:
                            with self.sess_codesplit.as_default():
                                with self.sess_codesplit.graph.as_default():
                                    inputs_code,outputs_code = self.getModel_code()
                                    predict_code = limitRun(self.sess_codesplit,[outputs_code],feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]})[0]

                                    #predict_code = self.sess_codesplit.run(outputs_code,feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]})
                                    #predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
                        for h in range(len(predict_code)):
                            if predict_code[h][0]>0.5:
                                the_code = self.fitDataByRule(code_text[h])
                                # print(the_code)
                                #add code to entitys
                                list_entity.append(temp_entitys[h])
                                in_att = 1 if temp_entitys[h].in_attachment else 0 # 是否在附件
                                if re.search(',|/|;|、|，', the_code) and len(the_code)>25:
                                    for it in re.split(',|/|;|、|，', the_code):
                                        if len(it) > 8:
                                            if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]):
                                                item['code'].append((it, in_att, 0, sentence.sentence_index))
                                            elif re.search('采购(计划)?编[号码]：?$', pre_text[h]):
                                                item['code'].append((it, in_att, 1, sentence.sentence_index))
                                            elif re.search('(询价|合同)编[号码]：?$', pre_text[h]):
                                                item['code'].append((it, in_att, 2, sentence.sentence_index))
                                            elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]):
                                                item['code'].append((it, in_att, 2.5, sentence.sentence_index))
                                            else:
                                                item['code'].append((it, in_att, 3, sentence.sentence_index))
                                        elif len(item['code']) > 0:
                                            new_it = item['code'][-1][0] + re.search(',|/|;|、|，', the_code).group(0) + it
                                            if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]):
                                                item['code'][-1] = (new_it, in_att, 0, sentence.sentence_index)
                                            elif re.search('采购(计划)?编[号码]：?$', pre_text[h]):
                                                item['code'][-1] = (new_it, in_att, 1, sentence.sentence_index)
                                            elif re.search('(询价|合同)编[号码]：?$', pre_text[h]):
                                                item['code'][-1] = (new_it, in_att, 2, sentence.sentence_index)
                                            elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]):
                                                item['code'].append((new_it, in_att, 2.5, sentence.sentence_index))
                                            else:
                                                item['code'][-1] = (new_it, in_att, 3, sentence.sentence_index)
                                        else:
                                            if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]):
                                                item['code'].append((the_code, in_att, 0, sentence.sentence_index))
                                            elif re.search('采购(计划)?编[号码]：?$', pre_text[h]):
                                                item['code'].append((the_code, in_att, 1, sentence.sentence_index))
                                            elif re.search('(询价|合同)编[号码]：?$', pre_text[h]):
                                                item['code'].append((the_code, in_att, 2, sentence.sentence_index))
                                            elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]):
                                                item['code'].append((the_code, in_att, 2.5, sentence.sentence_index))
                                            else:
                                                item['code'].append((the_code, in_att, 3, sentence.sentence_index))
                                            break
                                else:
                                    if len(the_code)<5: # 避免510545935 这种把 招标项目编[号码]：2024年第二期 只提取2024
                                        continue
                                    # item['code'].append(the_code)
                                    if re.search("(项目编[号码]|招标编[号码])：?$", pre_text[h]):
                                        item['code'].append((the_code, in_att, 0, sentence.sentence_index))
                                    elif re.search('采购(计划)?编[号码]：?$', pre_text[h]):
                                        item['code'].append((the_code, in_att, 1, sentence.sentence_index))
                                    elif re.search('(询价|合同)编[号码]：?$', pre_text[h]):
                                        item['code'].append((the_code, in_att, 2, sentence.sentence_index))
                                    elif re.search('(询价|合同|采购|招标|项目)[单标]号：?$', pre_text[h]):
                                        item['code'].append((the_code, in_att, 2.5, sentence.sentence_index))
                                    else:
                                        item['code'].append((the_code, in_att, 3, sentence.sentence_index))

                    for iter in re.finditer(self.PN_pattern,join_predict):
                        raw_name = pad_sentence[iter.start():iter.end()]
                        _name = self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
                        if len(_name)>200: # 避免模型预测类似 202750503 这种很长重复字很多的错误项目名称
                            continue
                        elif '公司：你单位在' in _name: # 避免类似 339900030 这种作为项目名称，导致中标角色作为招标角色
                            continue
                        elif _name.endswith('公司') and len(_name)<20: # 修复 456957250 雄县辉茂纸塑包装制品销售有限公司 作为项目名称
                            continue
                        elif re.search('(工程|项目)$', raw_name) and re.match('(有限)?公司', pad_sentence[iter.end():]) and re.search(
                                '(项目|工程)名称：', pad_sentence[max(0, iter.start()-8):iter.start()])==None: # 修复 150904150 威海经济技术开发区园林绿化工程公司 中的 威海经济技术开发区园林绿化工程 为项目名称
                            continue
                        elif re.search('恭喜中标', raw_name):  # 修复 536826334 type:name,text:浙江德邦印务有限公司恭喜中标标段1
                            continue
                        #add name to entitys
                        _entity = Entity(doc_id=sentence.doc_id,entity_id="%s_%s_%s_%s"%(sentence.doc_id,sentence.sentence_index,iter.span()[0],iter.span()[1]),entity_text=_name,entity_type="name",sentence_index=sentence.sentence_index,begin_index=0,end_index=0,wordOffset_begin=iter.span()[0],wordOffset_end=iter.span()[1],in_attachment=sentence.in_attachment)
                        list_entity.append(_entity)
                        # w = 1 if re.search('(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]', pad_sentence[iter.span()[0]-10:iter.span()[0]])!=None else 0.5
                        # w = 1 if re.search('(项目|工程|招标|采购|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题|项目)[:：\s]', pad_sentence[iter.span()[0]-10:iter.span()[0]])!=None else 0.5
                        # if _name not in dict_name_freq_score:
                        #     # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
                        #     len_name = len(_name) if len(_name) <50 else 100-len(_name) # 2023/03/02 超出50长度的逐渐递减
                        #     dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len_name * 0.05), w]
                        # else:
                        #     dict_name_freq_score[_name][0] += 1
                        # if w > dict_name_freq_score[_name][2]:
                        #     dict_name_freq_score[_name][2] = w
                        score = 1
                        if re.search('项目名称[:：\s]', pad_sentence[iter.span()[0]-8:iter.span()[0]]):
                            score += 1
                        elif re.search('(工程|招标|采购|合同|标项|标的|计划|询价|询价[单书]|询价通知书|申购)(名称|标题|主题|项目)[:：\s]', pad_sentence[iter.span()[0]-8:iter.span()[0]]):
                            score += 0.5
                        elif re.search('(招标|采购)(条件|范围|内容)|合同包|合同段|采购包', pad_sentence[iter.span()[0]-20:iter.span()[0]]):
                            score += 0.49
                        if raw_name in doctitle:
                            score += 0.4 + len(raw_name)/len(doctitle)/10
                        score += (1-sentence.sentence_index/len(list_sentence))/10
                        if sentence.in_attachment:
                            score /= 2
                        name_list.append((_name, score))
                    '''
                    for iter in re.finditer(self.PN_pattern,join_predict):
                        print("name-",self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]))
                    if item[1]['name']=="":
                        for iter in re.finditer(self.PN_pattern,join_predict):
                            #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
                            item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
                            break
                    '''
                if _begin_index+_LEN>=len(list_sentence):
                    break
                _begin_index += _LEN
            
            list_name_freq_score = []
            # print('模型预测项目名称：', dict_name_freq_score)

            # 2020/11/23 大网站规则调整
            # if len(dict_name_freq_score) == 0:
            if len(name_list) == 0:
                # name_re1 = '(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]+([^，。：；]{2,60})[，。]'
                name_re1 = '(项目|工程|招标|采购(条目)?|合同|标项|标的|计划|询价|询价单|询价通知书|申购单|申购)(名称|标名|标题|主题)[:：\s]+(?P<name>[^，。：；]{2,60})[，。]'
                name_re2 = '(合同|采购)包\d（(?P<name>[^，。：；]{2,60})）[：，。]' # 20241202 补充合同包 包名表达 558410976
                for sentence in list_sentence:
                    # pad_sentence = sentence.sentence_text
                    othername = re.search(name_re1, sentence.sentence_text)
                    if othername == None:
                        othername = re.search(name_re2, sentence.sentence_text)
                    if othername != None:
                        project_name = othername.group('name')
                        if re.search('[\u4e00-\u9fa5]+', project_name) == None:  # 没有中文的项目名称去除
                            # log('没有中文的项目名称去除')
                            continue
                        beg = find_index([project_name], sentence.sentence_text)[0]
                        end = beg + len(project_name)
                        raw_name = sentence.sentence_text[beg:end]
                        _name = self.fitDataByRule(sentence.sentence_text[beg:end])
                        # print('规则召回项目名称：', _name)
                        # add name to entitys
                        _entity = Entity(doc_id=sentence.doc_id, entity_id="%s_%s_%s_%s" % (
                        sentence.doc_id, sentence.sentence_index, beg, end), entity_text=_name,
                                         entity_type="name", sentence_index=sentence.sentence_index, begin_index=0,
                                         end_index=0, wordOffset_begin=beg, wordOffset_end=end,in_attachment=sentence.in_attachment)
                        list_entity.append(_entity)
                        # w = 1
                        # if _name not in dict_name_freq_score:
                        #     # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
                        #     dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05), w]
                        # else:
                        #     dict_name_freq_score[_name][0] += 1
                        score = 1
                        if re.search('项目名称[:：\s]', sentence.sentence_text[beg - 8:beg]):
                            score += 1
                        elif re.search('(工程|招标|采购|合同|标项|标的|计划|询价|询价[单书]|询价通知书|申购)(名称|标题|主题|项目)[:：\s]',
                                       sentence.sentence_text[beg - 8:beg]):
                            score += 0.5
                        elif re.search('(招标|采购)(条件|范围|内容)|合同包|合同段|采购包', sentence.sentence_text[beg - 20:beg]):
                            score += 0.49
                        if raw_name in doctitle:
                            score += 0.4 + len(raw_name) / len(doctitle) / 10
                        score += (1 - sentence.sentence_index / len(list_sentence)) / 10
                        if sentence.in_attachment:
                            score /= 2
                        name_list.append((_name, score))
                # othername = re.search(name_re1, sentence.sentence_text)
                # if othername != None:
                #     _name = othername.group(3)
                #     if _name not in dict_name_freq_score:
                #         dict_name_freq_score[_name] = [1, len(re.findall(pattern_score, _name)) + len(_name) * 0.1]
                #     else:
                #         dict_name_freq_score[_name][0] += 1

            # for _name in dict_name_freq_score.keys():
            #     list_name_freq_score.append([_name,dict_name_freq_score[_name]])
            # # print(list_name_freq_score)
            # if len(list_name_freq_score)>0:
            #     list_name_freq_score.sort(key=lambda x:x[1][0]*x[1][1]*x[1][2],reverse=True)
            #     item['name'] = list_name_freq_score[0][0]
                # for it in list_name_freq_score:
                    # print('项目名称及分值：',it[0],it[1], it[1][0]*it[1][1])
                # if list_name_freq_score[0][1][0]>1:
                #     item[1]['name'] = list_name_freq_score[0][0]
                # else:
                #     list_name_freq_score.sort(key=lambda x:x[1][1],reverse=True)
                #     item[1]["name"] = list_name_freq_score[0][0]
            if name_list:
                name_list.sort(key=lambda x: x[1], reverse=True)
                item['name'] = name_list[0][0]
                
            #下面代码加上去用正则添加某些识别不到的项目编号
            if item['code'] == []:
                for sentence in list_sentence:
                    # othercode = re.search('(采购计划编号|询价编号)[\)）]?[:：]?([\[\]a-zA-Z0-9\-]{5,30})', sentence.sentence_text)
                    # if othercode != None:
                    #     item[1]['code'].append(othercode.group(2))
                    # 2020/11/23 大网站规则调整
                    in_att = 1 if sentence.in_attachment else 0
                    othercode = re.search('(项目|采购|招标|品目|询价|竞价|询价[单书]|磋商|订单|账单|交易|文件|计划|场次|标的|标段|标包|分包|标段\(包\)|招标文件|合同|通知书|公告|工程|寻源|标书|包件|谈判|申购)(单据?号|编号|标号|编码|代码|备案号|号)[:：\s]+(?P<code>[^，。；：、]{6,30}[a-zA-Z0-9\号期])[\)，。\u4e00-\u9fa5]', sentence.sentence_text)
                    if othercode != None:
                        # item['code'].append(othercode.group('code'))
                        if re.search("(项目编[号码]|招标编[号码])：?", othercode.group(0)):
                            item['code'].append((othercode.group('code'), in_att, 0, sentence.sentence_index))
                        elif re.search('采购(计划)?编[号码]：?', othercode.group(0)):
                            item['code'].append((othercode.group('code'), in_att, 1, sentence.sentence_index))
                        elif re.search('(询价|合同)编[号码]：?', othercode.group(0)):
                            item['code'].append((othercode.group('code'), in_att, 2, sentence.sentence_index))
                        elif re.search('(询价|合同|采购|招标|项目)[单标]号：?', othercode.group(0)):
                            item['code'].append((othercode.group('code'), in_att, 2.5, sentence.sentence_index))
                        else:
                            item['code'].append((othercode.group('code'), in_att, 3, sentence.sentence_index))
                        # print('规则召回项目编号：', othercode.group('code'))
            # item['code'] = [code for code in item['code'] if len(code)<500]
            # item['code'].sort(key=lambda x:len(x),reverse=True)
            item['code'] = [code for code in item['code'] if len(code[0]) < 500]
            item['code'].sort(key=lambda x: [x[1],x[2],x[3]])
            code_list = []
            for it in item['code']:
                if it[0] not in code_list:
                    code_list.append(it[0])
            # item['code'] = [it[0] for it in item['code']]
            item['code'] = code_list
            result.append(item)

            list_sentence.sort(key=lambda x: x.sentence_index,reverse=False)
        return result
                        
                
    '''
    #当数据量过大时会报错
    def predict(self,articles,MAX_LEN = None):
        sentences = []
        for article in articles:
            for sentence in article.content.split("。"):
                sentences.append([sentence,article.id])
        if MAX_LEN is None:
            sent_len = [len(sentence[0]) for sentence in sentences]
            MAX_LEN = max(sent_len)
            #print(MAX_LEN)
           
        #若为空，则直接返回空
        result = [] 
        if MAX_LEN==0:
            for article in articles:
                result.append([article.id,{"code":[],"name":""}])
            return result
        
        index_unk = self.word2index.get("<unk>")
        index_pad = self.word2index.get("<pad>")
        
        x = [[self.word2index.get(word,index_unk)for word in sentence[0]]for sentence in sentences]
        x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post")
        
        predict_y = self.getModel().predict(x)
        
        
        last_doc_id = ""
        item = []
        for sentence,predict in zip(sentences,np.argmax(predict_y,-1)):
            pad_sentence = sentence[0][:MAX_LEN]
            doc_id = sentence[1]
            join_predict = "".join([str(s) for s in predict])
            if doc_id!=last_doc_id:
                if last_doc_id!="":
                    result.append(item)
                item = [doc_id,{"code":[],"name":""}]
                code_set = set()
            code_x = []
            code_text = []
            for iter in re.finditer(self.PC_pattern,join_predict):
                get_len = 40
                if iter.span()[0]<get_len:
                    begin = 0
                else:
                    begin = iter.span()[0]-get_len
                end = iter.span()[1]+get_len
                code_x.append(embedding_word([pad_sentence[begin:iter.span()[0]],pad_sentence[iter.span()[0]:iter.span()[1]],pad_sentence[iter.span()[1]:end]],shape=(3,get_len,60)))
                code_text.append(pad_sentence[iter.span()[0]:iter.span()[1]])
            if len(code_x)>0:
                code_x = np.transpose(np.array(code_x),(1,0,2,3))
                predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
                for h in range(len(predict_code)):
                    if predict_code[h][0]>0.5:
                        the_code = self.fitDataByRule(code_text[h])
                        if the_code not in code_set:
                            code_set.add(the_code)
                            item[1]['code'] = list(code_set)
            if item[1]['name']=="":
                for iter in re.finditer(self.PN_pattern,join_predict):
                    #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
                    item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
                    break
                
            last_doc_id = doc_id
        result.append(item)
        return result
    '''