predictor.py 151 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792
  1. '''
  2. Created on 2018年12月26日
  3. @author: User
  4. '''
  5. import os
  6. import sys
  7. from BiddingKG.dl.common.nerUtils import *
  8. sys.path.append(os.path.abspath("../.."))
  9. # from keras.engine import topology
  10. # from keras import models
  11. # from keras import layers
  12. # from keras_contrib.layers.crf import CRF
  13. # from keras.preprocessing.sequence import pad_sequences
  14. # from keras import optimizers,losses,metrics
  15. from BiddingKG.dl.common.Utils import *
  16. from BiddingKG.dl.interface.modelFactory import *
  17. import tensorflow as tf
  18. from BiddingKG.dl.product.data_util import decode, process_data
  19. from BiddingKG.dl.interface.Entitys import Entity
  20. from BiddingKG.dl.complaint.punish_predictor import Punish_Extract
  21. from BiddingKG.dl.money.re_money_total_unit import extract_total_money, extract_unit_money
  22. from bs4 import BeautifulSoup
  23. import copy
  24. import calendar
  25. import datetime
  26. from threading import RLock
  27. dict_predictor = {"codeName":{"predictor":None,"Lock":RLock()},
  28. "prem":{"predictor":None,"Lock":RLock()},
  29. "epc":{"predictor":None,"Lock":RLock()},
  30. "roleRule":{"predictor":None,"Lock":RLock()},
  31. "roleRuleFinal":{"predictor":None,"Lock":RLock()},
  32. "form":{"predictor":None,"Lock":RLock()},
  33. "time":{"predictor":None,"Lock":RLock()},
  34. "punish":{"predictor":None,"Lock":RLock()},
  35. "product":{"predictor":None,"Lock":RLock()},
  36. "product_attrs":{"predictor":None,"Lock":RLock()},
  37. "channel": {"predictor": None, "Lock": RLock()},
  38. "deposit_payment_way": {"predictor": None, "Lock": RLock()},
  39. "total_unit_money": {"predictor": None, "Lock": RLock()}
  40. }
  41. def getPredictor(_type):
  42. if _type in dict_predictor:
  43. with dict_predictor[_type]["Lock"]:
  44. if dict_predictor[_type]["predictor"] is None:
  45. if _type == "codeName":
  46. dict_predictor[_type]["predictor"] = CodeNamePredict()
  47. if _type == "prem":
  48. dict_predictor[_type]["predictor"] = PREMPredict()
  49. if _type == "epc":
  50. dict_predictor[_type]["predictor"] = EPCPredict()
  51. if _type == "roleRule":
  52. dict_predictor[_type]["predictor"] = RoleRulePredictor()
  53. if _type == "roleRuleFinal":
  54. dict_predictor[_type]["predictor"] = RoleRuleFinalAdd()
  55. if _type == "form":
  56. dict_predictor[_type]["predictor"] = FormPredictor()
  57. if _type == "time":
  58. dict_predictor[_type]["predictor"] = TimePredictor()
  59. if _type == "punish":
  60. dict_predictor[_type]["predictor"] = Punish_Extract()
  61. if _type == "product":
  62. dict_predictor[_type]["predictor"] = ProductPredictor()
  63. if _type == "product_attrs":
  64. dict_predictor[_type]["predictor"] = ProductAttributesPredictor()
  65. if _type == "channel":
  66. dict_predictor[_type]["predictor"] = DocChannel()
  67. if _type == 'deposit_payment_way':
  68. dict_predictor[_type]["predictor"] = DepositPaymentWay()
  69. if _type == 'total_unit_money':
  70. dict_predictor[_type]["predictor"] = TotalUnitMoney()
  71. return dict_predictor[_type]["predictor"]
  72. raise NameError("no this type of predictor")
  73. # 编号名称模型
  74. class CodeNamePredict():
  75. def __init__(self,EMBED_DIM=None,BiRNN_UNITS=None,lazyLoad=getLazyLoad()):
  76. self.model = None
  77. self.MAX_LEN = None
  78. self.model_code = None
  79. if EMBED_DIM is None:
  80. self.EMBED_DIM = 60
  81. else:
  82. self.EMBED_DIM = EMBED_DIM
  83. if BiRNN_UNITS is None:
  84. self.BiRNN_UNITS = 200
  85. else:
  86. self.BiRNN_UNITS = BiRNN_UNITS
  87. self.filepath = os.path.dirname(__file__)+"/../projectCode/models/model_project_"+str(self.EMBED_DIM)+"_"+str(self.BiRNN_UNITS)+".hdf5"
  88. #self.filepath = "../projectCode/models/model_project_60_200_200ep017-loss6.456-val_loss7.852-val_acc0.969.hdf5"
  89. self.filepath_code = os.path.dirname(__file__)+"/../projectCode/models/model_code.hdf5"
  90. vocabpath = os.path.dirname(__file__)+"/codename_vocab.pk"
  91. classlabelspath = os.path.dirname(__file__)+"/codename_classlabels.pk"
  92. self.vocab = load(vocabpath)
  93. self.class_labels = load(classlabelspath)
  94. #生成提取编号和名称的正则
  95. id_PC_B = self.class_labels.index("PC_B")
  96. id_PC_M = self.class_labels.index("PC_M")
  97. id_PC_E = self.class_labels.index("PC_E")
  98. id_PN_B = self.class_labels.index("PN_B")
  99. id_PN_M = self.class_labels.index("PN_M")
  100. id_PN_E = self.class_labels.index("PN_E")
  101. self.PC_pattern = re.compile(str(id_PC_B)+str(id_PC_M)+"*"+str(id_PC_E))
  102. self.PN_pattern = re.compile(str(id_PN_B)+str(id_PN_M)+"*"+str(id_PN_E))
  103. # print("pc",self.PC_pattern)
  104. # print("pn",self.PN_pattern)
  105. self.word2index = dict((w,i) for i,w in enumerate(np.array(self.vocab)))
  106. self.inputs = None
  107. self.outputs = None
  108. self.sess_codename = tf.Session(graph=tf.Graph())
  109. self.sess_codesplit = tf.Session(graph=tf.Graph())
  110. self.inputs_code = None
  111. self.outputs_code = None
  112. if not lazyLoad:
  113. self.getModel()
  114. self.getModel_code()
  115. def getModel(self):
  116. '''
  117. @summary: 取得编号和名称模型
  118. '''
  119. if self.inputs is None:
  120. log("get model of codename")
  121. with self.sess_codename.as_default():
  122. with self.sess_codename.graph.as_default():
  123. meta_graph_def = tf.saved_model.loader.load(self.sess_codename, ["serve"], export_dir=os.path.dirname(__file__)+"/codename_savedmodel_tf")
  124. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  125. signature_def = meta_graph_def.signature_def
  126. self.inputs = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
  127. self.inputs_length = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs_length"].name)
  128. self.keepprob = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].inputs["keepprob"].name)
  129. self.logits = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["logits"].name)
  130. self.trans = self.sess_codename.graph.get_tensor_by_name(signature_def[signature_key].outputs["trans"].name)
  131. return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans
  132. else:
  133. return self.inputs,self.inputs_length,self.keepprob,self.logits,self.trans
  134. '''
  135. if self.model is None:
  136. self.model = self.getBiLSTMCRFModel(self.MAX_LEN, self.vocab, self.EMBED_DIM, self.BiRNN_UNITS, self.class_labels,weights=None)
  137. self.model.load_weights(self.filepath)
  138. return self.model
  139. '''
  140. def getModel_code(self):
  141. if self.inputs_code is None:
  142. log("get model of code")
  143. with self.sess_codesplit.as_default():
  144. with self.sess_codesplit.graph.as_default():
  145. meta_graph_def = tf.saved_model.loader.load(self.sess_codesplit, ["serve"], export_dir=os.path.dirname(__file__)+"/codesplit_savedmodel")
  146. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  147. signature_def = meta_graph_def.signature_def
  148. self.inputs_code = []
  149. self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name))
  150. self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name))
  151. self.inputs_code.append(self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name))
  152. self.outputs_code = self.sess_codesplit.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  153. self.sess_codesplit.graph.finalize()
  154. return self.inputs_code,self.outputs_code
  155. else:
  156. return self.inputs_code,self.outputs_code
  157. '''
  158. if self.model_code is None:
  159. log("get model of model_code")
  160. with self.sess_codesplit.as_default():
  161. with self.sess_codesplit.graph.as_default():
  162. self.model_code = models.load_model(self.filepath_code, custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  163. return self.model_code
  164. '''
  165. def getBiLSTMCRFModel(self,MAX_LEN,vocab,EMBED_DIM,BiRNN_UNITS,chunk_tags,weights):
  166. '''
  167. model = models.Sequential()
  168. model.add(layers.Embedding(len(vocab), EMBED_DIM, mask_zero=True)) # Random embedding
  169. model.add(layers.Bidirectional(layers.LSTM(BiRNN_UNITS // 2, return_sequences=True)))
  170. crf = CRF(len(chunk_tags), sparse_target=True)
  171. model.add(crf)
  172. model.summary()
  173. model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
  174. return model
  175. '''
  176. input = layers.Input(shape=(None,))
  177. if weights is not None:
  178. embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True,weights=[weights],trainable=True)(input)
  179. else:
  180. embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True)(input)
  181. bilstm = layers.Bidirectional(layers.LSTM(BiRNN_UNITS//2,return_sequences=True))(embedding)
  182. bilstm_dense = layers.TimeDistributed(layers.Dense(len(chunk_tags)))(bilstm)
  183. crf = CRF(len(chunk_tags),sparse_target=True)
  184. crf_out = crf(bilstm_dense)
  185. model = models.Model(input=[input],output = [crf_out])
  186. model.summary()
  187. model.compile(optimizer = 'adam', loss = crf.loss_function, metrics = [crf.accuracy])
  188. return model
  189. #根据规则补全编号或名称两边的符号
  190. def fitDataByRule(self,data):
  191. symbol_dict = {"(":")",
  192. "(":")",
  193. "[":"]",
  194. "【":"】",
  195. ")":"(",
  196. ")":"(",
  197. "]":"[",
  198. "】":"【"}
  199. leftSymbol_pattern = re.compile("[\((\[【]")
  200. rightSymbol_pattern = re.compile("[\))\]】]")
  201. leftfinds = re.findall(leftSymbol_pattern,data)
  202. rightfinds = re.findall(rightSymbol_pattern,data)
  203. result = data
  204. if len(leftfinds)+len(rightfinds)==0:
  205. return data
  206. elif len(leftfinds)==len(rightfinds):
  207. return data
  208. elif abs(len(leftfinds)-len(rightfinds))==1:
  209. if len(leftfinds)>len(rightfinds):
  210. if symbol_dict.get(data[0]) is not None:
  211. result = data[1:]
  212. else:
  213. #print(symbol_dict.get(leftfinds[0]))
  214. result = data+symbol_dict.get(leftfinds[0])
  215. else:
  216. if symbol_dict.get(data[-1]) is not None:
  217. result = data[:-1]
  218. else:
  219. result = symbol_dict.get(rightfinds[0])+data
  220. return result
  221. def decode(self,logits, trans, sequence_lengths, tag_num):
  222. viterbi_sequences = []
  223. for logit, length in zip(logits, sequence_lengths):
  224. score = logit[:length]
  225. viterbi_seq, viterbi_score = viterbi_decode(score, trans)
  226. viterbi_sequences.append(viterbi_seq)
  227. return viterbi_sequences
  228. def predict(self,list_sentences,list_entitys=None,MAX_AREA = 5000):
  229. #@summary: 获取每篇文章的code和name
  230. pattern_score = re.compile("工程|服务|采购|施工|项目|系统|招标|中标|公告|学校|[大中小]学校?|医院|公司|分公司|研究院|政府采购中心|学院|中心校?|办公室|政府|财[政务]局|办事处|委员会|[部总支]队|警卫局|幼儿园|党委|党校|银行|分行|解放军|发电厂|供电局|管理所|供电公司|卷烟厂|机务段|研究[院所]|油厂|调查局|调查中心|出版社|电视台|监狱|水厂|服务站|信用合作联社|信用社|交易所|交易中心|交易中心党校|科学院|测绘所|运输厅|管理处|局|中心|机关|部门?|处|科|厂|集团|图书馆|馆|所|厅|楼|区|酒店|场|基地|矿|餐厅|酒店")
  231. result = []
  232. index_unk = self.word2index.get("<unk>")
  233. # index_pad = self.word2index.get("<pad>")
  234. if list_entitys is None:
  235. list_entitys = [[] for _ in range(len(list_sentences))]
  236. for list_sentence,list_entity in zip(list_sentences,list_entitys):
  237. if len(list_sentence)==0:
  238. result.append([{"code":[],"name":""}])
  239. continue
  240. doc_id = list_sentence[0].doc_id
  241. # sentences = []
  242. # for sentence in list_sentence:
  243. # if len(sentence.sentence_text)>MAX_AREA:
  244. # for _sentence_comma in re.split("[;;,\n]",sentence):
  245. # _comma_index = 0
  246. # while(_comma_index<len(_sentence_comma)):
  247. # sentences.append(_sentence_comma[_comma_index:_comma_index+MAX_AREA])
  248. # _comma_index += MAX_AREA
  249. # else:
  250. # sentences.append(sentence+"。")
  251. list_sentence.sort(key=lambda x:len(x.sentence_text),reverse=True)
  252. _begin_index = 0
  253. item = {"code":[],"name":""}
  254. code_set = set()
  255. dict_name_freq_score = dict()
  256. while(True):
  257. MAX_LEN = len(list_sentence[_begin_index].sentence_text)
  258. if MAX_LEN>MAX_AREA:
  259. MAX_LEN = MAX_AREA
  260. _LEN = MAX_AREA//MAX_LEN
  261. #预测
  262. x = [[self.word2index.get(word,index_unk)for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]]
  263. # x = [[getIndexOfWord(word) for word in sentence.sentence_text[:MAX_AREA]]for sentence in list_sentence[_begin_index:_begin_index+_LEN]]
  264. x_len = [len(_x) if len(_x) < MAX_LEN else MAX_LEN for _x in x]
  265. x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post")
  266. if USE_PAI_EAS:
  267. request = tf_predict_pb2.PredictRequest()
  268. request.inputs["inputs"].dtype = tf_predict_pb2.DT_INT32
  269. request.inputs["inputs"].array_shape.dim.extend(np.shape(x))
  270. request.inputs["inputs"].int_val.extend(np.array(x,dtype=np.int32).reshape(-1))
  271. request_data = request.SerializeToString()
  272. list_outputs = ["outputs"]
  273. _result = vpc_requests(codename_url, codename_authorization, request_data, list_outputs)
  274. if _result is not None:
  275. predict_y = _result["outputs"]
  276. else:
  277. with self.sess_codename.as_default():
  278. t_input,t_output = self.getModel()
  279. predict_y = self.sess_codename.run(t_output,feed_dict={t_input:x})
  280. else:
  281. with self.sess_codename.as_default():
  282. t_input,t_input_length,t_keepprob,t_logits,t_trans = self.getModel()
  283. _logits,_trans = self.sess_codename.run([t_logits,t_trans],feed_dict={t_input:x,
  284. t_input_length:x_len,
  285. t_keepprob:1.0})
  286. predict_y = self.decode(_logits,_trans,x_len,7)
  287. # print('==========',_logits)
  288. '''
  289. for item11 in np.argmax(predict_y,-1):
  290. print(item11)
  291. print(predict_y)
  292. '''
  293. # print(predict_y)
  294. for sentence,predict in zip(list_sentence[_begin_index:_begin_index+_LEN],np.array(predict_y)):
  295. pad_sentence = sentence.sentence_text[:MAX_LEN]
  296. join_predict = "".join([str(s) for s in predict])
  297. # print(pad_sentence)
  298. # print(join_predict)
  299. code_x = []
  300. code_text = []
  301. temp_entitys = []
  302. for iter in re.finditer(self.PC_pattern,join_predict):
  303. get_len = 40
  304. if iter.span()[0]<get_len:
  305. begin = 0
  306. else:
  307. begin = iter.span()[0]-get_len
  308. end = iter.span()[1]+get_len
  309. code_x.append(embedding_word([pad_sentence[begin:iter.span()[0]],pad_sentence[iter.span()[0]:iter.span()[1]],pad_sentence[iter.span()[1]:end]],shape=(3,get_len,60)))
  310. code_text.append(pad_sentence[iter.span()[0]:iter.span()[1]])
  311. _entity = Entity(doc_id=sentence.doc_id,entity_id="%s_%s_%s_%s"%(sentence.doc_id,sentence.sentence_index,iter.span()[0],iter.span()[1]),entity_text=pad_sentence[iter.span()[0]:iter.span()[1]],entity_type="code",sentence_index=sentence.sentence_index,begin_index=0,end_index=0,wordOffset_begin=iter.span()[0],wordOffset_end=iter.span()[1])
  312. temp_entitys.append(_entity)
  313. #print("code",code_text)
  314. if len(code_x)>0:
  315. code_x = np.transpose(np.array(code_x,dtype=np.float32),(1,0,2,3))
  316. if USE_PAI_EAS:
  317. request = tf_predict_pb2.PredictRequest()
  318. request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT
  319. request.inputs["input0"].array_shape.dim.extend(np.shape(code_x[0]))
  320. request.inputs["input0"].float_val.extend(np.array(code_x[0],dtype=np.float64).reshape(-1))
  321. request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT
  322. request.inputs["input1"].array_shape.dim.extend(np.shape(code_x[1]))
  323. request.inputs["input1"].float_val.extend(np.array(code_x[1],dtype=np.float64).reshape(-1))
  324. request.inputs["input2"].dtype = tf_predict_pb2.DT_FLOAT
  325. request.inputs["input2"].array_shape.dim.extend(np.shape(code_x[2]))
  326. request.inputs["input2"].float_val.extend(np.array(code_x[2],dtype=np.float64).reshape(-1))
  327. request_data = request.SerializeToString()
  328. list_outputs = ["outputs"]
  329. _result = vpc_requests(codeclasses_url, codeclasses_authorization, request_data, list_outputs)
  330. if _result is not None:
  331. predict_code = _result["outputs"]
  332. else:
  333. with self.sess_codesplit.as_default():
  334. with self.sess_codesplit.graph.as_default():
  335. predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
  336. else:
  337. with self.sess_codesplit.as_default():
  338. with self.sess_codesplit.graph.as_default():
  339. inputs_code,outputs_code = self.getModel_code()
  340. predict_code = limitRun(self.sess_codesplit,[outputs_code],feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]},MAX_BATCH=2)[0]
  341. #predict_code = self.sess_codesplit.run(outputs_code,feed_dict={inputs_code[0]:code_x[0],inputs_code[1]:code_x[1],inputs_code[2]:code_x[2]})
  342. #predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
  343. for h in range(len(predict_code)):
  344. if predict_code[h][0]>0.5:
  345. the_code = self.fitDataByRule(code_text[h])
  346. #add code to entitys
  347. list_entity.append(temp_entitys[h])
  348. if the_code not in code_set:
  349. code_set.add(the_code)
  350. item['code'] = list(code_set)
  351. for iter in re.finditer(self.PN_pattern,join_predict):
  352. _name = self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
  353. #add name to entitys
  354. _entity = Entity(doc_id=sentence.doc_id,entity_id="%s_%s_%s_%s"%(sentence.doc_id,sentence.sentence_index,iter.span()[0],iter.span()[1]),entity_text=_name,entity_type="name",sentence_index=sentence.sentence_index,begin_index=0,end_index=0,wordOffset_begin=iter.span()[0],wordOffset_end=iter.span()[1])
  355. list_entity.append(_entity)
  356. w = 1 if re.search('(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[::\s]', pad_sentence[iter.span()[0]-10:iter.span()[0]])!=None else 0.5
  357. if _name not in dict_name_freq_score:
  358. # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
  359. dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05)*w]
  360. else:
  361. dict_name_freq_score[_name][0] += 1
  362. '''
  363. for iter in re.finditer(self.PN_pattern,join_predict):
  364. print("name-",self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]]))
  365. if item[1]['name']=="":
  366. for iter in re.finditer(self.PN_pattern,join_predict):
  367. #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
  368. item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
  369. break
  370. '''
  371. if _begin_index+_LEN>=len(list_sentence):
  372. break
  373. _begin_index += _LEN
  374. list_name_freq_score = []
  375. # 2020/11/23 大网站规则调整
  376. if len(dict_name_freq_score) == 0:
  377. name_re1 = '(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[::\s]+([^,。:;]{2,60})[,。]'
  378. for sentence in list_sentence:
  379. # pad_sentence = sentence.sentence_text
  380. othername = re.search(name_re1, sentence.sentence_text)
  381. if othername != None:
  382. project_name = othername.group(3)
  383. beg = find_index([project_name], sentence.sentence_text)[0]
  384. end = beg + len(project_name)
  385. _name = self.fitDataByRule(sentence.sentence_text[beg:end])
  386. # add name to entitys
  387. _entity = Entity(doc_id=sentence.doc_id, entity_id="%s_%s_%s_%s" % (
  388. sentence.doc_id, sentence.sentence_index, beg, end), entity_text=_name,
  389. entity_type="name", sentence_index=sentence.sentence_index, begin_index=0,
  390. end_index=0, wordOffset_begin=beg, wordOffset_end=end)
  391. list_entity.append(_entity)
  392. w = 1
  393. if _name not in dict_name_freq_score:
  394. # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
  395. dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05) * w]
  396. else:
  397. dict_name_freq_score[_name][0] += 1
  398. # othername = re.search(name_re1, sentence.sentence_text)
  399. # if othername != None:
  400. # _name = othername.group(3)
  401. # if _name not in dict_name_freq_score:
  402. # dict_name_freq_score[_name] = [1, len(re.findall(pattern_score, _name)) + len(_name) * 0.1]
  403. # else:
  404. # dict_name_freq_score[_name][0] += 1
  405. for _name in dict_name_freq_score.keys():
  406. list_name_freq_score.append([_name,dict_name_freq_score[_name]])
  407. # print(list_name_freq_score)
  408. if len(list_name_freq_score)>0:
  409. list_name_freq_score.sort(key=lambda x:x[1][0]*x[1][1],reverse=True)
  410. item['name'] = list_name_freq_score[0][0]
  411. # if list_name_freq_score[0][1][0]>1:
  412. # item[1]['name'] = list_name_freq_score[0][0]
  413. # else:
  414. # list_name_freq_score.sort(key=lambda x:x[1][1],reverse=True)
  415. # item[1]["name"] = list_name_freq_score[0][0]
  416. #下面代码加上去用正则添加某些识别不到的项目编号
  417. if item['code'] == []:
  418. for sentence in list_sentence:
  419. # othercode = re.search('(采购计划编号|询价编号)[\))]?[::]?([\[\]a-zA-Z0-9\-]{5,30})', sentence.sentence_text)
  420. # if othercode != None:
  421. # item[1]['code'].append(othercode.group(2))
  422. # 2020/11/23 大网站规则调整
  423. othercode = re.search('(项目|采购|招标|品目|询价|竞价|询价单|磋商|订单|账单|交易|文件|计划|场次|标的|标段|标包|分包|标段\(包\)|招标文件|合同|通知书|公告)(单号|编号|标号|编码|代码|备案号|号)[::\s]+([^,。;:、]{8,30}[a-zA-Z0-9\号])[\),。]', sentence.sentence_text)
  424. if othercode != None:
  425. item['code'].append(othercode.group(3))
  426. item['code'].sort(key=lambda x:len(x),reverse=True)
  427. result.append(item)
  428. list_sentence.sort(key=lambda x: x.sentence_index,reverse=False)
  429. return result
  430. '''
  431. #当数据量过大时会报错
  432. def predict(self,articles,MAX_LEN = None):
  433. sentences = []
  434. for article in articles:
  435. for sentence in article.content.split("。"):
  436. sentences.append([sentence,article.id])
  437. if MAX_LEN is None:
  438. sent_len = [len(sentence[0]) for sentence in sentences]
  439. MAX_LEN = max(sent_len)
  440. #print(MAX_LEN)
  441. #若为空,则直接返回空
  442. result = []
  443. if MAX_LEN==0:
  444. for article in articles:
  445. result.append([article.id,{"code":[],"name":""}])
  446. return result
  447. index_unk = self.word2index.get("<unk>")
  448. index_pad = self.word2index.get("<pad>")
  449. x = [[self.word2index.get(word,index_unk)for word in sentence[0]]for sentence in sentences]
  450. x = pad_sequences(x,maxlen=MAX_LEN,padding="post",truncating="post")
  451. predict_y = self.getModel().predict(x)
  452. last_doc_id = ""
  453. item = []
  454. for sentence,predict in zip(sentences,np.argmax(predict_y,-1)):
  455. pad_sentence = sentence[0][:MAX_LEN]
  456. doc_id = sentence[1]
  457. join_predict = "".join([str(s) for s in predict])
  458. if doc_id!=last_doc_id:
  459. if last_doc_id!="":
  460. result.append(item)
  461. item = [doc_id,{"code":[],"name":""}]
  462. code_set = set()
  463. code_x = []
  464. code_text = []
  465. for iter in re.finditer(self.PC_pattern,join_predict):
  466. get_len = 40
  467. if iter.span()[0]<get_len:
  468. begin = 0
  469. else:
  470. begin = iter.span()[0]-get_len
  471. end = iter.span()[1]+get_len
  472. code_x.append(embedding_word([pad_sentence[begin:iter.span()[0]],pad_sentence[iter.span()[0]:iter.span()[1]],pad_sentence[iter.span()[1]:end]],shape=(3,get_len,60)))
  473. code_text.append(pad_sentence[iter.span()[0]:iter.span()[1]])
  474. if len(code_x)>0:
  475. code_x = np.transpose(np.array(code_x),(1,0,2,3))
  476. predict_code = self.getModel_code().predict([code_x[0],code_x[1],code_x[2]])
  477. for h in range(len(predict_code)):
  478. if predict_code[h][0]>0.5:
  479. the_code = self.fitDataByRule(code_text[h])
  480. if the_code not in code_set:
  481. code_set.add(the_code)
  482. item[1]['code'] = list(code_set)
  483. if item[1]['name']=="":
  484. for iter in re.finditer(self.PN_pattern,join_predict):
  485. #item[1]['name']=item[1]['name']+";"+self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
  486. item[1]['name']=self.fitDataByRule(pad_sentence[iter.span()[0]:iter.span()[1]])
  487. break
  488. last_doc_id = doc_id
  489. result.append(item)
  490. return result
  491. '''
  492. #角色金额模型
  493. class PREMPredict():
  494. def __init__(self):
  495. #self.model_role_file = os.path.abspath("../role/models/model_role.model.hdf5")
  496. self.model_role_file = os.path.dirname(__file__)+"/../role/log/new_biLSTM-ep012-loss0.028-val_loss0.040-f10.954.h5"
  497. self.model_role = Model_role_classify_word()
  498. self.model_money = Model_money_classify()
  499. return
  500. def search_role_data(self,list_sentences,list_entitys):
  501. '''
  502. @summary:根据句子list和实体list查询角色模型的输入数据
  503. @param:
  504. list_sentences:文章的sentences
  505. list_entitys:文章的entitys
  506. @return:角色模型的输入数据
  507. '''
  508. text_list = []
  509. data_x = []
  510. points_entitys = []
  511. for list_entity,list_sentence in zip(list_entitys,list_sentences):
  512. list_entity.sort(key=lambda x:x.sentence_index)
  513. list_sentence.sort(key=lambda x:x.sentence_index)
  514. p_entitys = 0
  515. p_sentences = 0
  516. while(p_entitys<len(list_entity)):
  517. entity = list_entity[p_entitys]
  518. if entity.entity_type in ['org','company']:
  519. while(p_sentences<len(list_sentence)):
  520. sentence = list_sentence[p_sentences]
  521. if entity.doc_id==sentence.doc_id and entity.sentence_index==sentence.sentence_index:
  522. text_list.append(sentence.sentence_text[max(0, entity.wordOffset_begin-10):entity.wordOffset_end+10])
  523. #item_x = embedding(spanWindow(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,size=settings.MODEL_ROLE_INPUT_SHAPE[1]),shape=settings.MODEL_ROLE_INPUT_SHAPE)
  524. item_x = self.model_role.encode(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,entity_text=entity.entity_text)
  525. data_x.append(item_x)
  526. points_entitys.append(entity)
  527. break
  528. p_sentences += 1
  529. p_entitys += 1
  530. if len(points_entitys)==0:
  531. return None
  532. return [data_x,points_entitys, text_list]
  533. def search_money_data(self,list_sentences,list_entitys):
  534. '''
  535. @summary:根据句子list和实体list查询金额模型的输入数据
  536. @param:
  537. list_sentences:文章的sentences
  538. list_entitys:文章的entitys
  539. @return:金额模型的输入数据
  540. '''
  541. text_list = []
  542. data_x = []
  543. points_entitys = []
  544. for list_entity,list_sentence in zip(list_entitys,list_sentences):
  545. list_entity.sort(key=lambda x:x.sentence_index)
  546. list_sentence.sort(key=lambda x:x.sentence_index)
  547. p_entitys = 0
  548. while(p_entitys<len(list_entity)):
  549. entity = list_entity[p_entitys]
  550. if entity.entity_type=="money":
  551. p_sentences = 0
  552. while(p_sentences<len(list_sentence)):
  553. sentence = list_sentence[p_sentences]
  554. if entity.doc_id==sentence.doc_id and entity.sentence_index==sentence.sentence_index:
  555. text_list.append(sentence.sentence_text[max(0, entity.wordOffset_begin - 8):entity.wordOffset_end])
  556. #item_x = embedding(spanWindow(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,size=settings.MODEL_MONEY_INPUT_SHAPE[1]),shape=settings.MODEL_MONEY_INPUT_SHAPE)
  557. #item_x = embedding_word(spanWindow(tokens=sentence.tokens, begin_index=entity.begin_index, end_index=entity.end_index, size=10, center_include=True, word_flag=True),shape=settings.MODEL_MONEY_INPUT_SHAPE)
  558. item_x = self.model_money.encode(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index)
  559. data_x.append(item_x)
  560. points_entitys.append(entity)
  561. break
  562. p_sentences += 1
  563. p_entitys += 1
  564. if len(points_entitys)==0:
  565. return None
  566. return [data_x,points_entitys, text_list]
  567. def predict_role(self,list_sentences, list_entitys):
  568. datas = self.search_role_data(list_sentences, list_entitys)
  569. if datas is None:
  570. return
  571. points_entitys = datas[1]
  572. text_list = datas[2]
  573. if USE_PAI_EAS:
  574. _data = datas[0]
  575. _data = np.transpose(np.array(_data),(1,0,2))
  576. request = tf_predict_pb2.PredictRequest()
  577. request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT
  578. request.inputs["input0"].array_shape.dim.extend(np.shape(_data[0]))
  579. request.inputs["input0"].float_val.extend(np.array(_data[0],dtype=np.float64).reshape(-1))
  580. request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT
  581. request.inputs["input1"].array_shape.dim.extend(np.shape(_data[1]))
  582. request.inputs["input1"].float_val.extend(np.array(_data[1],dtype=np.float64).reshape(-1))
  583. request.inputs["input2"].dtype = tf_predict_pb2.DT_FLOAT
  584. request.inputs["input2"].array_shape.dim.extend(np.shape(_data[2]))
  585. request.inputs["input2"].float_val.extend(np.array(_data[2],dtype=np.float64).reshape(-1))
  586. request_data = request.SerializeToString()
  587. list_outputs = ["outputs"]
  588. _result = vpc_requests(role_url, role_authorization, request_data, list_outputs)
  589. if _result is not None:
  590. predict_y = _result["outputs"]
  591. else:
  592. predict_y = self.model_role.predict(datas[0])
  593. else:
  594. predict_y = self.model_role.predict(np.array(datas[0],dtype=np.float64))
  595. for i in range(len(predict_y)):
  596. entity = points_entitys[i]
  597. label = np.argmax(predict_y[i])
  598. values = predict_y[i]
  599. text = text_list[i]
  600. if label == 2:
  601. if re.search('中标单位和.{,25}签订合同', text):
  602. label = 0
  603. values[label] = 0.501
  604. elif re.search('尊敬的供应商:.{,25}我公司', text):
  605. label = 0
  606. values[label] = 0.801
  607. elif label == 1 and re.search('委托(单位|人|方)[是为:]+', text[:10]) and re.search('受委托(单位|人|方)[是为:]+', text[:10])==None:
  608. label = 0
  609. values[label] = 0.501
  610. elif label == 1 and re.search('([,。:]|^)(服务|中选)机构(名称)?', text[:-10]):
  611. label = 2
  612. values[label] = 0.501
  613. entity.set_Role(label, values)
  614. def predict_money(self,list_sentences,list_entitys):
  615. datas = self.search_money_data(list_sentences, list_entitys)
  616. if datas is None:
  617. return
  618. points_entitys = datas[1]
  619. _data = datas[0]
  620. text_list = datas[2]
  621. if USE_PAI_EAS:
  622. _data = np.transpose(np.array(_data),(1,0,2,3))
  623. request = tf_predict_pb2.PredictRequest()
  624. request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT
  625. request.inputs["input0"].array_shape.dim.extend(np.shape(_data[0]))
  626. request.inputs["input0"].float_val.extend(np.array(_data[0],dtype=np.float64).reshape(-1))
  627. request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT
  628. request.inputs["input1"].array_shape.dim.extend(np.shape(_data[1]))
  629. request.inputs["input1"].float_val.extend(np.array(_data[1],dtype=np.float64).reshape(-1))
  630. request.inputs["input2"].dtype = tf_predict_pb2.DT_FLOAT
  631. request.inputs["input2"].array_shape.dim.extend(np.shape(_data[2]))
  632. request.inputs["input2"].float_val.extend(np.array(_data[2],dtype=np.float64).reshape(-1))
  633. request_data = request.SerializeToString()
  634. list_outputs = ["outputs"]
  635. _result = vpc_requests(money_url, money_authorization, request_data, list_outputs)
  636. if _result is not None:
  637. predict_y = _result["outputs"]
  638. else:
  639. predict_y = self.model_money.predict(_data)
  640. else:
  641. predict_y = self.model_money.predict(_data)
  642. for i in range(len(predict_y)):
  643. entity = points_entitys[i]
  644. label = np.argmax(predict_y[i])
  645. values = predict_y[i]
  646. text = text_list[i]
  647. if label == 1 and re.search('[::,。](总金额|总价|单价)', text):
  648. values[label] = 0.49
  649. elif label ==0 and entity.notes in ["投资", "工程造价"]:
  650. values[label] = 0.49
  651. entity.set_Money(label, values)
  652. def predict(self,list_sentences,list_entitys):
  653. self.predict_role(list_sentences,list_entitys)
  654. self.predict_money(list_sentences,list_entitys)
  655. #联系人模型
  656. class EPCPredict():
  657. def __init__(self):
  658. self.model_person = Model_person_classify()
  659. def search_person_data(self,list_sentences,list_entitys):
  660. '''
  661. @summary:根据句子list和实体list查询联系人模型的输入数据
  662. @param:
  663. list_sentences:文章的sentences
  664. list_entitys:文章的entitys
  665. @return:联系人模型的输入数据
  666. '''
  667. data_x = []
  668. points_entitys = []
  669. for list_entity,list_sentence in zip(list_entitys,list_sentences):
  670. p_entitys = 0
  671. dict_index_sentence = {}
  672. for _sentence in list_sentence:
  673. dict_index_sentence[_sentence.sentence_index] = _sentence
  674. _list_entity = [entity for entity in list_entity if entity.entity_type=="person"]
  675. while(p_entitys<len(_list_entity)):
  676. entity = _list_entity[p_entitys]
  677. if entity.entity_type=="person":
  678. sentence = dict_index_sentence[entity.sentence_index]
  679. item_x = self.model_person.encode(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index)
  680. data_x.append(item_x)
  681. points_entitys.append(entity)
  682. p_entitys += 1
  683. if len(points_entitys)==0:
  684. return None
  685. # return [data_x,points_entitys,dianhua]
  686. return [data_x,points_entitys]
  687. def predict_person(self,list_sentences, list_entitys):
  688. datas = self.search_person_data(list_sentences, list_entitys)
  689. if datas is None:
  690. return
  691. points_entitys = datas[1]
  692. # phone = datas[2]
  693. if USE_PAI_EAS:
  694. _data = datas[0]
  695. _data = np.transpose(np.array(_data),(1,0,2,3))
  696. request = tf_predict_pb2.PredictRequest()
  697. request.inputs["input0"].dtype = tf_predict_pb2.DT_FLOAT
  698. request.inputs["input0"].array_shape.dim.extend(np.shape(_data[0]))
  699. request.inputs["input0"].float_val.extend(np.array(_data[0],dtype=np.float64).reshape(-1))
  700. request.inputs["input1"].dtype = tf_predict_pb2.DT_FLOAT
  701. request.inputs["input1"].array_shape.dim.extend(np.shape(_data[1]))
  702. request.inputs["input1"].float_val.extend(np.array(_data[1],dtype=np.float64).reshape(-1))
  703. request_data = request.SerializeToString()
  704. list_outputs = ["outputs"]
  705. _result = vpc_requests(person_url, person_authorization, request_data, list_outputs)
  706. if _result is not None:
  707. predict_y = _result["outputs"]
  708. else:
  709. predict_y = self.model_person.predict(datas[0])
  710. else:
  711. predict_y = self.model_person.predict(datas[0])
  712. # assert len(predict_y)==len(points_entitys)==len(phone)
  713. assert len(predict_y)==len(points_entitys)
  714. for i in range(len(predict_y)):
  715. entity = points_entitys[i]
  716. label = np.argmax(predict_y[i])
  717. values = []
  718. for item in predict_y[i]:
  719. values.append(item)
  720. # phone_number = phone[i]
  721. # entity.set_Person(label,values,phone_number)
  722. entity.set_Person(label,values,[])
  723. # 为联系人匹配电话
  724. # self.person_search_phone(list_sentences, list_entitys)
  725. def person_search_phone(self,list_sentences, list_entitys):
  726. def phoneFromList(phones):
  727. # for phone in phones:
  728. # if len(phone)==11:
  729. # return re.sub('电话[:|:]|联系方式[:|:]','',phone)
  730. return re.sub('电话[:|:]|联系方式[:|:]', '', phones[0])
  731. for list_entity, list_sentence in zip(list_entitys, list_sentences):
  732. # p_entitys = 0
  733. # p_sentences = 0
  734. #
  735. # key_word = re.compile('电话[:|:].{0,4}\d{7,12}|联系方式[:|:].{0,4}\d{7,12}')
  736. # # phone = re.compile('1[3|4|5|7|8][0-9][-—-]?\d{4}[-—-]?\d{4}|\d{3,4}[-—-]\d{7,8}/\d{3,8}|\d{3,4}[-—-]\d{7,8}转\d{1,4}|\d{3,4}[-—-]\d{7,8}|[\(|\(]0\d{2,3}[\)|\)]-?\d{7,8}-?\d{,4}') # 联系电话
  737. # # 2020/11/25 增加发现的号码段
  738. # phone = re.compile('1[3|4|5|6|7|8|9][0-9][-—-]?\d{4}[-—-]?\d{4}|'
  739. # '\d{3,4}[-—-][1-9]\d{6,7}/\d{3,8}|'
  740. # '\d{3,4}[-—-]\d{7,8}转\d{1,4}|'
  741. # '\d{3,4}[-—-]?[1-9]\d{6,7}|'
  742. # '[\(|\(]0\d{2,3}[\)|\)]-?\d{7,8}-?\d{,4}|'
  743. # '[1-9]\d{6,7}') # 联系电话
  744. # dict_index_sentence = {}
  745. # for _sentence in list_sentence:
  746. # dict_index_sentence[_sentence.sentence_index] = _sentence
  747. #
  748. # dict_context_itemx = {}
  749. # last_person = "####****++++$$^"
  750. # last_person_phone = "####****++++$^"
  751. # _list_entity = [entity for entity in list_entity if entity.entity_type == "person"]
  752. # while (p_entitys < len(_list_entity)):
  753. # entity = _list_entity[p_entitys]
  754. # if entity.entity_type == "person" and entity.label in [1,2,3]:
  755. # sentence = dict_index_sentence[entity.sentence_index]
  756. # # item_x = embedding(spanWindow(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,size=settings.MODEL_PERSON_INPUT_SHAPE[1]),shape=settings.MODEL_PERSON_INPUT_SHAPE)
  757. #
  758. # # s = spanWindow(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,size=20)
  759. #
  760. # # 2021/5/8 取上下文的句子,解决表格处理的分句问题
  761. # left_sentence = dict_index_sentence.get(entity.sentence_index - 1)
  762. # left_sentence_tokens = left_sentence.tokens if left_sentence else []
  763. # right_sentence = dict_index_sentence.get(entity.sentence_index + 1)
  764. # right_sentence_tokens = right_sentence.tokens if right_sentence else []
  765. # entity_beginIndex = entity.begin_index + len(left_sentence_tokens)
  766. # entity_endIndex = entity.end_index + len(left_sentence_tokens)
  767. # context_sentences_tokens = left_sentence_tokens + sentence.tokens + right_sentence_tokens
  768. # s = spanWindow(tokens=context_sentences_tokens, begin_index=entity_beginIndex,
  769. # end_index=entity_endIndex, size=20)
  770. #
  771. # _key = "".join(["".join(x) for x in s])
  772. # if _key in dict_context_itemx:
  773. # _dianhua = dict_context_itemx[_key][0]
  774. # else:
  775. # s1 = ''.join(s[1])
  776. # # s1 = re.sub(',)', '-', s1)
  777. # s1 = re.sub('\s', '', s1)
  778. # have_key = re.findall(key_word, s1)
  779. # have_phone = re.findall(phone, s1)
  780. # s0 = ''.join(s[0])
  781. # # s0 = re.sub(',)', '-', s0)
  782. # s0 = re.sub('\s', '', s0)
  783. # have_key2 = re.findall(key_word, s0)
  784. # have_phone2 = re.findall(phone, s0)
  785. #
  786. # s3 = ''.join(s[1])
  787. # # s0 = re.sub(',)', '-', s0)
  788. # s3 = re.sub(',|,|\s', '', s3)
  789. # have_key3 = re.findall(key_word, s3)
  790. # have_phone3 = re.findall(phone, s3)
  791. #
  792. # s4 = ''.join(s[0])
  793. # # s0 = re.sub(',)', '-', s0)
  794. # s4 = re.sub(',|,|\s', '', s0)
  795. # have_key4 = re.findall(key_word, s4)
  796. # have_phone4 = re.findall(phone, s4)
  797. #
  798. # _dianhua = ""
  799. # if have_phone:
  800. # if entity.entity_text != last_person and s0.find(last_person) != -1 and s1.find(
  801. # last_person_phone) != -1:
  802. # if len(have_phone) > 1:
  803. # _dianhua = phoneFromList(have_phone[1:])
  804. # else:
  805. # _dianhua = phoneFromList(have_phone)
  806. # elif have_key:
  807. # if entity.entity_text != last_person and s0.find(last_person) != -1 and s1.find(
  808. # last_person_phone) != -1:
  809. # if len(have_key) > 1:
  810. # _dianhua = phoneFromList(have_key[1:])
  811. # else:
  812. # _dianhua = phoneFromList(have_key)
  813. # elif have_phone2:
  814. # if entity.entity_text != last_person and s0.find(last_person) != -1 and s0.find(
  815. # last_person_phone) != -1:
  816. # if len(have_phone2) > 1:
  817. # _dianhua = phoneFromList(have_phone2[1:])
  818. # else:
  819. # _dianhua = phoneFromList(have_phone2)
  820. # elif have_key2:
  821. # if entity.entity_text != last_person and s0.find(last_person) != -1 and s0.find(
  822. # last_person_phone) != -1:
  823. # if len(have_key2) > 1:
  824. # _dianhua = phoneFromList(have_key2[1:])
  825. # else:
  826. # _dianhua = phoneFromList(have_key2)
  827. # elif have_phone3:
  828. # if entity.entity_text != last_person and s4.find(last_person) != -1 and s3.find(
  829. # last_person_phone) != -1:
  830. # if len(have_phone3) > 1:
  831. # _dianhua = phoneFromList(have_phone3[1:])
  832. # else:
  833. # _dianhua = phoneFromList(have_phone3)
  834. # elif have_key3:
  835. # if entity.entity_text != last_person and s4.find(last_person) != -1 and s3.find(
  836. # last_person_phone) != -1:
  837. # if len(have_key3) > 1:
  838. # _dianhua = phoneFromList(have_key3[1:])
  839. # else:
  840. # _dianhua = phoneFromList(have_key3)
  841. # elif have_phone4:
  842. # if entity.entity_text != last_person and s4.find(last_person) != -1 and s4.find(
  843. # last_person_phone) != -1:
  844. # if len(have_phone4) > 1:
  845. # _dianhua = phoneFromList(have_phone4)
  846. # else:
  847. # _dianhua = phoneFromList(have_phone4)
  848. # elif have_key4:
  849. # if entity.entity_text != last_person and s4.find(last_person) != -1 and s4.find(
  850. # last_person_phone) != -1:
  851. # if len(have_key4) > 1:
  852. # _dianhua = phoneFromList(have_key4)
  853. # else:
  854. # _dianhua = phoneFromList(have_key4)
  855. # else:
  856. # _dianhua = ""
  857. # # dict_context_itemx[_key] = [item_x, _dianhua]
  858. # dict_context_itemx[_key] = [_dianhua]
  859. # # points_entitys.append(entity)
  860. # # dianhua.append(_dianhua)
  861. # last_person = entity.entity_text
  862. # if _dianhua:
  863. # # 更新联系人entity联系方式(person_phone)
  864. # entity.person_phone = _dianhua
  865. # last_person_phone = _dianhua
  866. # else:
  867. # last_person_phone = "####****++++$^"
  868. # p_entitys += 1
  869. from scipy.optimize import linear_sum_assignment
  870. from BiddingKG.dl.interface.Entitys import Match
  871. def dispatch(match_list):
  872. main_roles = list(set([match.main_role for match in match_list]))
  873. attributes = list(set([match.attribute for match in match_list]))
  874. label = np.zeros(shape=(len(main_roles), len(attributes)))
  875. for match in match_list:
  876. main_role = match.main_role
  877. attribute = match.attribute
  878. value = match.value
  879. label[main_roles.index(main_role), attributes.index(attribute)] = value + 10000
  880. # print(label)
  881. gragh = -label
  882. # km算法
  883. row, col = linear_sum_assignment(gragh)
  884. max_dispatch = [(i, j) for i, j, value in zip(row, col, gragh[row, col]) if value]
  885. return [Match(main_roles[row], attributes[col]) for row, col in max_dispatch]
  886. # km算法
  887. key_word = re.compile('((?:电话|联系方式|联系人).{0,4}?)(\d{7,12})')
  888. phone = re.compile('1[3|4|5|6|7|8|9][0-9][-—-―]?\d{4}[-—-―]?\d{4}|'
  889. '\+86.?1[3|4|5|6|7|8|9]\d{9}|'
  890. '0\d{2,3}[-—-―][1-9]\d{6,7}/[1-9]\d{6,10}|'
  891. '0\d{2,3}[-—-―]\d{7,8}转\d{1,4}|'
  892. '0\d{2,3}[-—-―]?[1-9]\d{6,7}|'
  893. '[\(|\(]0\d{2,3}[\)|\)]-?\d{7,8}-?\d{,4}|'
  894. '[1-9]\d{6,7}')
  895. phone_entitys = []
  896. for _sentence in list_sentence:
  897. sentence_text = _sentence.sentence_text
  898. res_set = set()
  899. for i in re.finditer(phone,sentence_text):
  900. res_set.add((i.group(),i.start(),i.end()))
  901. for i in re.finditer(key_word,sentence_text):
  902. res_set.add((i.group(2),i.start()+len(i.group(1)),i.end()))
  903. for item in list(res_set):
  904. phone_left = sentence_text[max(0,item[1]-10):item[1]]
  905. phone_right = sentence_text[item[2]:item[2]+8]
  906. # 排除传真号 和 其它错误项
  907. if re.search("传,?真|信,?箱|邮,?箱",phone_left):
  908. if not re.search("电,?话",phone_left):
  909. continue
  910. if re.search("帐,?号|编,?号|报,?价|证,?号|价,?格|[\((]万?元[\))]",phone_left):
  911. continue
  912. if re.search("[.,]\d{2,}",phone_right):
  913. continue
  914. _entity = Entity(_sentence.doc_id, None, item[0], "phone", _sentence.sentence_index, None, None,item[1], item[2])
  915. phone_entitys.append(_entity)
  916. person_entitys = []
  917. for entity in list_entity:
  918. if entity.entity_type == "person":
  919. entity.person_phone = ""
  920. person_entitys.append(entity)
  921. _list_entity = phone_entitys + person_entitys
  922. _list_entity = sorted(_list_entity,key=lambda x:(x.sentence_index,x.wordOffset_begin))
  923. words_num_dict = dict()
  924. last_words_num = 0
  925. list_sentence = sorted(list_sentence, key=lambda x: x.sentence_index)
  926. for sentence in list_sentence:
  927. _index = sentence.sentence_index
  928. if _index == 0:
  929. words_num_dict[_index] = 0
  930. else:
  931. words_num_dict[_index] = words_num_dict[_index - 1] + last_words_num
  932. last_words_num = len(sentence.sentence_text)
  933. match_list = []
  934. for index in range(len(_list_entity)):
  935. entity = _list_entity[index]
  936. if entity.entity_type=="person" and entity.label in [1,2,3]:
  937. match_nums = 0
  938. for after_index in range(index + 1, min(len(_list_entity), index + 5)):
  939. after_entity = _list_entity[after_index]
  940. if after_entity.entity_type=="phone":
  941. sentence_distance = after_entity.sentence_index - entity.sentence_index
  942. distance = (words_num_dict[after_entity.sentence_index] + after_entity.wordOffset_begin) - (
  943. words_num_dict[entity.sentence_index] + entity.wordOffset_end)
  944. if sentence_distance < 2 and distance < 50:
  945. value = (-1 / 2 * (distance ** 2)) / 10000
  946. match_list.append(Match(entity, after_entity, value))
  947. match_nums += 1
  948. else:
  949. break
  950. if after_entity.entity_type=="person":
  951. if after_entity.label not in [1,2,3]:
  952. break
  953. if not match_nums:
  954. for previous_index in range(index-1, max(0,index-5), -1):
  955. previous_entity = _list_entity[previous_index]
  956. if previous_entity.entity_type == "phone":
  957. sentence_distance = entity.sentence_index - previous_entity.sentence_index
  958. distance = (words_num_dict[entity.sentence_index] + entity.wordOffset_begin) - (
  959. words_num_dict[previous_entity.sentence_index] + previous_entity.wordOffset_end)
  960. if sentence_distance < 1 and distance<30:
  961. # 前向 没有 /10000
  962. value = (-1 / 2 * (distance ** 2))
  963. match_list.append(Match(entity, previous_entity, value))
  964. else:
  965. break
  966. result = dispatch(match_list)
  967. for match in result:
  968. entity = match.main_role
  969. # 更新 list_entity
  970. entity_index = list_entity.index(entity)
  971. list_entity[entity_index].person_phone = match.attribute.entity_text
  972. def predict(self,list_sentences,list_entitys):
  973. self.predict_person(list_sentences,list_entitys)
  974. #表格预测
  975. class FormPredictor():
  976. def __init__(self,lazyLoad=getLazyLoad()):
  977. self.model_file_line = os.path.dirname(__file__)+"/../form/model/model_form.model_line.hdf5"
  978. self.model_file_item = os.path.dirname(__file__)+"/../form/model/model_form.model_item.hdf5"
  979. self.model_form_item = Model_form_item()
  980. self.model_form_context = Model_form_context()
  981. self.model_dict = {"line":[None,self.model_file_line]}
  982. def getModel(self,type):
  983. if type=="item":
  984. return self.model_form_item
  985. elif type=="context":
  986. return self.model_form_context
  987. else:
  988. return self.getModel(type)
  989. def encode(self,data,**kwargs):
  990. return encodeInput([data], word_len=50, word_flag=True,userFool=False)[0]
  991. return encodeInput_form(data)
  992. def predict(self,form_datas,type):
  993. if type=="item":
  994. return self.model_form_item.predict(form_datas)
  995. elif type=="context":
  996. return self.model_form_context.predict(form_datas)
  997. else:
  998. return self.getModel(type).predict(form_datas)
  999. #角色规则
  1000. #依据正则给所有无角色的实体赋予角色,给予等于阈值的最低概率
  1001. class RoleRulePredictor():
  1002. def __init__(self):
  1003. # (?P<tenderee_left_w1> 正则组名 后面的 w1 为概率权重关键词
  1004. self.pattern_tenderee_left = "(?P<tenderee_left>((项目|需求|最终|建设|业主|转让|招租|甲|议标|合同主体|挂牌|出租|出让|买受|选取|抽取|抽选|出售|标卖|比价|处置)" \
  1005. "(人|公司|单位|组织|用户|业主|主体|方|部门)|文章来源|委托机构|产权所有人|需方|买方|业主|(业主|采购人|招标人)联系方式[,:]公司名称:|权属人|甲方当事人|询价书企业|比选发起人|项目单位[,:]单位名称|结算单位)"\
  1006. "[))]?(信息[,:])?(名称)?([((](全称|盖章)[))])?(是|为|:|:|\s*)+$)"
  1007. self.pattern_tenderee_left_w1 = "(?P<tenderee_left_w1>((遴选|采购|招标|竞价|议价|比选|委托|询价|评选|谈判|邀标|邀请|洽谈|约谈)" \
  1008. "(人|公司|单位|组织|用户|业主|主体|方|部门))" \
  1009. "(名称)?([((](全称|盖章)[))])?(是|为|:|:|\s*)+$)"
  1010. self.pattern_tenderee_center = "(?P<tenderee_center>(受.{5,20}委托))"
  1011. self.pattern_tenderee_right = "(?P<tenderee_right>^([((](以下简称)?[,\"“]*(招标|采购)(人|单位|机构)[,\"”]*[))])|^委托|^现委托|^的\w{2,10}正在进行)" #|(^[^.。,,::](采购|竞价|招标|施工|监理|中标|物资)(公告|公示|项目|结果|招标))|的.*正在进行询比价)
  1012. self.pattern_tendereeORagency_right = "(?P<tendereeORagency_right>(^拟对|^现?就|^现对))"
  1013. self.pattern_agency_left = "(?P<agency_left>(代理(?:人|机构|公司|单位|组织)|专业采购机构|集中采购机构|招标组织机构|集采机构|[招议))]+标机构)(.{,4}名,?称|全称|是|为|:|:|[,,]?\s*)$|(受.{5,20}委托,?$))"
  1014. self.pattern_agency_right = "(?P<agency_right>^([((](以下简称)?[,\"“]*(代理)(人|单位|机构)[,\"”]*[))])|^受.{5,20}委托|^受委?托,)" # |^受托 会与 受托生产等冲突,代理表达一般会在后面有逗号
  1015. # 2020//11/24 大网站规则 中标关键词添加 选定单位|指定的中介服务机构
  1016. self.pattern_winTenderer_left = "(?P<winTenderer_left>(乙|承做|施工|供货|承包|承建|竞得|受让|签约)(候选)?(人|单位|机构|供应商|方|公司|厂商|商)[::是为]+$|" \
  1017. "(选定单位|指定的中介服务机构|实施主体|承制单位|供方)[::是为]+$|((评审结果|名次|排名|中标结果)[::]*第?[一1]名?)[::是为]+$|" \
  1018. "单一来源(采购)?(供应商|供货商|服务商|方式向)$|((中标|成交)(结果|信息))[::是为]+$|(供应|供货|承销|服务|实施)(机构|单位|商|方)(名称)?[::是为]+$)"
  1019. self.pattern_winTenderer_left_w1 = "(?P<winTenderer_left_w1>(中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|方|公司|厂商|商)[::是为]+$)" #取消逗号 并拒绝执行改进计划的供应商,华新水泥将可能终止与其合作关系
  1020. # self.pattern_winTenderer_center = "(?P<winTenderer_center>第[一1].{,20}[是为]((中标|中选|中价|成交|施工)(人|单位|机构|供应商|公司)|供应商)[::是为])"
  1021. # self.pattern_winTenderer_right = "(?P<winTenderer_right>(^[是为\(]((采购(供应商|供货商|服务商)|(第[一1]|预)?(拟?(中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司|厂商)))))|^(报价|价格)最低,确定为本项目成交供应商)"
  1022. self.pattern_winTenderer_right = "(?P<winTenderer_right>(^[是为]((采购|中标)(供应商|供货商|服务商)|(第[一1]|预)?(拟?(中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司|厂商)))|" \
  1023. "^(报价|价格)最低,确定为本项目成交供应商|^:贵公司参与|^:?你方于|^中标。|^成为[\w、()()]+项目的成交供应商))"
  1024. self.pattern_winTenderer_whole = "(?P<winTenderer_center>贵公司.{,15}以.{,15}中标|最终由.{,15}竞买成功|经.{,15}决定[以由].{,15}公司中标|谈判结果:由.{5,20}供货)|中标通知书.{,15}你方" # 2020//11/24 大网站规则 中标关键词添加 谈判结果:由.{5,20}供货
  1025. # self.pattern_winTenderer_location = "(中标|中选|中价|乙|成交|承做|施工|供货|承包|竞得|受让)(候选)?(人|单位|机构|供应商|方|公司|厂商|商)|(供应商|供货商|服务商)[::]?$|(第[一1](名|((中标|中选|中价|成交)?(候选)?(人|单位|机构|供应商))))(是|为|:|:|\s*$)|((评审结果|名次|排名)[::]第?[一1]名?)|(单一来源(采购)?方式向.?$)"
  1026. self.pattern_secondTenderer_left = "(?P<secondTenderer_left>((第[二2](名|((中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司))))[::是为]+$)|((评审结果|名次|排名)[::]第?[二2]名?,?投标商名称[::]+$))"
  1027. self.pattern_secondTenderer_right = "(?P<secondTenderer_right>^[是为\(]第[二2](名|(中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司)))"
  1028. self.pattern_thirdTenderer_left = "(?P<thirdTenderer_left>(第[三3](名|((中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司))))[::是为]+$|((评审结果|名次|排名)[::]第?[三3]名?,?投标商名称[::]+$))"
  1029. self.pattern_thirdTenderer_right = "(?P<thirdTenderer_right>^[是为\(]第[三3](名|(中标|中选|中价|成交)(候选)?(人|单位|机构|供应商|公司)))"
  1030. self.pattern_whole = [self.pattern_tenderee_left,
  1031. self.pattern_tenderee_left_w1,
  1032. self.pattern_tenderee_center,
  1033. self.pattern_tenderee_right,
  1034. self.pattern_tendereeORagency_right,
  1035. self.pattern_agency_left,
  1036. self.pattern_agency_right,
  1037. self.pattern_winTenderer_left,
  1038. self.pattern_winTenderer_left_w1,
  1039. self.pattern_winTenderer_whole,
  1040. self.pattern_winTenderer_right,
  1041. self.pattern_secondTenderer_left,
  1042. self.pattern_secondTenderer_right,
  1043. self.pattern_thirdTenderer_left,
  1044. self.pattern_thirdTenderer_right
  1045. ] # 需按顺序排列, 第二、三中标要在中标正则后面
  1046. self.SET_NOT_TENDERER = set(["人民政府","人民法院","中华人民共和国","人民检察院","评标委员会","中国政府","中国海关","中华人民共和国政府"])
  1047. self.pattern_money_tenderee = re.compile("投标最高限价|采购计划金额|项目预算|招标金额|采购金额|项目金额|建安费用|采购(单位|人)委托价|限价|拦标价|预算金额")
  1048. self.pattern_money_tenderer = re.compile("((合同|成交|中标|应付款|交易|投标|验收)[)\)]?(总?金额|结果|[单报]?价))|总价|标的基本情况")
  1049. self.pattern_money_tenderer_whole = re.compile("(以金额.*中标)|中标供应商.*单价|以.*元中标")
  1050. self.pattern_money_other = re.compile("代理费|服务费")
  1051. self.pattern_pack = "(([^承](包|标[段号的包]|分?包|包组)编?号?|项目)[::]?[\((]?[0-9A-Za-z一二三四五六七八九十]{1,4})[^至]?|(第?[0-9A-Za-z一二三四五六七八九十]{1,4}(包号|标[段号的包]|分?包))|[0-9]个(包|标[段号的包]|分?包|包组)"
  1052. def _check_input(self,text, ignore=False):
  1053. if not text:
  1054. return []
  1055. if not isinstance(text, list):
  1056. text = [text]
  1057. null_index = [i for i, t in enumerate(text) if not t]
  1058. if null_index and not ignore:
  1059. raise Exception("null text in input ")
  1060. return text
  1061. def predict(self, list_articles, list_sentences, list_entitys, list_codenames, on_value=0.5):
  1062. for article, list_entity, list_sentence, list_codename in zip(list_articles, list_entitys, list_sentences,
  1063. list_codenames):
  1064. list_sentence.sort(key=lambda x: x.sentence_index) # 2022/1/5 按句子顺序排序
  1065. # list_name = list_codename["name"]
  1066. list_name = [] # 2022/1/5 改为实体列表内所有项目名称
  1067. for entity in list_entity:
  1068. if entity.entity_type == 'name':
  1069. list_name.append(entity.entity_text)
  1070. list_name = self._check_input(list_name) + [article.title]
  1071. for p_entity in list_entity:
  1072. if p_entity.entity_type in ["org", "company"]:
  1073. # 只解析角色为无的或者概率低于阈值的
  1074. if p_entity.label is None:
  1075. continue
  1076. # 将上下文包含标题的实体概率置为0.6,因为标题中的实体不一定是招标人
  1077. if str(p_entity.label) == "0":
  1078. find_flag = False
  1079. for _sentence in list_sentence:
  1080. if _sentence.sentence_index == p_entity.sentence_index:
  1081. _span = spanWindow(tokens=_sentence.tokens, begin_index=p_entity.begin_index,
  1082. end_index=p_entity.end_index, size=20, center_include=True,
  1083. word_flag=True, use_text=True,
  1084. text=re.sub(")", ")", re.sub("(", "(", p_entity.entity_text)))
  1085. for _name in list_name:
  1086. if _name != "" and str(_span[1] + _span[2][:len(str(_name))]).find(_name) >= 0:
  1087. find_flag = True
  1088. if p_entity.values[0] > on_value:
  1089. p_entity.values[0] = 0.6 + (p_entity.values[0] - 0.6) / 10
  1090. else:
  1091. p_entity.values[0] = on_value # 2022/03/08 修正类似 223985179 公司在文章开头的项目名称概率又没达到0.5的情况
  1092. if find_flag:
  1093. continue
  1094. # 正则从概率低于阈值或其他类别中召回角色
  1095. role_prob = float(p_entity.values[int(p_entity.label)])
  1096. if role_prob < on_value or str(p_entity.label) == "5":
  1097. # 将标题中的实体置为招标人
  1098. _list_name = self._check_input(list_name, ignore=True)
  1099. find_flag = False
  1100. for _name in _list_name: # 2022/1/5修正只要项目名称出现过的角色,所有位置都标注为招标人
  1101. if str(_name).find(re.sub(")", ")", re.sub("(", "(",
  1102. p_entity.entity_text))) >= 0 and p_entity.sentence_index < 4:
  1103. for _sentence in list_sentence:
  1104. if _sentence.sentence_index == p_entity.sentence_index:
  1105. _span = spanWindow(tokens=_sentence.tokens, begin_index=p_entity.begin_index,
  1106. end_index=p_entity.end_index, size=20, center_include=True,
  1107. word_flag=True, use_text=True, text=re.sub(")", ")",
  1108. re.sub("(", "(",
  1109. p_entity.entity_text)))
  1110. if str(_span[1] + _span[2][:len(str(_name))]).find(
  1111. _name) >= 0:
  1112. find_flag = True
  1113. _label = 0
  1114. p_entity.label = _label
  1115. p_entity.values[int(_label)] = on_value
  1116. break
  1117. if p_entity.sentence_index >= 4:
  1118. break
  1119. if find_flag:
  1120. break
  1121. # if str(_name).find(p_entity.entity_text)>=0:
  1122. # find_flag = True
  1123. # _label = 0
  1124. # p_entity.label = _label
  1125. # p_entity.values[int(_label)] = on_value
  1126. # break
  1127. # 若是实体在标题中,默认为招标人,不进行以下的规则匹配
  1128. if find_flag:
  1129. continue
  1130. for s_index in range(len(list_sentence)):
  1131. if p_entity.doc_id == list_sentence[s_index].doc_id and p_entity.sentence_index == \
  1132. list_sentence[s_index].sentence_index:
  1133. tokens = list_sentence[s_index].tokens
  1134. begin_index = p_entity.begin_index
  1135. end_index = p_entity.end_index
  1136. size = 15
  1137. spans = spanWindow(tokens, begin_index, end_index, size, center_include=True,
  1138. word_flag=True, use_text=False)
  1139. # _flag = False
  1140. # 使用正则+距离解决冲突
  1141. # 2021/6/11update center: spans[1] --> spans[0][-30:]+spans[1]
  1142. list_spans = [spans[0][-30:], spans[0][-10:] + spans[1] + spans[2][:10], spans[2]] # 实体左、中、右 信息
  1143. for _i_span in range(len(list_spans)):
  1144. _flag = False
  1145. _prob_weight = 1
  1146. # print(list_spans[_i_span],p_entity.entity_text)
  1147. for _pattern in self.pattern_whole:
  1148. for _iter in re.finditer(_pattern, list_spans[_i_span]):
  1149. for _group, _v_group in _iter.groupdict().items():
  1150. if _v_group is not None and _v_group != "":
  1151. _role = _group.split("_")[0]
  1152. if _role == "tendereeORagency": # 2022/3/9 新增不确定招标代理判断逻辑
  1153. print('p_entity_sentenceindex:', p_entity.sentence_index)
  1154. if p_entity.sentence_index>=1: # 只在第一句进行这种模糊匹配
  1155. continue
  1156. if re.search('医院|学校|大学|中学|小学|幼儿园|政府|部|委员会|署|行|局|厅|处|室|科|股|站', p_entity.entity_text)\
  1157. or re.search('(采购|招标|投标|交易|代理|拍卖|咨询|顾问|管理)', p_entity.entity_text) == None:
  1158. _role = 'tenderee'
  1159. else:
  1160. _role = "agency"
  1161. _direct = _group.split("_")[1]
  1162. _weight = _group.split("_")[2] if len(_group.split("_"))==3 else ""
  1163. # _label = {"tenderee": 0, "agency": 1, "winTenderer": 2,
  1164. # "secondTenderer": 3, "thirdTenderer": 4}.get(_role)
  1165. if _i_span == 0 and _direct == "left" and re.search('各.{,5}供应商|尊敬的供应商|业绩|拟招|交易服务单位', #135463002 拟招一家供应商为宜宾市第三人民医院、李庄同济医院院区提供消防维保服务
  1166. list_spans[
  1167. 0]) == None: # 2021/12/22 修正错误中标召回 例子208668937
  1168. _flag = True
  1169. _label = {"tenderee": 0, "agency": 1, "winTenderer": 2,
  1170. "secondTenderer": 3, "thirdTenderer": 4}.get(_role)
  1171. _prob_weight = 1.2 if _weight=='w1' else 1
  1172. # print('_v_group:',_group, _v_group, p_entity.entity_text)
  1173. if _i_span == 1 and _direct == "center":
  1174. _flag = True
  1175. _label = {"tenderee": 0, "agency": 1, "winTenderer": 2,
  1176. "secondTenderer": 3, "thirdTenderer": 4}.get(_role)
  1177. _prob_weight = 1.2 if _weight == 'w1' else 1
  1178. # print('_v_group:', _group, _v_group, p_entity.entity_text)
  1179. if _i_span == 2 and _direct == "right":
  1180. _flag = True
  1181. _label = {"tenderee": 0, "agency": 1, "winTenderer": 2,
  1182. "secondTenderer": 3, "thirdTenderer": 4}.get(_role)
  1183. _prob_weight = 1.2 if _weight == 'w1' else 1
  1184. # print('_v_group:', _group, _v_group, p_entity.entity_text)
  1185. # 得到结果
  1186. if _flag:
  1187. p_entity.label = _label
  1188. p_entity.values[int(_label)] = on_value*_prob_weight + p_entity.values[int(_label)] / 10
  1189. # log('正则召回实体: %s, %s, %s, %d, %.4f, %s'%(_group, _v_group, p_entity.entity_text, p_entity.label, p_entity.values[p_entity.label], list_spans[_i_span]))
  1190. break
  1191. # 其他金额通过正则召回可能是招标或中投标的金额
  1192. if p_entity.entity_type in ["money"]:
  1193. if str(p_entity.label) == "2":
  1194. for _sentence in list_sentence:
  1195. if _sentence.sentence_index == p_entity.sentence_index:
  1196. _span = spanWindow(tokens=_sentence.tokens, begin_index=p_entity.begin_index,
  1197. end_index=p_entity.end_index, size=20, center_include=True,
  1198. word_flag=True, text=p_entity.entity_text)
  1199. if re.search(self.pattern_money_tenderee, _span[0]) is not None and re.search(
  1200. self.pattern_money_other, _span[0]) is None:
  1201. p_entity.values[0] = 0.8 + p_entity.values[0] / 10
  1202. p_entity.label = 0
  1203. if re.search(self.pattern_money_tenderer, _span[0]) is not None:
  1204. if re.search(self.pattern_money_other, _span[0]) is not None:
  1205. if re.search(self.pattern_money_tenderer, _span[0]).span()[1] > \
  1206. re.search(self.pattern_money_other, _span[0]).span()[1]:
  1207. p_entity.values[1] = 0.8 + p_entity.values[1] / 10
  1208. p_entity.label = 1
  1209. else:
  1210. p_entity.values[1] = 0.8 + p_entity.values[1] / 10
  1211. p_entity.label = 1
  1212. if re.search(self.pattern_money_tenderer_whole,
  1213. "".join(_span)) is not None and re.search(self.pattern_money_other,
  1214. _span[0]) is None:
  1215. p_entity.values[1] = 0.8 + p_entity.values[1] / 10
  1216. p_entity.label = 1
  1217. # 增加招标金额扩展,招标金额+连续的未识别金额,并且都可以匹配到标段信息,则将为识别的金额设置为招标金额
  1218. list_p = []
  1219. state = 0
  1220. for p_entity in list_entity:
  1221. for _sentence in list_sentence:
  1222. if _sentence.sentence_index == p_entity.sentence_index:
  1223. _span = spanWindow(tokens=_sentence.tokens, begin_index=p_entity.begin_index,
  1224. end_index=p_entity.end_index, size=20, center_include=True, word_flag=True,
  1225. text=p_entity.entity_text)
  1226. if state == 2:
  1227. for _p in list_p[1:]:
  1228. _p.values[0] = 0.8 + _p.values[0] / 10
  1229. _p.label = 0
  1230. state = 0
  1231. list_p = []
  1232. if state == 0:
  1233. if p_entity.entity_type in ["money"]:
  1234. if str(p_entity.label) == "0" and re.search(self.pattern_pack,
  1235. _span[0] + "-" + _span[2]) is not None:
  1236. state = 1
  1237. list_p.append(p_entity)
  1238. elif state == 1:
  1239. if p_entity.entity_type in ["money"]:
  1240. if str(p_entity.label) in ["0", "2"] and re.search(self.pattern_pack,
  1241. _span[0] + "-" + _span[
  1242. 2]) is not None and re.search(
  1243. self.pattern_money_other,
  1244. _span[0] + "-" + _span[2]) is None and p_entity.sentence_index == list_p[
  1245. 0].sentence_index:
  1246. list_p.append(p_entity)
  1247. else:
  1248. state = 2
  1249. if len(list_p) > 1:
  1250. for _p in list_p[1:]:
  1251. # print("==",_p.entity_text,_p.sentence_index,_p.label)
  1252. _p.values[0] = 0.8 + _p.values[0] / 10
  1253. _p.label = 0
  1254. state = 0
  1255. list_p = []
  1256. for p_entity in list_entity:
  1257. # 将属于集合中的不可能是中标人的标签置为无
  1258. if p_entity.entity_text in self.SET_NOT_TENDERER:
  1259. p_entity.label = 5
  1260. '''正则补充最后一句实体日期格式为招标或代理 2021/12/30'''
  1261. class RoleRuleFinalAdd():
  1262. def predict(self, list_articles, list_entitys, list_codenames):
  1263. text_end = list_articles[0].content[-40:]
  1264. # sear_ent = re.search('[,。]([\u4e00-\u9fa5()()]{5,20}),?\s*[.]{2,4}年.{1,2}月.{1,2}日', text_end)
  1265. sear_ent = re.search('[,。]([\u4e00-\u9fa5()()]{5,20}(,?[\u4e00-\u9fa5]{,6}(分公司|部))?),?\s*[0-9零一二三四五六七八九十]{2,4}[年\-/][0-9零一二三四五六七八九十]{1,2}[月\-/][0-9零一二三四五六七八九十]{1,2}日?', text_end)
  1266. sear_ent2 = re.search('(户名|开户名称)[::]([\u4e00-\u9fa5()()]{5,20})[,。]', list_articles[0].content[:5000])
  1267. sear_ent3 = re.search('(报名咨询|收货地点|送货地点)[,:]([\u4e00-\u9fa5()()]{5,20})[0-9\-]*[,。]', list_articles[0].content[:5000])
  1268. if sear_ent or sear_ent2 or sear_ent3:
  1269. if sear_ent3:
  1270. ent_re = sear_ent3.group(2)
  1271. elif sear_ent2:
  1272. ent_re = sear_ent2.group(2)
  1273. else:
  1274. ent_re = sear_ent.group(1)
  1275. ent_re = ent_re.replace(',', '').replace("(","(").replace(")",")")
  1276. tenderee_notfound = True
  1277. agency_notfound = True
  1278. ents = []
  1279. for ent in list_entitys[0]:
  1280. if ent.entity_type in ['org', 'company']:
  1281. if ent.label == 0:
  1282. tenderee_notfound = False
  1283. elif ent.label == 1:
  1284. agency_notfound = False
  1285. elif ent.label == 5:
  1286. ents.append(ent)
  1287. if tenderee_notfound == True and (re.search('医院|学校|大学|中学|小学|幼儿园|政府|部|委员会|署|行|局|厅|处|室|科|股|站', ent_re)
  1288. or re.search('(采购|招标|投标|交易|代理|拍卖|咨询|顾问|管理)', ent_re) == None):
  1289. n = 0
  1290. for i in range(len(ents) - 1, -1, -1):
  1291. n += 1
  1292. if n > 3 and sear_ent: # 文章末尾角色加日期这种只找后三个实体
  1293. break
  1294. if ents[i].entity_text == ent_re or (ents[i].entity_text in ent_re and len(ents[i].entity_text)/len(ent_re)>0.6):
  1295. ents[i].label = 0
  1296. ents[i].values[0] = 0.5
  1297. # log('正则最后补充实体: %s'%(ent_re))
  1298. break
  1299. elif agency_notfound == True and re.search('(采购|招标|投标|交易|代理|拍卖|咨询|顾问|管理)', ent_re):
  1300. n = 0
  1301. for i in range(len(ents) - 1, -1, -1):
  1302. n += 1
  1303. if n > 3 and sear_ent: # 文章末尾角色加日期这种只找后三个实体
  1304. break
  1305. if ents[i].entity_text == ent_re or (ents[i].entity_text in ent_re and len(ents[i].entity_text)/len(ent_re)>0.6):
  1306. ents[i].label = 1
  1307. ents[i].values[1] = 0.5
  1308. # log('正则最后补充实体: %s'%(ent_re))
  1309. break
  1310. elif list_codenames[0]['name'] != "": #把标题包含的公司实体作为招标人
  1311. tenderee_notfound = True
  1312. ents = []
  1313. for ent in list_entitys[0]:
  1314. if ent.entity_type in ['org', 'company']:
  1315. if ent.label == 0:
  1316. tenderee_notfound = False
  1317. elif ent.label == 1:
  1318. agency_notfound = False
  1319. elif ent.label == 5:
  1320. ents.append(ent)
  1321. if tenderee_notfound == True:
  1322. print('list_codenames',list_codenames[0]['name'])
  1323. for ent in ents:
  1324. if ent.entity_text in list_codenames[0]['name']:
  1325. ent.label = 0
  1326. ent.values[0] = 0.5
  1327. # log('正则召回标题中包含的实体:%s'%ent.entity_text)
  1328. break
  1329. # 时间类别
  1330. class TimePredictor():
  1331. def __init__(self):
  1332. self.sess = tf.Session(graph=tf.Graph())
  1333. self.inputs_code = None
  1334. self.outputs_code = None
  1335. self.input_shape = (2,40,128)
  1336. self.load_model()
  1337. def load_model(self):
  1338. model_path = os.path.dirname(__file__)+'/timesplit_model'
  1339. if self.inputs_code is None:
  1340. log("get model of time")
  1341. with self.sess.as_default():
  1342. with self.sess.graph.as_default():
  1343. meta_graph_def = tf.saved_model.loader.load(self.sess, tags=["serve"], export_dir=model_path)
  1344. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  1345. signature_def = meta_graph_def.signature_def
  1346. self.inputs_code = []
  1347. self.inputs_code.append(
  1348. self.sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name))
  1349. self.inputs_code.append(
  1350. self.sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name))
  1351. self.outputs_code = self.sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  1352. return self.inputs_code, self.outputs_code
  1353. else:
  1354. return self.inputs_code, self.outputs_code
  1355. def search_time_data(self,list_sentences,list_entitys):
  1356. data_x = []
  1357. points_entitys = []
  1358. for list_sentence, list_entity in zip(list_sentences, list_entitys):
  1359. p_entitys = 0
  1360. p_sentences = 0
  1361. list_sentence.sort(key=lambda x: x.sentence_index)
  1362. while(p_entitys<len(list_entity)):
  1363. entity = list_entity[p_entitys]
  1364. if entity.entity_type in ['time']:
  1365. while(p_sentences<len(list_sentence)):
  1366. sentence = list_sentence[p_sentences]
  1367. if entity.doc_id == sentence.doc_id and entity.sentence_index == sentence.sentence_index:
  1368. # left = sentence.sentence_text[max(0,entity.wordOffset_begin-self.input_shape[1]):entity.wordOffset_begin]
  1369. # right = sentence.sentence_text[entity.wordOffset_end:entity.wordOffset_end+self.input_shape[1]]
  1370. s = spanWindow(tokens=sentence.tokens,begin_index=entity.begin_index,end_index=entity.end_index,size=self.input_shape[1])
  1371. left = s[0]
  1372. right = s[1]
  1373. context = [left, right]
  1374. x = self.embedding_words(context, shape=self.input_shape)
  1375. data_x.append(x)
  1376. points_entitys.append(entity)
  1377. break
  1378. p_sentences += 1
  1379. p_entitys += 1
  1380. if len(points_entitys)==0:
  1381. return None
  1382. data_x = np.transpose(np.array(data_x), (1, 0, 2, 3))
  1383. return [data_x, points_entitys]
  1384. def embedding_words(self, datas, shape):
  1385. '''
  1386. @summary:查找词汇对应的词向量
  1387. @param:
  1388. datas:词汇的list
  1389. shape:结果的shape
  1390. @return: array,返回对应shape的词嵌入
  1391. '''
  1392. model_w2v = getModel_w2v()
  1393. embed = np.zeros(shape)
  1394. length = shape[1]
  1395. out_index = 0
  1396. for data in datas:
  1397. index = 0
  1398. for item in data:
  1399. item_not_space = re.sub("\s*", "", item)
  1400. if index >= length:
  1401. break
  1402. if item_not_space in model_w2v.vocab:
  1403. embed[out_index][index] = model_w2v[item_not_space]
  1404. index += 1
  1405. else:
  1406. embed[out_index][index] = model_w2v['unk']
  1407. index += 1
  1408. out_index += 1
  1409. return embed
  1410. def predict(self, list_sentences,list_entitys):
  1411. datas = self.search_time_data(list_sentences, list_entitys)
  1412. if datas is None:
  1413. return
  1414. points_entitys = datas[1]
  1415. with self.sess.as_default():
  1416. predict_y = limitRun(self.sess,[self.outputs_code], feed_dict={self.inputs_code[0]:datas[0][0]
  1417. ,self.inputs_code[1]:datas[0][1]})[0]
  1418. for i in range(len(predict_y)):
  1419. entity = points_entitys[i]
  1420. label = np.argmax(predict_y[i])
  1421. values = []
  1422. for item in predict_y[i]:
  1423. values.append(item)
  1424. if label != 0:
  1425. if not timeFormat(entity.entity_text):
  1426. label = 0
  1427. values[0] = 0.5
  1428. entity.set_Role(label, values)
  1429. # 产品字段提取
  1430. class ProductPredictor():
  1431. def __init__(self):
  1432. vocabpath = os.path.dirname(__file__) + "/codename_vocab.pk"
  1433. self.vocab = load(vocabpath)
  1434. self.word2index = dict((w, i) for i, w in enumerate(np.array(self.vocab)))
  1435. self.sess = tf.Session(graph=tf.Graph())
  1436. self.load_model()
  1437. def load_model(self):
  1438. # model_path = os.path.dirname(__file__)+'/product_savedmodel/product.pb'
  1439. model_path = os.path.dirname(__file__)+'/product_savedmodel/productAndfailreason.pb'
  1440. with self.sess.as_default():
  1441. with self.sess.graph.as_default():
  1442. output_graph_def = tf.GraphDef()
  1443. with open(model_path, 'rb') as f:
  1444. output_graph_def.ParseFromString(f.read())
  1445. tf.import_graph_def(output_graph_def, name='')
  1446. self.sess.run(tf.global_variables_initializer())
  1447. self.char_input = self.sess.graph.get_tensor_by_name('CharInputs:0')
  1448. self.length = self.sess.graph.get_tensor_by_name("Sum:0")
  1449. self.dropout = self.sess.graph.get_tensor_by_name("Dropout:0")
  1450. self.logit = self.sess.graph.get_tensor_by_name("logits/Reshape:0")
  1451. self.tran = self.sess.graph.get_tensor_by_name("crf_loss/transitions:0")
  1452. def decode(self,logits, lengths, matrix):
  1453. paths = []
  1454. small = -1000.0
  1455. # start = np.asarray([[small] * 4 + [0]])
  1456. start = np.asarray([[small]*7+[0]])
  1457. for score, length in zip(logits, lengths):
  1458. score = score[:length]
  1459. pad = small * np.ones([length, 1])
  1460. logits = np.concatenate([score, pad], axis=1)
  1461. logits = np.concatenate([start, logits], axis=0)
  1462. path, _ = viterbi_decode(logits, matrix)
  1463. paths.append(path[1:])
  1464. return paths
  1465. def predict(self, list_sentences,list_entitys=None,list_articles=[], fail=False, MAX_AREA=5000):
  1466. '''
  1467. 预测实体代码,每个句子最多取MAX_AREA个字,超过截断
  1468. :param list_sentences: 多篇公告句子列表,[[一篇公告句子列表],[公告句子列表]]
  1469. :param list_entitys: 多篇公告实体列表
  1470. :param MAX_AREA: 每个句子最多截取多少字
  1471. :return: 把预测出来的实体放进实体类
  1472. '''
  1473. with self.sess.as_default() as sess:
  1474. with self.sess.graph.as_default():
  1475. result = []
  1476. if fail and list_articles!=[]:
  1477. text_list = [list_articles[0].content[:MAX_AREA]]
  1478. chars = [[self.word2index.get(it, self.word2index.get('<unk>')) for it in text] for text in text_list]
  1479. lengths, scores, tran_ = sess.run([self.length, self.logit, self.tran],
  1480. feed_dict={
  1481. self.char_input: np.asarray(chars),
  1482. self.dropout: 1.0
  1483. })
  1484. batch_paths = self.decode(scores, lengths, tran_)
  1485. for text, path, length in zip(text_list, batch_paths, lengths):
  1486. tags = ''.join([str(it) for it in path[:length]])
  1487. for it in re.finditer("12*3", tags):
  1488. start = it.start()
  1489. end = it.end()
  1490. _entity = Entity(doc_id=list_articles[0].doc_id, entity_id="%s_%s_%s_%s" % (
  1491. list_articles[0].doc_id, 0, start, end),
  1492. entity_text=text[start:end],
  1493. entity_type="product", sentence_index=0,
  1494. begin_index=0, end_index=0, wordOffset_begin=start,
  1495. wordOffset_end=end)
  1496. list_entitys[0].append(_entity)
  1497. for it in re.finditer("45*6", tags):
  1498. start = it.start()
  1499. end = it.end()
  1500. result.append(text[start:end].replace('?', '').strip())
  1501. reasons = []
  1502. for it in result:
  1503. if "(√)" in it or "(√)" in it:
  1504. reasons = [it]
  1505. break
  1506. if reasons != [] and (it not in reasons[-1] and it not in reasons):
  1507. reasons.append(it)
  1508. elif reasons == []:
  1509. reasons.append(it)
  1510. return {'fail_reason':';'.join(reasons)}
  1511. if list_entitys is None:
  1512. list_entitys = [[] for _ in range(len(list_sentences))]
  1513. for list_sentence, list_entity in zip(list_sentences,list_entitys):
  1514. if len(list_sentence)==0:
  1515. result.append({"product":[]})
  1516. continue
  1517. list_sentence.sort(key=lambda x:len(x.sentence_text), reverse=True)
  1518. _begin_index = 0
  1519. item = {"product":[]}
  1520. temp_list = []
  1521. while True:
  1522. MAX_LEN = len(list_sentence[_begin_index].sentence_text)
  1523. if MAX_LEN > MAX_AREA:
  1524. MAX_LEN = MAX_AREA
  1525. _LEN = MAX_AREA//MAX_LEN
  1526. chars = [sentence.sentence_text[:MAX_LEN] for sentence in list_sentence[_begin_index:_begin_index+_LEN]]
  1527. chars = [[self.word2index.get(it, self.word2index.get('<unk>')) for it in l] for l in chars]
  1528. chars = pad_sequences(chars, maxlen=MAX_LEN, padding="post", truncating="post")
  1529. lengths, scores, tran_ = sess.run([self.length, self.logit, self.tran],
  1530. feed_dict={
  1531. self.char_input: np.asarray(chars),
  1532. self.dropout: 1.0
  1533. })
  1534. batch_paths = self.decode(scores, lengths, tran_)
  1535. for sentence, path, length in zip(list_sentence[_begin_index:_begin_index+_LEN],batch_paths, lengths):
  1536. tags = ''.join([str(it) for it in path[:length]])
  1537. for it in re.finditer("12*3", tags):
  1538. start = it.start()
  1539. end = it.end()
  1540. _entity = Entity(doc_id=sentence.doc_id, entity_id="%s_%s_%s_%s" % (
  1541. sentence.doc_id, sentence.sentence_index, start, end),
  1542. entity_text=sentence.sentence_text[start:end],
  1543. entity_type="product", sentence_index=sentence.sentence_index,
  1544. begin_index=0, end_index=0, wordOffset_begin=start,
  1545. wordOffset_end=end)
  1546. list_entity.append(_entity)
  1547. temp_list.append(sentence.sentence_text[start:end])
  1548. # item["product"] = list(set(temp_list))
  1549. # result.append(item)
  1550. if _begin_index+_LEN >= len(list_sentence):
  1551. break
  1552. _begin_index += _LEN
  1553. item["product"] = list(set(temp_list))
  1554. result.append(item) # 修正bug
  1555. return {'fail_reason': ""}
  1556. # 产品数量单价品牌规格提取 #2021/11/10 添加表格中的项目、需求、预算、时间要素提取
  1557. class ProductAttributesPredictor():
  1558. def __init__(self,):
  1559. self.p1 = '(设备|货物|商品|产品|物品|货品|材料|物资|物料|物件|耗材|备件|食材|食品|品目|标的|标的物|标项|资产|拍卖物|仪器|器材|器械|药械|药品|药材|采购品?|项目|招标|工程|服务)[\))]?(名称|内容|描述)'
  1560. self.p2 = '设备|货物|商品|产品|物品|货品|材料|物资|物料|物件|耗材|备件|食材|食品|品目|标的|标的物|资产|拍卖物|仪器|器材|器械|药械|药品|药材|采购品|项目|品名|菜名|内容|名称'
  1561. with open(os.path.dirname(__file__)+'/header_set.pkl', 'rb') as f:
  1562. self.header_set = pickle.load(f)
  1563. def isTrueTable(self, table):
  1564. '''真假表格规则:
  1565. 1、包含<caption>或<th>标签为真
  1566. 2、包含大量链接、表单、图片或嵌套表格为假
  1567. 3、表格尺寸太小为假
  1568. 4、外层<table>嵌套子<table>,一般子为真,外为假'''
  1569. if table.find_all(['caption', 'th']) != []:
  1570. return True
  1571. elif len(table.find_all(['form', 'a', 'img'])) > 5:
  1572. return False
  1573. elif len(table.find_all(['tr'])) < 2:
  1574. return False
  1575. elif len(table.find_all(['table'])) >= 1:
  1576. return False
  1577. else:
  1578. return True
  1579. def getTrs(self, tbody):
  1580. # 获取所有的tr
  1581. trs = []
  1582. objs = tbody.find_all(recursive=False)
  1583. for obj in objs:
  1584. if obj.name == "tr":
  1585. trs.append(obj)
  1586. if obj.name == "tbody":
  1587. for tr in obj.find_all("tr", recursive=False):
  1588. trs.append(tr)
  1589. return trs
  1590. def getTable(self, tbody):
  1591. trs = self.getTrs(tbody)
  1592. inner_table = []
  1593. if len(trs) < 2:
  1594. return inner_table
  1595. for tr in trs:
  1596. tr_line = []
  1597. tds = tr.findChildren(['td', 'th'], recursive=False)
  1598. if len(tds) < 2:
  1599. continue
  1600. for td in tds:
  1601. td_text = re.sub('\s', '', td.get_text())
  1602. tr_line.append(td_text)
  1603. inner_table.append(tr_line)
  1604. return inner_table
  1605. def fixSpan(self, tbody):
  1606. # 处理colspan, rowspan信息补全问题
  1607. trs = self.getTrs(tbody)
  1608. ths_len = 0
  1609. ths = list()
  1610. trs_set = set()
  1611. # 修改为先进行列补全再进行行补全,否则可能会出现表格解析混乱
  1612. # 遍历每一个tr
  1613. for indtr, tr in enumerate(trs):
  1614. ths_tmp = tr.findChildren('th', recursive=False)
  1615. # 不补全含有表格的tr
  1616. if len(tr.findChildren('table')) > 0:
  1617. continue
  1618. if len(ths_tmp) > 0:
  1619. ths_len = ths_len + len(ths_tmp)
  1620. for th in ths_tmp:
  1621. ths.append(th)
  1622. trs_set.add(tr)
  1623. # 遍历每行中的element
  1624. tds = tr.findChildren(recursive=False)
  1625. if len(tds) < 3:
  1626. continue # 列数太少的不补全
  1627. for indtd, td in enumerate(tds):
  1628. # 若有colspan 则补全同一行下一个位置
  1629. if 'colspan' in td.attrs and str(re.sub("[^0-9]", "", str(td['colspan']))) != "":
  1630. col = int(re.sub("[^0-9]", "", str(td['colspan'])))
  1631. if col < 10 and len(td.get_text()) < 500:
  1632. td['colspan'] = 1
  1633. for i in range(1, col, 1):
  1634. td.insert_after(copy.copy(td))
  1635. for indtr, tr in enumerate(trs):
  1636. ths_tmp = tr.findChildren('th', recursive=False)
  1637. # 不补全含有表格的tr
  1638. if len(tr.findChildren('table')) > 0:
  1639. continue
  1640. if len(ths_tmp) > 0:
  1641. ths_len = ths_len + len(ths_tmp)
  1642. for th in ths_tmp:
  1643. ths.append(th)
  1644. trs_set.add(tr)
  1645. # 遍历每行中的element
  1646. tds = tr.findChildren(recursive=False)
  1647. same_span = 0
  1648. if len(tds) > 1 and 'rowspan' in tds[0].attrs:
  1649. span0 = tds[0].attrs['rowspan']
  1650. for td in tds:
  1651. if 'rowspan' in td.attrs and td.attrs['rowspan'] == span0:
  1652. same_span += 1
  1653. if same_span == len(tds):
  1654. continue
  1655. for indtd, td in enumerate(tds):
  1656. # 若有rowspan 则补全下一行同样位置
  1657. if 'rowspan' in td.attrs and str(re.sub("[^0-9]", "", str(td['rowspan']))) != "":
  1658. row = int(re.sub("[^0-9]", "", str(td['rowspan'])))
  1659. td['rowspan'] = 1
  1660. for i in range(1, row, 1):
  1661. # 获取下一行的所有td, 在对应的位置插入
  1662. if indtr + i < len(trs):
  1663. tds1 = trs[indtr + i].findChildren(['td', 'th'], recursive=False)
  1664. if len(tds1) >= (indtd) and len(tds1) > 0:
  1665. if indtd > 0:
  1666. tds1[indtd - 1].insert_after(copy.copy(td))
  1667. else:
  1668. tds1[0].insert_before(copy.copy(td))
  1669. elif len(tds1) > 0 and len(tds1) == indtd - 1:
  1670. tds1[indtd - 2].insert_after(copy.copy(td))
  1671. def get_monthlen(self, year, month):
  1672. '''输入年份、月份 int类型 得到该月份天数'''
  1673. try:
  1674. weekday, num = calendar.monthrange(int(year), int(month))
  1675. except:
  1676. num = 30
  1677. return str(num)
  1678. def fix_time(self, text, html, page_time):
  1679. '''输入日期字段返回格式化日期'''
  1680. for it in [('十二', '12'),('十一', '11'),('十','10'),('九','9'),('八','8'),('七','7'),
  1681. ('六','6'),('五','5'),('四','4'),('三','3'),('二','2'),('一','1')]:
  1682. if it[0] in text:
  1683. text = text.replace(it[0], it[1])
  1684. if re.search('^\d{1,2}月$', text):
  1685. m = re.search('^(\d{1,2})月$', text).group(1)
  1686. if len(m) < 2:
  1687. m = '0' + m
  1688. year = re.search('(\d{4})年(.{,12}采购意向)?', html)
  1689. if year:
  1690. y = year.group(1)
  1691. num = self.get_monthlen(y, m)
  1692. if len(num) < 2:
  1693. num = '0' + num
  1694. order_begin = "%s-%s-01" % (y, m)
  1695. order_end = "%s-%s-%s" % (y, m, num)
  1696. elif page_time != "":
  1697. year = re.search('\d{4}', page_time)
  1698. if year:
  1699. y = year.group(0)
  1700. num = self.get_monthlen(y, m)
  1701. if len(num) < 2:
  1702. num = '0' + num
  1703. order_begin = "%s-%s-01" % (y, m)
  1704. order_end = "%s-%s-%s" % (y, m, num)
  1705. else:
  1706. y = str(datetime.datetime.now().year)
  1707. num = self.get_monthlen(y, m)
  1708. if len(num) < 2:
  1709. num = '0' + num
  1710. order_begin = "%s-%s-01" % (y, m)
  1711. order_end = "%s-%s-%s" % (y, m, num)
  1712. else:
  1713. y = str(datetime.datetime.now().year)
  1714. num = self.get_monthlen(y, m)
  1715. if len(num) < 2:
  1716. num = '0' + num
  1717. order_begin = "%s-%s-01" % (y, m)
  1718. order_end = "%s-%s-%s" % (y, m, num)
  1719. return order_begin, order_end
  1720. t1 = re.search('^(\d{4})(年|/|.|-)(\d{1,2})月?$', text)
  1721. if t1:
  1722. year = t1.group(1)
  1723. month = t1.group(3)
  1724. num = self.get_monthlen(year, month)
  1725. if len(month)<2:
  1726. month = '0'+month
  1727. if len(num) < 2:
  1728. num = '0'+num
  1729. order_begin = "%s-%s-01" % (year, month)
  1730. order_end = "%s-%s-%s" % (year, month, num)
  1731. return order_begin, order_end
  1732. t2 = re.search('^(\d{4})(年|/|.|-)(\d{1,2})(月|/|.|-)(\d{1,2})日?$', text)
  1733. if t2:
  1734. y = t2.group(1)
  1735. m = t2.group(3)
  1736. d = t2.group(5)
  1737. m = '0'+ m if len(m)<2 else m
  1738. d = '0'+d if len(d)<2 else d
  1739. order_begin = order_end = "%s-%s-%s"%(y,m,d)
  1740. return order_begin, order_end
  1741. all_match = re.finditer('^(?P<y1>\d{4})(年|/|.)(?P<m1>\d{1,2})(?:(月|/|.)(?:(?P<d1>\d{1,2})日)?)?'
  1742. '(到|至|-)(?:(?P<y2>\d{4})(年|/|.))?(?P<m2>\d{1,2})(?:(月|/|.)'
  1743. '(?:(?P<d2>\d{1,2})日)?)?$', text)
  1744. y1 = m1 = d1 = y2 = m2 = d2 = ""
  1745. found_math = False
  1746. for _match in all_match:
  1747. if len(_match.group()) > 0:
  1748. found_math = True
  1749. for k, v in _match.groupdict().items():
  1750. if v!="" and v is not None:
  1751. if k == 'y1':
  1752. y1 = v
  1753. elif k == 'm1':
  1754. m1 = v
  1755. elif k == 'd1':
  1756. d1 = v
  1757. elif k == 'y2':
  1758. y2 = v
  1759. elif k == 'm2':
  1760. m2 = v
  1761. elif k == 'd2':
  1762. d2 = v
  1763. if not found_math:
  1764. return "", ""
  1765. y2 = y1 if y2 == "" else y2
  1766. d1 = '1' if d1 == "" else d1
  1767. d2 = self.get_monthlen(y2, m2) if d2 == "" else d2
  1768. m1 = '0' + m1 if len(m1) < 2 else m1
  1769. m2 = '0' + m2 if len(m2) < 2 else m2
  1770. d1 = '0' + d1 if len(d1) < 2 else d1
  1771. d2 = '0' + d2 if len(d2) < 2 else d2
  1772. order_begin = "%s-%s-%s"%(y1,m1,d1)
  1773. order_end = "%s-%s-%s"%(y2,m2,d2)
  1774. return order_begin, order_end
  1775. def find_header(self, items, p1, p2):
  1776. '''
  1777. inner_table 每行正则检查是否为表头,是则返回表头所在列序号,及表头内容
  1778. :param items: 列表,内容为每个td 文本内容
  1779. :param p1: 优先表头正则
  1780. :param p2: 第二表头正则
  1781. :return: 表头所在列序号,是否表头,表头内容
  1782. '''
  1783. flag = False
  1784. header_dic = {'名称': '', '数量': '', '单价': '', '品牌': '', '规格': '', '需求': '', '预算': '', '时间': ''}
  1785. product = "" # 产品
  1786. quantity = "" # 数量
  1787. unitPrice = "" # 单价
  1788. brand = "" # 品牌
  1789. specs = "" # 规格
  1790. demand = "" # 采购需求
  1791. budget = "" # 预算金额
  1792. order_time = "" # 采购时间
  1793. for i in range(min(4, len(items))):
  1794. it = items[i]
  1795. if len(it) < 15 and re.search(p1, it) != None:
  1796. flag = True
  1797. product = it
  1798. header_dic['名称'] = i
  1799. break
  1800. if not flag:
  1801. for i in range(min(4, len(items))):
  1802. it = items[i]
  1803. if len(it) < 15 and re.search(p2, it) and re.search(
  1804. '编号|编码|号|情况|报名|单位|位置|地址|数量|单价|价格|金额|品牌|规格类型|型号|公司|中标人|企业|供应商|候选人', it) == None:
  1805. flag = True
  1806. product = it
  1807. header_dic['名称'] = i
  1808. break
  1809. if flag:
  1810. for j in range(i + 1, len(items)):
  1811. if len(items[j]) > 20 and len(re.sub('[\((].*[)\)]|[^\u4e00-\u9fa5]', '', items[j])) > 10:
  1812. continue
  1813. if header_dic['数量']=="" and re.search('数量', items[j]):
  1814. header_dic['数量'] = j
  1815. quantity = items[j]
  1816. elif re.search('单价', items[j]):
  1817. header_dic['单价'] = j
  1818. unitPrice = items[j]
  1819. elif re.search('品牌', items[j]):
  1820. header_dic['品牌'] = j
  1821. brand = items[j]
  1822. elif re.search('规格', items[j]):
  1823. header_dic['规格'] = j
  1824. specs = items[j]
  1825. elif re.search('需求', items[j]):
  1826. header_dic['需求'] = j
  1827. demand = items[j]
  1828. elif re.search('预算', items[j]):
  1829. header_dic['预算'] = j
  1830. budget = items[j]
  1831. elif re.search('时间|采购实施月份|采购月份', items[j]):
  1832. header_dic['时间'] = j
  1833. order_time = items[j]
  1834. if header_dic.get('名称', "") != "" :
  1835. num = 0
  1836. for it in (quantity, unitPrice, brand, specs, product, demand, budget, order_time):
  1837. if it != "":
  1838. num += 1
  1839. if num >=2:
  1840. return header_dic, flag, (product, quantity, unitPrice, brand, specs), (product, demand, budget, order_time)
  1841. flag = False
  1842. return header_dic, flag, (product, quantity, unitPrice, brand, specs), (product, demand, budget, order_time)
  1843. def predict(self, docid='', html='', page_time=""):
  1844. '''
  1845. 正则寻找table表格内 产品相关信息
  1846. :param html:公告HTML原文
  1847. :return:公告表格内 产品、数量、单价、品牌、规格 ,表头,表头列等信息
  1848. '''
  1849. soup = BeautifulSoup(html, 'lxml')
  1850. flag_yx = True if re.search('采购意向', html) else False
  1851. tables = soup.find_all(['table'])
  1852. headers = []
  1853. headers_demand = []
  1854. header_col = []
  1855. product_link = []
  1856. demand_link = []
  1857. total_product_money = 0
  1858. for i in range(len(tables)-1, -1, -1):
  1859. table = tables[i]
  1860. if table.parent.name == 'td' and len(table.find_all('td')) <= 3:
  1861. table.string = table.get_text()
  1862. table.name = 'turntable'
  1863. continue
  1864. if not self.isTrueTable(table):
  1865. continue
  1866. self.fixSpan(table)
  1867. inner_table = self.getTable(table)
  1868. i = 0
  1869. found_header = False
  1870. header_colnum = 0
  1871. if flag_yx:
  1872. col0_l = []
  1873. col1_l = []
  1874. for tds in inner_table:
  1875. if len(tds) == 2:
  1876. col0_l.append(re.sub(':', '', tds[0]))
  1877. col1_l.append(tds[1])
  1878. if len(set(col0_l) & self.header_set) > len(col0_l) * 0.2:
  1879. header_list2 = []
  1880. product = demand = budget = order_begin = order_end = ""
  1881. for i in range(len(col0_l)):
  1882. if re.search('项目名称', col0_l[i]):
  1883. header_list2.append(col0_l[i])
  1884. product = col1_l[i]
  1885. elif re.search('采购需求|需求概况', col0_l[i]):
  1886. header_list2.append(col0_l[i])
  1887. demand = col1_l[i]
  1888. elif re.search('采购预算|预算金额', col0_l[i]):
  1889. header_list2.append(col0_l[i])
  1890. budget = col1_l[i]
  1891. if '万元' in col0_l[i] and '万' not in budget:
  1892. budget += '万元'
  1893. budget = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", budget)
  1894. budget = str(getUnifyMoney(budget))
  1895. elif re.search('采购时间|采购实施月份|采购月份', col0_l[i]):
  1896. header_list2.append(col0_l[i])
  1897. order_time = col1_l[i].strip()
  1898. order_begin, order_end = self.fix_time(order_time, html, page_time)
  1899. if product!= "" and demand != "" and budget!="" and order_begin != "":
  1900. link = {'project_name': product, 'product': [], 'demand': demand, 'budget': budget,
  1901. 'order_begin': order_begin, 'order_end': order_end}
  1902. if link not in demand_link:
  1903. demand_link.append(link)
  1904. headers_demand.append('_'.join(header_list2))
  1905. continue
  1906. while i < (len(inner_table)):
  1907. tds = inner_table[i]
  1908. not_empty = [it for it in tds if it != ""]
  1909. if len(set(not_empty)) < len(not_empty) * 0.5 or len(tds)<2:
  1910. i += 1
  1911. continue
  1912. product = "" # 产品
  1913. quantity = "" # 数量
  1914. unitPrice = "" # 单价
  1915. brand = "" # 品牌
  1916. specs = "" # 规格
  1917. demand = "" # 采购需求
  1918. budget = "" # 预算金额
  1919. order_time = "" # 采购时间
  1920. order_begin = ""
  1921. order_end = ""
  1922. if len(set(tds) & self.header_set) > len(tds) * 0.2:
  1923. header_dic, found_header, header_list, header_list2 = self.find_header(tds, self.p1, self.p2)
  1924. if found_header:
  1925. headers.append('_'.join(header_list))
  1926. headers_demand.append('_'.join(header_list2))
  1927. header_colnum = len(tds)
  1928. header_col.append('_'.join(tds))
  1929. i += 1
  1930. continue
  1931. elif found_header:
  1932. if len(tds) != header_colnum: # 表头、属性列数不一致跳过
  1933. i += 1
  1934. continue
  1935. id1 = header_dic.get('名称', "")
  1936. id2 = header_dic.get('数量', "")
  1937. id3 = header_dic.get('单价', "")
  1938. id4 = header_dic.get('品牌', "")
  1939. id5 = header_dic.get('规格', "")
  1940. id6 = header_dic.get('需求', "")
  1941. id7 = header_dic.get('预算', "")
  1942. id8 = header_dic.get('时间', "")
  1943. if re.search('[a-zA-Z\u4e00-\u9fa5]', tds[id1]) and tds[id1] not in self.header_set and \
  1944. re.search('备注|汇总|合计|总价|价格|金额|公司|附件|详见|无$|xxx', tds[id1]) == None:
  1945. product = tds[id1]
  1946. if id2 != "":
  1947. if re.search('\d+|[壹贰叁肆伍陆柒捌玖拾一二三四五六七八九十]', tds[id2]):
  1948. quantity = tds[id2]
  1949. else:
  1950. quantity = ""
  1951. if id3 != "":
  1952. if re.search('\d+|[零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,}', tds[id3]):
  1953. unitPrice = tds[id3]
  1954. if '万元' in header_list[2] and '万' not in unitPrice:
  1955. unitPrice += '万元'
  1956. unitPrice = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", unitPrice)
  1957. unitPrice = str(getUnifyMoney(unitPrice))
  1958. else:
  1959. unitPrice = ""
  1960. if id4 != "":
  1961. if re.search('\w', tds[id4]):
  1962. brand = tds[id4]
  1963. else:
  1964. brand = ""
  1965. if id5 != "":
  1966. if re.search('\w', tds[id5]):
  1967. specs = tds[id5]
  1968. else:
  1969. specs = ""
  1970. if id6 != "":
  1971. if re.search('\w', tds[id6]):
  1972. demand = tds[id6]
  1973. else:
  1974. demand = ""
  1975. if id7 != "":
  1976. if re.search('\d+|[零壹贰叁肆伍陆柒捌玖拾佰仟萬億十百千万亿元角分]{3,}', tds[id7]):
  1977. budget = tds[id7]
  1978. if '万元' in header_list2[2] and '万' not in budget:
  1979. budget += '万元'
  1980. budget = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", budget)
  1981. budget = str(getUnifyMoney(budget))
  1982. else:
  1983. budget = ""
  1984. if id8 != "":
  1985. if re.search('\w', tds[id8]):
  1986. order_time = tds[id8].strip()
  1987. order_begin, order_end = self.fix_time(order_time, html, page_time)
  1988. if quantity != "" or unitPrice != "" or brand != "" or specs != "":
  1989. link = {'product': product, 'quantity': quantity, 'unitPrice': unitPrice,
  1990. 'brand': brand[:50], 'specs':specs}
  1991. if link not in product_link:
  1992. product_link.append(link)
  1993. mat = re.match('([0-9.,]+)[((]?\w{,3}[))]?$', link['quantity'])
  1994. if link['unitPrice'] != "" and mat:
  1995. try:
  1996. total_product_money += float(link['unitPrice'])*float(mat.group(1).replace(',', ''))
  1997. except:
  1998. log('产品属性单价数量相乘出错, 单价: %s, 数量: %s'%(link['unitPrice'], link['quantity']))
  1999. if budget != "" and order_time != "" :
  2000. link = {'project_name': product, 'product':[], 'demand': demand, 'budget': budget, 'order_begin':order_begin, 'order_end':order_end}
  2001. if link not in demand_link:
  2002. demand_link.append(link)
  2003. i += 1
  2004. else:
  2005. i += 1
  2006. if len(product_link)>0:
  2007. attr_dic = {'product_attrs':{'data':product_link, 'header':headers, 'header_col':header_col}}
  2008. else:
  2009. attr_dic = {'product_attrs': {'data': [], 'header': [], 'header_col': []}}
  2010. if len(demand_link)>0:
  2011. demand_dic = {'demand_info':{'data':demand_link, 'header':headers_demand, 'header_col':header_col}}
  2012. else:
  2013. demand_dic = {'demand_info':{'data':[], 'header':[], 'header_col':[]}}
  2014. return [attr_dic, demand_dic], total_product_money
  2015. # docchannel类型提取
  2016. class DocChannel():
  2017. def __init__(self, life_model='/channel_savedmodel/channel.pb', type_model='/channel_savedmodel/doctype.pb'):
  2018. self.lift_sess, self.lift_title, self.lift_content, self.lift_prob, self.lift_softmax,\
  2019. self.mask, self.mask_title = self.load_life(life_model)
  2020. self.type_sess, self.type_title, self.type_content, self.type_prob, self.type_softmax,\
  2021. self.type_mask, self.type_mask_title = self.load_type(type_model)
  2022. self.sequen_len = 200 # 150 200
  2023. self.title_len = 30
  2024. self.sentence_num = 10
  2025. self.kws = '供货商|候选人|供应商|入选人|项目|选定|预告|中标|成交|补遗|延期|报名|暂缓|结果|意向|出租|补充|合同|限价|比选|指定|工程|废标|取消|中止|流标|资质|资格|地块|招标|采购|货物|租赁|计划|宗地|需求|来源|土地|澄清|失败|探矿|预审|变更|变卖|遴选|撤销|意见|恢复|采矿|更正|终止|废置|报建|流拍|供地|登记|挂牌|答疑|中选|受让|拍卖|竞拍|审查|入围|更改|条件|洽谈|乙方|后审|控制|暂停|用地|询价|预'
  2026. lb_type = ['采招数据', '土地矿产', '拍卖出让', '产权交易', '新闻资讯']
  2027. lb_life = ['采购意向', '招标预告', '招标公告', '招标答疑', '公告变更', '资审结果', '中标信息', '合同公告', '废标公告']
  2028. self.id2type = {k: v for k, v in enumerate(lb_type)}
  2029. self.id2life = {k: v for k, v in enumerate(lb_life)}
  2030. def load_life(self,life_model):
  2031. with tf.Graph().as_default() as graph:
  2032. output_graph_def = graph.as_graph_def()
  2033. with open(os.path.dirname(__file__)+life_model, 'rb') as f:
  2034. output_graph_def.ParseFromString(f.read())
  2035. tf.import_graph_def(output_graph_def, name='')
  2036. print("%d ops in the final graph" % len(output_graph_def.node))
  2037. del output_graph_def
  2038. sess = tf.Session(graph=graph)
  2039. sess.run(tf.global_variables_initializer())
  2040. inputs = sess.graph.get_tensor_by_name('inputs/inputs:0')
  2041. prob = sess.graph.get_tensor_by_name('inputs/dropout:0')
  2042. title = sess.graph.get_tensor_by_name('inputs/title:0')
  2043. mask = sess.graph.get_tensor_by_name('inputs/mask:0')
  2044. mask_title = sess.graph.get_tensor_by_name('inputs/mask_title:0')
  2045. # logit = sess.graph.get_tensor_by_name('output/logit:0')
  2046. softmax = sess.graph.get_tensor_by_name('output/softmax:0')
  2047. return sess, title, inputs, prob, softmax, mask, mask_title
  2048. def load_type(self,type_model):
  2049. with tf.Graph().as_default() as graph:
  2050. output_graph_def = graph.as_graph_def()
  2051. with open(os.path.dirname(__file__)+type_model, 'rb') as f:
  2052. output_graph_def.ParseFromString(f.read())
  2053. tf.import_graph_def(output_graph_def, name='')
  2054. print("%d ops in the final graph" % len(output_graph_def.node))
  2055. del output_graph_def
  2056. sess = tf.Session(graph=graph)
  2057. sess.run(tf.global_variables_initializer())
  2058. inputs = sess.graph.get_tensor_by_name('inputs/inputs:0')
  2059. prob = sess.graph.get_tensor_by_name('inputs/dropout:0')
  2060. title = sess.graph.get_tensor_by_name('inputs/title:0')
  2061. mask = sess.graph.get_tensor_by_name('inputs/mask:0')
  2062. mask_title = sess.graph.get_tensor_by_name('inputs/mask_title:0')
  2063. # logit = sess.graph.get_tensor_by_name('output/logit:0')
  2064. softmax = sess.graph.get_tensor_by_name('output/softmax:0')
  2065. return sess, title, inputs, prob, softmax, mask, mask_title
  2066. def predict_process(self, docid='', doctitle='', dochtmlcon=''):
  2067. # print('准备预处理')
  2068. def get_kw_senten(s, span=10):
  2069. doc_sens = []
  2070. tmp = 0
  2071. num = 0
  2072. end_idx = 0
  2073. for it in re.finditer(self.kws, s): # '|'.join(keywordset)
  2074. left = s[end_idx:it.end()].split()
  2075. right = s[it.end():].split()
  2076. tmp_seg = s[tmp:it.start()].split()
  2077. if len(tmp_seg) > span or tmp == 0:
  2078. doc_sens.append(' '.join(left[-span:] + right[:span]))
  2079. end_idx = it.end() + 1 + len(' '.join(right[:span]))
  2080. tmp = it.end()
  2081. num += 1
  2082. if num >= self.sentence_num:
  2083. break
  2084. if doc_sens == []:
  2085. doc_sens.append(s)
  2086. return doc_sens
  2087. def word2id(wordlist, max_len=self.sequen_len):
  2088. ids = [getIndexOfWords(w) for w in wordlist]
  2089. ids = ids[:max_len] if len(ids) >= max_len else ids + [0] * (max_len - len(ids))
  2090. assert len(ids) == max_len
  2091. return ids
  2092. cost_time = dict()
  2093. datas = []
  2094. datas_title = []
  2095. try:
  2096. segword_title = ' '.join(selffool.cut(doctitle)[0])
  2097. segword_content = dochtmlcon
  2098. except:
  2099. segword_content = ''
  2100. segword_title = ''
  2101. if isinstance(segword_content, float):
  2102. segword_content = ''
  2103. if isinstance(segword_title, float):
  2104. segword_title = ''
  2105. segword_content = segword_content.replace(' 中 选 ', ' 中选 ').replace(' 中 标 ', ' 中标 ').replace(' 补 遗 ', ' 补遗 '). \
  2106. replace(' 更 多', '').replace(' 更多', '').replace(' 中 号 ', ' 中标 ').replace(' 中 选人 ', ' 中选人 '). \
  2107. replace(' 点击 下载 查看', '').replace(' 咨询 报价 请 点击', '').replace('终结', '终止')
  2108. segword_title = re.sub('[^\s\u4e00-\u9fa5]', '', segword_title)
  2109. segword_content = re.sub('[^\s\u4e00-\u9fa5]', '', segword_content)
  2110. doc_word_list = segword_content.split()
  2111. if len(doc_word_list) > self.sequen_len / 2:
  2112. doc_sens = get_kw_senten(' '.join(doc_word_list[100:500]))
  2113. doc_sens = ' '.join(doc_word_list[:100]) + '\n' + '\n'.join(doc_sens)
  2114. else:
  2115. doc_sens = ' '.join(doc_word_list[:self.sequen_len])
  2116. # print('标题:',segword_title)
  2117. # print('正文:',segword_content)
  2118. datas.append(doc_sens.split())
  2119. datas_title.append(segword_title.split())
  2120. # print('完成预处理')
  2121. return datas, datas_title
  2122. def is_houxuan(self, title, content):
  2123. '''
  2124. 通过标题和中文内容判断是否属于候选人公示类别
  2125. :param title: 公告标题
  2126. :param content: 公告正文文本内容
  2127. :return: 1 是候选人公示 ;0 不是
  2128. '''
  2129. if re.search('候选人的?公示|评标结果|评审结果|中标公示', title): # (中标|成交|中选|入围)
  2130. if re.search('变更公告|更正公告|废标|终止|答疑|澄清', title):
  2131. return 0
  2132. return 1
  2133. if re.search('候选人的?公示', content[:100]):
  2134. if re.search('公示(期|活动)?已经?结束|公示期已满|中标结果公告|中标结果公示|变更公告|更正公告|废标|终止|答疑|澄清', content[:100]):
  2135. return 0
  2136. return 1
  2137. else:
  2138. return 0
  2139. def predict(self, title='', list_sentence='', web_source_no=''):
  2140. if web_source_no in ['02104-7']:
  2141. return {'docchannel': {'docchannel':'', 'doctype':'采招数据'}}
  2142. if isinstance(list_sentence, list):
  2143. token_l = [it.tokens for it in list_sentence]
  2144. tokens = [it for l in token_l for it in l]
  2145. content = ' '.join(tokens[:500])
  2146. title = re.sub('[^\u4e00-\u9fa5]', '', title)
  2147. if len(title)>50:
  2148. title = title[:20]+title[-30:]
  2149. data_content, data_title = self.predict_process(docid='', doctitle=title[-50:], dochtmlcon=content) # 标题最多取50字
  2150. text_len = len(data_content[0]) if len(data_content[0])<self.sequen_len else self.sequen_len
  2151. title_len = len(data_title[0]) if len(data_title[0])<self.title_len else self.title_len
  2152. result = {'docchannel': {'docchannel':'', 'doctype':''}}
  2153. array_content = embedding(data_content, shape=(len(data_content), self.sequen_len, 128))
  2154. array_title = embedding(data_title, shape=(len(data_title), self.title_len, 128))
  2155. pred = self.type_sess.run(self.type_softmax,
  2156. feed_dict={
  2157. self.type_title: array_title,
  2158. self.type_content: array_content,
  2159. self.type_mask:[[0]*text_len+[1]*(self.sequen_len-text_len)],
  2160. self.type_mask_title:[[0]*title_len+[1]*(self.title_len-title_len)],
  2161. self.type_prob:1}
  2162. )
  2163. id = np.argmax(pred, axis=1)[0]
  2164. prob = pred[0][id]
  2165. result['docchannel']['doctype'] = self.id2type[id]
  2166. # print('公告类别:', self.id2type[id], '概率:',prob)
  2167. # if id == 0:
  2168. if result['docchannel']['doctype'] not in ['', '新闻资讯']:
  2169. pred = self.lift_sess.run(self.lift_softmax,
  2170. feed_dict={
  2171. self.lift_title: array_title,
  2172. self.lift_content: array_content,
  2173. self.mask: [[0] * text_len + [1] * (self.sequen_len - text_len)],
  2174. self.mask_title: [[0] * title_len + [1] * (self.title_len - title_len)],
  2175. self.lift_prob:1}
  2176. )
  2177. id = np.argmax(pred, axis=1)[0]
  2178. prob = pred[0][id]
  2179. result['docchannel']['docchannel'] = self.id2life[id]
  2180. # print('生命周期:',self.id2life[id], '概率:',prob)
  2181. # if id == 6:
  2182. if result['docchannel']['docchannel'] == '中标信息':
  2183. if self.is_houxuan(''.join([it for it in title if it.isalpha()]), ''.join([it for it in content if it.isalpha()])):
  2184. result['docchannel']['docchannel'] = '候选人公示'
  2185. # return '候选人公示', prob
  2186. # return [{'docchannel': '候选人公示'}]
  2187. return result
  2188. # return [{'docchannel':self.id2life[id]}]
  2189. # else:
  2190. # # return self.id2type[id], prob
  2191. # return [{'docchannel':self.id2type[id]}]
  2192. def predict_rule(self, title, content, channel_dic, prem_dic):
  2193. '''2022/2/10加入规则去除某些数据源及内容过短且不包含类别关键词的公告不做预测'''
  2194. hetong = '(合同|验收|履约)(公告|公示)|合同号?$' # 合同标题正则
  2195. zhongbiao_t = '(中标|中选|成交|入选|入围|结果|确认)(公告|公示|结果)|(遴选|采购|招标|竞价|议价|比选|询价|评选|谈判|邀标|邀请|洽谈|约谈|评标|发包|遴选)结果|开标(记录|信息|情况)|单一来源|直接(选取|选定)|中标通知书|中标$'
  2196. zhongbiao_c = '(中标|中选|成交|拟选用|拟邀请|最终选定的?|拟定)(供应商|供货商|服务商|企业|公司|单位|(候选)?人)(名称)?[::]|[,。:.](供应商|供货商|服务商)(名称)?:|指定的中介服务机构:|建设服务单位:'
  2197. zhaobiao_t = '(遴选|采购|招标|竞价|议价|比选|询价|评选|谈判|邀标|邀请|洽谈|约谈)(公告|公示|$)'
  2198. title_cn = re.sub('[^\u4e00-\u9fa5]', '', title)
  2199. if len(re.sub('[^\u4e00-\u9fa5]', "", content))<50 and channel_dic['docchannel']['doctype'] != '新闻资讯':
  2200. if re.search(hetong, title_cn) != None:
  2201. channel_dic['docchannel']['docchannel'] = '合同公告'
  2202. elif re.search(zhongbiao_t, title_cn):
  2203. channel_dic['docchannel']['docchannel'] = '中标信息'
  2204. elif re.search(zhaobiao_t, title_cn):
  2205. channel_dic['docchannel']['docchannel'] = '招标公告'
  2206. else:
  2207. channel_dic['docchannel']['docchannel'] = ''
  2208. elif channel_dic['docchannel'].get('docchannel', '') == '招标公告' and 'win_tenderer' in json.dumps(prem_dic,
  2209. ensure_ascii=False):
  2210. if re.search(hetong, title_cn) != None:
  2211. channel_dic['docchannel']['docchannel'] = '合同公告'
  2212. log('正则把招标公告修改为合同公告')
  2213. elif re.search(zhongbiao_t, title_cn) or re.search(zhongbiao_t, content[:200]) or re.search(zhongbiao_c,
  2214. content):
  2215. channel_dic['docchannel']['docchannel'] = '中标信息'
  2216. log('正则把招标公告修改为中标信息')
  2217. elif channel_dic['docchannel'].get('docchannel', '') == '中标信息' and 'win_tenderer' not in json.dumps(prem_dic,
  2218. ensure_ascii=False):
  2219. if re.search(hetong, title_cn):
  2220. channel_dic['docchannel']['docchannel'] = '合同公告'
  2221. log('正则把中标信息修改为合同公告')
  2222. elif re.search(zhongbiao_t, title_cn) or re.search(zhongbiao_t, content[:200]) or re.search(zhongbiao_c,
  2223. content):
  2224. pass
  2225. elif re.search(zhaobiao_t, title_cn):
  2226. channel_dic['docchannel']['docchannel'] = '招标公告'
  2227. log('正则把中标信息修改为招标公告')
  2228. elif re.search('中标|成交|中选|入选|入围|结果|供应商|供货商|候选人', title_cn+content)==None:
  2229. channel_dic['docchannel']['docchannel'] = ''
  2230. log('正则把中标信息修改为空')
  2231. return channel_dic
  2232. # 保证金支付方式提取
  2233. class DepositPaymentWay():
  2234. def __init__(self,):
  2235. self.pt = '(保证金的?(交纳|缴纳|应按下列|入账|支付)方式)[::]*([^,。]{,60})'
  2236. self.pt2 = '保证金(必?须以|必?须?通过|以)(.{,8})方式'
  2237. kws = ['银行转账', '公?对公方?式?转账', '对公转账', '柜台转账', '(线上|网上)自?行?(缴纳|交纳|缴退|收退)',
  2238. '网上银行支付', '现金存入', '直接缴纳', '支票', '汇票', '本票', '电汇', '转账', '汇款', '随机码',
  2239. '入账', '基本账户转出', '基本账户汇入', '诚信库中登记的账户转出',
  2240. '银行保函', '电子保函', '担保函', '保证保险', '合法担保机构出具的担保', '金融机构、担保机构出具的保函']
  2241. self.kws = sorted(kws, key=lambda x: len(x), reverse=True)
  2242. def predict(self,content):
  2243. pay_way = {'deposit_patment_way':''}
  2244. result = []
  2245. pay = re.search(self.pt, content)
  2246. if pay:
  2247. # print(pay.group(0))
  2248. pay = pay.group(3)
  2249. for it in re.finditer('|'.join(self.kws), pay):
  2250. result.append(it.group(0))
  2251. pay_way['deposit_patment_way'] = ';'.join(result)
  2252. return pay_way
  2253. pay = re.search(self.pt2, content)
  2254. if pay:
  2255. # print(pay.group(0))
  2256. pay = pay.group(2)
  2257. for it in re.finditer('|'.join(self.kws), pay):
  2258. result.append(it.group(0))
  2259. pay_way['deposit_patment_way'] = ';'.join(result)
  2260. return pay_way
  2261. else:
  2262. return pay_way
  2263. # 总价单价提取
  2264. class TotalUnitMoney:
  2265. def __init__(self):
  2266. pass
  2267. def predict(self, list_sentences, list_entitys):
  2268. for i in range(len(list_entitys)):
  2269. list_entity = list_entitys[i]
  2270. # 总价单价
  2271. for _entity in list_entity:
  2272. if _entity.entity_type == 'money':
  2273. word_of_sentence = list_sentences[i][_entity.sentence_index].sentence_text
  2274. # 总价在中投标金额中
  2275. if _entity.label == 1:
  2276. result = extract_total_money(word_of_sentence,
  2277. _entity.entity_text,
  2278. [_entity.wordOffset_begin, _entity.wordOffset_end])
  2279. if result:
  2280. _entity.is_total_money = 1
  2281. # 单价在普通金额中
  2282. else:
  2283. result = extract_unit_money(word_of_sentence,
  2284. _entity.entity_text,
  2285. [_entity.wordOffset_begin, _entity.wordOffset_end])
  2286. if result:
  2287. _entity.is_unit_money = 1
  2288. # print("total_unit_money", _entity.entity_text,
  2289. # _entity.is_total_money, _entity.is_unit_money)
  2290. def getSavedModel():
  2291. #predictor = FormPredictor()
  2292. graph = tf.Graph()
  2293. with graph.as_default():
  2294. model = tf.keras.models.load_model("../form/model/model_form.model_item.hdf5",custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  2295. #print(tf.graph_util.remove_training_nodes(model))
  2296. tf.saved_model.simple_save(
  2297. tf.keras.backend.get_session(),
  2298. "./h5_savedmodel/",
  2299. inputs={"image": model.input},
  2300. outputs={"scores": model.output}
  2301. )
  2302. def getBiLSTMCRFModel(MAX_LEN,vocab,EMBED_DIM,BiRNN_UNITS,chunk_tags,weights):
  2303. '''
  2304. model = models.Sequential()
  2305. model.add(layers.Embedding(len(vocab), EMBED_DIM, mask_zero=True)) # Random embedding
  2306. model.add(layers.Bidirectional(layers.LSTM(BiRNN_UNITS // 2, return_sequences=True)))
  2307. crf = CRF(len(chunk_tags), sparse_target=True)
  2308. model.add(crf)
  2309. model.summary()
  2310. model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])
  2311. return model
  2312. '''
  2313. input = layers.Input(shape=(None,),dtype="int32")
  2314. if weights is not None:
  2315. embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True,weights=[weights],trainable=True)(input)
  2316. else:
  2317. embedding = layers.embeddings.Embedding(len(vocab),EMBED_DIM,mask_zero=True)(input)
  2318. bilstm = layers.Bidirectional(layers.LSTM(BiRNN_UNITS//2,return_sequences=True))(embedding)
  2319. bilstm_dense = layers.TimeDistributed(layers.Dense(len(chunk_tags)))(bilstm)
  2320. crf = CRF(len(chunk_tags),sparse_target=True)
  2321. crf_out = crf(bilstm_dense)
  2322. model = models.Model(input=[input],output = [crf_out])
  2323. model.summary()
  2324. model.compile(optimizer = 'adam', loss = crf.loss_function, metrics = [crf.accuracy])
  2325. return model
  2326. import h5py
  2327. def h5_to_graph(sess,graph,h5file):
  2328. f = h5py.File(h5file,'r') #打开h5文件
  2329. def getValue(v):
  2330. _value = f["model_weights"]
  2331. list_names = str(v.name).split("/")
  2332. for _index in range(len(list_names)):
  2333. print(v.name)
  2334. if _index==1:
  2335. _value = _value[list_names[0]]
  2336. _value = _value[list_names[_index]]
  2337. return _value.value
  2338. def _load_attributes_from_hdf5_group(group, name):
  2339. """Loads attributes of the specified name from the HDF5 group.
  2340. This method deals with an inherent problem
  2341. of HDF5 file which is not able to store
  2342. data larger than HDF5_OBJECT_HEADER_LIMIT bytes.
  2343. # Arguments
  2344. group: A pointer to a HDF5 group.
  2345. name: A name of the attributes to load.
  2346. # Returns
  2347. data: Attributes data.
  2348. """
  2349. if name in group.attrs:
  2350. data = [n.decode('utf8') for n in group.attrs[name]]
  2351. else:
  2352. data = []
  2353. chunk_id = 0
  2354. while ('%s%d' % (name, chunk_id)) in group.attrs:
  2355. data.extend([n.decode('utf8')
  2356. for n in group.attrs['%s%d' % (name, chunk_id)]])
  2357. chunk_id += 1
  2358. return data
  2359. def readGroup(gr,parent_name,data):
  2360. for subkey in gr:
  2361. print(subkey)
  2362. if parent_name!=subkey:
  2363. if parent_name=="":
  2364. _name = subkey
  2365. else:
  2366. _name = parent_name+"/"+subkey
  2367. else:
  2368. _name = parent_name
  2369. if str(type(gr[subkey]))=="<class 'h5py._hl.group.Group'>":
  2370. readGroup(gr[subkey],_name,data)
  2371. else:
  2372. data.append([_name,gr[subkey].value])
  2373. print(_name,gr[subkey].shape)
  2374. layer_names = _load_attributes_from_hdf5_group(f["model_weights"], 'layer_names')
  2375. list_name_value = []
  2376. readGroup(f["model_weights"], "", list_name_value)
  2377. '''
  2378. for k, name in enumerate(layer_names):
  2379. g = f["model_weights"][name]
  2380. weight_names = _load_attributes_from_hdf5_group(g, 'weight_names')
  2381. #weight_values = [np.asarray(g[weight_name]) for weight_name in weight_names]
  2382. for weight_name in weight_names:
  2383. list_name_value.append([weight_name,np.asarray(g[weight_name])])
  2384. '''
  2385. for name_value in list_name_value:
  2386. name = name_value[0]
  2387. '''
  2388. if re.search("dense",name) is not None:
  2389. name = name[:7]+"_1"+name[7:]
  2390. '''
  2391. value = name_value[1]
  2392. print(name,graph.get_tensor_by_name(name),np.shape(value))
  2393. sess.run(tf.assign(graph.get_tensor_by_name(name),value))
  2394. def initialize_uninitialized(sess):
  2395. global_vars = tf.global_variables()
  2396. is_not_initialized = sess.run([tf.is_variable_initialized(var) for var in global_vars])
  2397. not_initialized_vars = [v for (v, f) in zip(global_vars, is_not_initialized) if not f]
  2398. adam_vars = []
  2399. for _vars in not_initialized_vars:
  2400. if re.search("Adam",_vars.name) is not None:
  2401. adam_vars.append(_vars)
  2402. print([str(i.name) for i in adam_vars]) # only for testing
  2403. if len(adam_vars):
  2404. sess.run(tf.variables_initializer(adam_vars))
  2405. def save_codename_model():
  2406. # filepath = "../projectCode/models/model_project_"+str(60)+"_"+str(200)+".hdf5"
  2407. filepath = "../projectCode/models_tf/59-L0.471516189943-F0.8802154826344823-P0.8789179683459191-R0.8815168335321886/model.ckpt"
  2408. vocabpath = "../projectCode/models/vocab.pk"
  2409. classlabelspath = "../projectCode/models/classlabels.pk"
  2410. # vocab = load(vocabpath)
  2411. # class_labels = load(classlabelspath)
  2412. w2v_matrix = load('codename_w2v_matrix.pk')
  2413. graph = tf.get_default_graph()
  2414. with graph.as_default() as g:
  2415. ''''''
  2416. # model = getBiLSTMCRFModel(None, vocab, 60, 200, class_labels,weights=None)
  2417. #model = models.load_model(filepath,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score,"CRF":CRF,"loss":CRF.loss_function})
  2418. sess = tf.Session(graph=g)
  2419. # sess = tf.keras.backend.get_session()
  2420. char_input, logits, target, keepprob, length, crf_loss, trans, train_op = BiLSTM_CRF_tfmodel(sess, w2v_matrix)
  2421. #with sess.as_default():
  2422. sess.run(tf.global_variables_initializer())
  2423. # print(sess.run("time_distributed_1/kernel:0"))
  2424. # model.load_weights(filepath)
  2425. saver = tf.train.Saver()
  2426. saver.restore(sess, filepath)
  2427. # print("logits",sess.run(logits))
  2428. # print("#",sess.run("time_distributed_1/kernel:0"))
  2429. # x = load("codename_x.pk")
  2430. #y = model.predict(x)
  2431. # y = sess.run(model.output,feed_dict={model.input:x})
  2432. # for item in np.argmax(y,-1):
  2433. # print(item)
  2434. tf.saved_model.simple_save(
  2435. sess,
  2436. "./codename_savedmodel_tf/",
  2437. inputs={"inputs": char_input,
  2438. "inputs_length":length,
  2439. 'keepprob':keepprob},
  2440. outputs={"logits": logits,
  2441. "trans":trans}
  2442. )
  2443. def save_role_model():
  2444. '''
  2445. @summary: 保存model为savedModel,部署到PAI平台上调用
  2446. '''
  2447. model_role = PREMPredict().model_role
  2448. with model_role.graph.as_default():
  2449. model = model_role.getModel()
  2450. sess = tf.Session(graph=model_role.graph)
  2451. print(type(model.input))
  2452. sess.run(tf.global_variables_initializer())
  2453. h5_to_graph(sess, model_role.graph, model_role.model_role_file)
  2454. model = model_role.getModel()
  2455. tf.saved_model.simple_save(sess,
  2456. "./role_savedmodel/",
  2457. inputs={"input0":model.input[0],
  2458. "input1":model.input[1],
  2459. "input2":model.input[2]},
  2460. outputs={"outputs":model.output}
  2461. )
  2462. def save_money_model():
  2463. model_file = os.path.dirname(__file__)+"/../money/models/model_money_word.h5"
  2464. graph = tf.Graph()
  2465. with graph.as_default():
  2466. sess = tf.Session(graph=graph)
  2467. with sess.as_default():
  2468. # model = model_money.getModel()
  2469. # model.summary()
  2470. # sess.run(tf.global_variables_initializer())
  2471. # h5_to_graph(sess, model_money.graph, model_money.model_money_file)
  2472. model = models.load_model(model_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  2473. model.summary()
  2474. print(model.weights)
  2475. tf.saved_model.simple_save(sess,
  2476. "./money_savedmodel2/",
  2477. inputs = {"input0":model.input[0],
  2478. "input1":model.input[1],
  2479. "input2":model.input[2]},
  2480. outputs = {"outputs":model.output}
  2481. )
  2482. def save_person_model():
  2483. model_person = EPCPredict().model_person
  2484. with model_person.graph.as_default():
  2485. x = load("person_x.pk")
  2486. _data = np.transpose(np.array(x),(1,0,2,3))
  2487. model = model_person.getModel()
  2488. sess = tf.Session(graph=model_person.graph)
  2489. with sess.as_default():
  2490. sess.run(tf.global_variables_initializer())
  2491. model_person.load_weights()
  2492. #h5_to_graph(sess, model_person.graph, model_person.model_person_file)
  2493. predict_y = sess.run(model.output,feed_dict={model.input[0]:_data[0],model.input[1]:_data[1]})
  2494. #predict_y = model.predict([_data[0],_data[1]])
  2495. print(np.argmax(predict_y,-1))
  2496. tf.saved_model.simple_save(sess,
  2497. "./person_savedmodel/",
  2498. inputs={"input0":model.input[0],
  2499. "input1":model.input[1]},
  2500. outputs = {"outputs":model.output})
  2501. def save_form_model():
  2502. model_form = FormPredictor()
  2503. with model_form.graph.as_default():
  2504. model = model_form.getModel("item")
  2505. sess = tf.Session(graph=model_form.graph)
  2506. sess.run(tf.global_variables_initializer())
  2507. h5_to_graph(sess, model_form.graph, model_form.model_file_item)
  2508. tf.saved_model.simple_save(sess,
  2509. "./form_savedmodel/",
  2510. inputs={"inputs":model.input},
  2511. outputs = {"outputs":model.output})
  2512. def save_codesplit_model():
  2513. filepath_code = "../projectCode/models/model_code.hdf5"
  2514. graph = tf.Graph()
  2515. with graph.as_default():
  2516. model_code = models.load_model(filepath_code, custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  2517. sess = tf.Session()
  2518. sess.run(tf.global_variables_initializer())
  2519. h5_to_graph(sess, graph, filepath_code)
  2520. tf.saved_model.simple_save(sess,
  2521. "./codesplit_savedmodel/",
  2522. inputs={"input0":model_code.input[0],
  2523. "input1":model_code.input[1],
  2524. "input2":model_code.input[2]},
  2525. outputs={"outputs":model_code.output})
  2526. def save_timesplit_model():
  2527. filepath = '../time/model_label_time_classify.model.hdf5'
  2528. with tf.Graph().as_default() as graph:
  2529. time_model = models.load_model(filepath, custom_objects={'precision': precision, 'recall': recall, 'f1_score': f1_score})
  2530. with tf.Session() as sess:
  2531. sess.run(tf.global_variables_initializer())
  2532. h5_to_graph(sess, graph, filepath)
  2533. tf.saved_model.simple_save(sess,
  2534. "./timesplit_model/",
  2535. inputs={"input0":time_model.input[0],
  2536. "input1":time_model.input[1]},
  2537. outputs={"outputs":time_model.output})
  2538. if __name__=="__main__":
  2539. #save_role_model()
  2540. # save_codename_model()
  2541. # save_money_model()
  2542. #save_person_model()
  2543. #save_form_model()
  2544. #save_codesplit_model()
  2545. # save_timesplit_model()
  2546. '''
  2547. # with tf.Session(graph=tf.Graph()) as sess:
  2548. # from tensorflow.python.saved_model import tag_constants
  2549. # meta_graph_def = tf.saved_model.loader.load(sess, [tag_constants.SERVING], "./person_savedModel")
  2550. # graph = tf.get_default_graph()
  2551. # signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  2552. # signature = meta_graph_def.signature_def
  2553. # input0 = sess.graph.get_tensor_by_name(signature[signature_key].inputs["input0"].name)
  2554. # input1 = sess.graph.get_tensor_by_name(signature[signature_key].inputs["input1"].name)
  2555. # outputs = sess.graph.get_tensor_by_name(signature[signature_key].outputs["outputs"].name)
  2556. # x = load("person_x.pk")
  2557. # _data = np.transpose(x,[1,0,2,3])
  2558. # y = sess.run(outputs,feed_dict={input0:_data[0],input1:_data[1]})
  2559. # print(np.argmax(y,-1))
  2560. '''