modelFactory.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638
  1. '''
  2. Created on 2019年5月16日
  3. @author: User
  4. '''
  5. import os
  6. import sys
  7. sys.path.append(os.path.abspath("../.."))
  8. from keras import models
  9. from keras import layers
  10. # from keras_contrib.layers import CRF
  11. from keras.preprocessing.sequence import pad_sequences
  12. from keras import optimizers,losses,metrics
  13. from BiddingKG.dl.common.Utils import *
  14. import tensorflow as tf
  15. import json
  16. class Model_role_classify():
  17. def __init__(self,lazyLoad=getLazyLoad()):
  18. #self.model_role_file = os.path.abspath("../role/models/model_role.model.hdf5")
  19. self.model_role_file = os.path.dirname(__file__)+"/../role/log/new_biLSTM-ep012-loss0.028-val_loss0.040-f10.954.h5"
  20. self.model_role = None
  21. self.graph = tf.get_default_graph()
  22. if not lazyLoad:
  23. self.getModel()
  24. def getModel(self):
  25. if self.model_role is None:
  26. self.model_role = models.load_model(self.model_role_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  27. return self.model_role
  28. def encode(self,tokens,begin_index,end_index,**kwargs):
  29. return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=10),shape=(2,10,128))
  30. def predict(self,x):
  31. x = np.transpose(np.array(x),(1,0,2,3))
  32. with self.graph.as_default():
  33. return self.getModel().predict([x[0],x[1]])
  34. class Model_role_classify_word():
  35. def __init__(self,lazyLoad=getLazyLoad(),config=None):
  36. if USE_PAI_EAS:
  37. lazyLoad = True
  38. #self.model_role_file = os.path.abspath("../role/log/ep071-loss0.107-val_loss0.122-f10.956.h5")
  39. # self.model_role_file = os.path.dirname(__file__)+"/../role/models/ep038-loss0.140-val_loss0.149-f10.947.h5"
  40. #self.model_role_file = os.path.abspath("../role/log/textcnn_ep017-loss0.088-val_loss0.125-f10.955.h5")
  41. self.model_role = None
  42. self.sess_role = tf.Session(graph=tf.Graph(),config=config)
  43. if not lazyLoad:
  44. self.getModel()
  45. def getModel(self):
  46. if self.model_role is None:
  47. with self.sess_role.as_default() as sess:
  48. with self.sess_role.graph.as_default():
  49. meta_graph_def = tf.saved_model.loader.load(sess=self.sess_role, tags=["serve"], export_dir=os.path.dirname(__file__)+"/role_savedmodel")
  50. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  51. signature_def = meta_graph_def.signature_def
  52. input0 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  53. input1 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  54. # input2 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)
  55. output = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  56. self.model_role = [[input0,input1],output] #,input2
  57. return self.model_role
  58. '''
  59. def load_weights(self):
  60. model = self.getModel()
  61. model.load_weights(self.model_role_file)
  62. '''
  63. def encode(self,tokens,begin_index,end_index,entity_text,**kwargs):
  64. _span = spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=20,center_include=False,word_flag=True,text=entity_text) #size=12 center_include=True
  65. # print(_span)
  66. _encode_span = encodeInput(_span, word_len=20, word_flag=True,userFool=False) # word_len=20
  67. # print(_encode_span)
  68. return _encode_span
  69. def fix_digit_eng(self, text):
  70. '''
  71. 处理数字及英文编号等
  72. :param text:
  73. :return:
  74. '''
  75. text = re.sub('第[一二三1-3]([条项章]|中学|医院|附属)|第三方(服务机构)?', 'xxx', text)
  76. text = re.sub('第01(中标|成交)?候选人', '第一中标候选人', text)
  77. text = re.sub('(标[段的包项]|品目)[一二三1-3]', '标段', text)
  78. text = re.sub('第?[一二三1-3](标段?|[分子标]?包)', 'd标段', text)
  79. text = re.sub('[a-zA-Z][a-zA-Z0-9=&_—-]{3,}', 'abc', text)
  80. text = re.sub('[【(\[][0-9]{2,}[\])】]|\d+([::.-]\d+)+', 'd', text)
  81. text = re.sub('[一二三四五六七八九十]{2,}|[四五六七八九十]+', 'd', text)
  82. text = re.sub('\d{2,}(\.\d+)?|\d\.\d+|[04-9]', 'd', text)
  83. text = re.sub('序号:\d+|第?[一二三四五六七八九十\d]+次|[一二三四五六七八九十\d]+、|([^\w]|^)序:?\d+', ' d', text) # ,序:1,单位名称:
  84. text = re.sub('(中标|成交|中选|入围)(工程|项目)', '工程', text) # 修复易错分为中标人
  85. text = re.sub('约定|(盖章|签名):?', ' ', text) # 修复 233233636 错分为中标人 国有产权网上竞价有关约定 辽阳市公共资源交易中心 ,标 修复 273505905 乡镇签名:盖章: 次村产权交易服务中心 预测为中标
  86. text = re.sub('中介机构', '投标机构', text) # 251058999 错分为中标人 序号:2,中介机构名称:
  87. text = re.sub('(采购|招标)人名称、地址和联系方式:|采购方,指', '采购人:', text) # 275065998 修复 224703143 采购的中标人;采购方,指 预测为中标
  88. if re.search('(最终)?排名:', text) and re.search('(最终)?排名:第?[123一二三]', text)==None:
  89. text = re.sub('(最终)?排名:', ' ', text)
  90. text = re.sub('交易单位', '发布单位', text)
  91. text = re.sub('[,:]各种数据:', ':', text) # 20240620优化 478331984 山东省交通运输厅站源提取不到 各种数据:中标单位,各种数据:济南金曰公路工程有限公司,
  92. text = re.sub('电子签章', '', text) # 20240924 修复 529923459 电子签名:投标人名称(电子签章:西君兰信息科技有限公司,2024年9月7日 预测为中标
  93. return text.replace('(', '(').replace(')', ')').replace('單', '单').replace('稱','承').replace('標', '标').replace('採購', '采购').replace('機構', '机构')
  94. def encode_word(self, sentence_text, begin_index, end_index, size=20, **kwargs):
  95. '''
  96. 上下文数字化,使用字偏移
  97. :param sentence_text: 句子文本
  98. :param begin_index: 实体字开始位置
  99. :param end_index: 实体字结束位置
  100. :param size: 字偏移量
  101. :param kwargs:
  102. :return:
  103. '''
  104. _span = get_context(sentence_text, begin_index, end_index,size=size, center_include=False) # size=12 center_include=True
  105. # print(_span)
  106. _span = [self.fix_digit_eng(text) for text in _span]
  107. _encode_span = encodeInput(_span, word_len=30, word_flag=True, userFool=False) # word_len=20
  108. # print(_encode_span)
  109. return _encode_span
  110. def predict(self,x):
  111. x = np.transpose(np.array(x),(1,0,2))
  112. model_role = self.getModel()
  113. assert len(x)==len(model_role[0])
  114. feed_dict = {}
  115. for _x,_t in zip(x,model_role[0]):
  116. feed_dict[_t] = _x
  117. list_result = limitRun(self.sess_role,[model_role[1]],feed_dict)[0]
  118. return list_result
  119. #return self.sess_role.run(model_role[1],feed_dict=feed_dict)
  120. class Model_money_classify():
  121. def __init__(self,lazyLoad=getLazyLoad(),config=None):
  122. if USE_PAI_EAS:
  123. lazyLoad = True
  124. self.model_money_file = os.path.dirname(__file__)+"/../money/models/model_money_word.h5"
  125. self.model_money = None
  126. self.sess_money = tf.Session(graph=tf.Graph(),config=config)
  127. if not lazyLoad:
  128. self.getModel()
  129. def getModel(self):
  130. if self.model_money is None:
  131. with self.sess_money.as_default() as sess:
  132. with sess.graph.as_default():
  133. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/money_savedmodel")
  134. # meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/money_savedmodel_bilstmonly")
  135. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  136. signature_def = meta_graph_def.signature_def
  137. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  138. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  139. input2 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)
  140. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  141. self.model_money = [[input0,input1,input2],output]
  142. return self.model_money
  143. '''
  144. if self.model_money is None:
  145. self.model_money = models.load_model(self.model_money_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  146. return self.model_money
  147. '''
  148. '''
  149. def load_weights(self):
  150. model = self.getModel()
  151. model.load_weights(self.model_money_file)
  152. '''
  153. def encode(self,tokens,begin_index,end_index,**kwargs):
  154. _span = spanWindow(tokens=tokens, begin_index=begin_index, end_index=end_index, size=10, center_include=True, word_flag=True)
  155. # print(_span)
  156. return encodeInput(_span, word_len=30, word_flag=True,userFool=False)
  157. return embedding_word(_span,shape=(3,100,60))
  158. def predict(self,x):
  159. # print("shape",np.shape(x))
  160. x = np.transpose(np.array(x),(1,0,2))
  161. model_money = self.getModel()
  162. assert len(x)==len(model_money[0])
  163. feed_dict = {}
  164. for _x,_t in zip(x,model_money[0]):
  165. feed_dict[_t] = _x
  166. list_result = limitRun(self.sess_money,[model_money[1]],feed_dict)[0]
  167. #return self.sess_money.run(model_money[1],feed_dict=feed_dict)
  168. return list_result
  169. '''
  170. with self.graph.as_default():
  171. return self.getModel().predict([x[0],x[1],x[2]])
  172. '''
  173. from itertools import groupby
  174. from BiddingKG.dl.relation_extraction.model import get_words_matrix
  175. class Model_relation_extraction():
  176. def __init__(self,lazyLoad=getLazyLoad()):
  177. if USE_PAI_EAS:
  178. lazyLoad = True
  179. self.subject_model_file = os.path.dirname(__file__)+"/../relation_extraction/models2/subject_model"
  180. self.object_model_file = os.path.dirname(__file__)+"/../relation_extraction/models2/object_model"
  181. self.model_subject = None
  182. self.model_object = None
  183. self.sess_subject = tf.Session(graph=tf.Graph())
  184. self.sess_object = tf.Session(graph=tf.Graph())
  185. if not lazyLoad:
  186. self.getModel1()
  187. self.getModel2()
  188. self.entity_type_dict = {
  189. 'org': '<company/org>',
  190. 'company': '<company/org>',
  191. 'location': '<location>',
  192. 'phone': '<phone>',
  193. 'person': '<contact_person>'
  194. }
  195. self.id2predicate = {
  196. 0: "rel_person", # 公司——联系人
  197. 1: "rel_phone", # 联系人——电话
  198. 2: "rel_address" # 公司——地址
  199. }
  200. self.words_size = 128
  201. # subject_model
  202. def getModel1(self):
  203. if self.model_subject is None:
  204. with self.sess_subject.as_default() as sess:
  205. with sess.graph.as_default():
  206. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=self.subject_model_file)
  207. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  208. signature_def = meta_graph_def.signature_def
  209. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  210. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  211. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  212. self.model_subject = [[input0,input1],output]
  213. return self.model_subject
  214. # object_model
  215. def getModel2(self):
  216. if self.model_object is None:
  217. with self.sess_object.as_default() as sess:
  218. with sess.graph.as_default():
  219. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=self.object_model_file)
  220. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  221. signature_def = meta_graph_def.signature_def
  222. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  223. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  224. input2 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)
  225. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  226. self.model_object = [[input0,input1,input2],output]
  227. return self.model_object
  228. def encode(self,entity_list,list_sentence):
  229. list_sentence = sorted(list_sentence, key=lambda x: x.sentence_index)
  230. entity_list = sorted(entity_list, key=lambda x: (x.sentence_index, x.begin_index))
  231. pre_data = []
  232. text_data = []
  233. last_sentence_index = -1
  234. for key, group in groupby(entity_list, key=lambda x: x.sentence_index):
  235. if key - last_sentence_index > 1:
  236. for i in range(last_sentence_index + 1, key):
  237. pre_data.extend(list_sentence[i].tokens)
  238. text_data.extend([0] * len(list_sentence[i].tokens))
  239. group = list(group)
  240. for i in range(len(group)):
  241. ent = group[i]
  242. _tokens = list_sentence[key].tokens
  243. if i == len(group) - 1:
  244. if i == 0:
  245. pre_data.extend(_tokens[:ent.begin_index])
  246. text_data.extend([0] * len(_tokens[:ent.begin_index]))
  247. pre_data.append(self.entity_type_dict[ent.entity_type])
  248. text_data.append(ent)
  249. pre_data.extend(_tokens[ent.end_index + 1:])
  250. text_data.extend([0] * len(_tokens[ent.end_index + 1:]))
  251. break
  252. else:
  253. pre_data.append(self.entity_type_dict[ent.entity_type])
  254. text_data.append(ent)
  255. pre_data.extend(_tokens[ent.end_index + 1:])
  256. text_data.extend([0] * len(_tokens[ent.end_index + 1:]))
  257. break
  258. if i == 0:
  259. pre_data.extend(_tokens[:ent.begin_index])
  260. text_data.extend([0] * len(_tokens[:ent.begin_index]))
  261. pre_data.append(self.entity_type_dict[ent.entity_type])
  262. text_data.append(ent)
  263. pre_data.extend(_tokens[ent.end_index + 1:group[i + 1].begin_index])
  264. text_data.extend([0] * len(_tokens[ent.end_index + 1:group[i + 1].begin_index]))
  265. else:
  266. pre_data.append(self.entity_type_dict[ent.entity_type])
  267. text_data.append(ent)
  268. pre_data.extend(_tokens[ent.end_index + 1:group[i + 1].begin_index])
  269. text_data.extend([0] * len(_tokens[ent.end_index + 1:group[i + 1].begin_index]))
  270. last_sentence_index = key
  271. return text_data, pre_data
  272. def check_data(self, words):
  273. # 检查数据是否包含可预测的subject和object
  274. # 没有需要预测的链接属性,直接return
  275. company_relation = 0
  276. person_relation = 0
  277. if '<company/org>' in words:
  278. company_relation += 1
  279. if '<contact_person>' in words:
  280. person_relation += 1
  281. if company_relation:
  282. company_relation += 1
  283. # 暂时不考虑地址location实体
  284. # if '<location>' in words and company_relation:
  285. # company_relation += 1
  286. if '<phone>' in words and company_relation:
  287. person_relation += 1
  288. if company_relation < 2 and person_relation < 2:
  289. return False
  290. return True
  291. def predict_by_api(self,text_in,words,sentence_vetor):
  292. status_code = 0
  293. # save([words,sentence_vetor.tolist()],"C:/Users/Administrator/Desktop/test_data.pk")
  294. try:
  295. requests_result = requests.post(API_URL + "/predict_relation", json={"sentence_vetor": sentence_vetor.tolist(), "words": words},
  296. verify=True)
  297. status_code = requests_result.status_code
  298. triple_index_list = json.loads(requests_result.text)['triple_list']
  299. # print("triple_list:",json.loads(requests_result.text)['triple_list'])
  300. print("cost_time:",json.loads(requests_result.text)['cost_time'])
  301. triple_list = [(text_in[triple[0]], triple[1], text_in[triple[2]]) for triple in triple_index_list]
  302. return triple_list,status_code
  303. except Exception as e:
  304. print(e)
  305. return [],status_code
  306. def predict(self,text_in, words, rate=0.5):
  307. _t2 = np.zeros((len(words), self.words_size))
  308. for i in range(len(words)):
  309. _t2[i] = np.array(get_words_matrix(words[i]))
  310. # a = time.time()
  311. # triple_list, status_code = self.predict_by_api(text_in, words,_t2)
  312. # print('time',time.time()-a)
  313. # print("status_code",status_code)
  314. # if status_code==200:
  315. # return triple_list
  316. # else:
  317. # 使用模型预测
  318. triple_list = []
  319. # print("tokens:",words)
  320. # _t2 = [self.words2id.get(c, 1) for c in words]
  321. _t2 = np.array([_t2])
  322. _t3 = [1 for _ in words]
  323. _t3 = np.array([_t3])
  324. # _k1 = self.model_subject.predict([_t2, _t3])
  325. _k1 = limitRun(self.sess_subject,[self.model_subject[1]],feed_dict={self.model_subject[0][0]:_t2,
  326. self.model_subject[0][1]:_t3})[0]
  327. _k1 = _k1[0, :, 0]
  328. _k1 = np.where(_k1 > rate)[0]
  329. # print('k1',_k1)
  330. _subjects = []
  331. for i in _k1:
  332. _subject = text_in[i]
  333. _subjects.append((_subject, i, i))
  334. if _subjects:
  335. _t2 = np.repeat(_t2, len(_subjects), 0)
  336. _t3 = np.repeat(_t3, len(_subjects), 0)
  337. _k1, _ = np.array([_s[1:] for _s in _subjects]).T.reshape((2, -1, 1))
  338. # _o1 = self.model_object.predict([_t2, _t3, _k1])
  339. _o1 = limitRun(self.sess_object, [self.model_object[1]], feed_dict={self.model_object[0][0]: _t2,
  340. self.model_object[0][1]: _t3,
  341. self.model_object[0][2]: _k1})[0]
  342. for i, _subject in enumerate(_subjects):
  343. _oo1 = np.where(_o1[i] > 0.5)
  344. # print('_oo1', _oo1)
  345. for _ooo1, _c1 in zip(*_oo1):
  346. _object = text_in[_ooo1]
  347. _predicate = self.id2predicate[_c1]
  348. triple_list.append((_subject[0], _predicate, _object))
  349. # print([(t[0].entity_text,t[1],t[2].entity_text) for t in triple_list])
  350. return triple_list
  351. else:
  352. return []
  353. class Model_person_classify():
  354. def __init__(self,lazyLoad=getLazyLoad(),config=None):
  355. if USE_PAI_EAS:
  356. lazyLoad = True
  357. self.model_person_file = os.path.dirname(__file__)+"/../person/models/model_person.model.hdf5"
  358. self.model_person = None
  359. self.sess_person = tf.Session(graph=tf.Graph(),config=config)
  360. if not lazyLoad:
  361. self.getModel()
  362. def getModel(self):
  363. if self.model_person is None:
  364. with self.sess_person.as_default() as sess:
  365. with sess.graph.as_default():
  366. # meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/person_savedmodel_new")
  367. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/person_savedmodel_new_znj")
  368. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  369. signature_def = meta_graph_def.signature_def
  370. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  371. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  372. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  373. self.model_person = [[input0,input1],output]
  374. return self.model_person
  375. '''
  376. if self.model_person is None:
  377. self.model_person = models.load_model(self.model_person_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  378. return self.model_person
  379. '''
  380. '''
  381. def load_weights(self):
  382. model = self.getModel()
  383. model.load_weights(self.model_person_file)
  384. '''
  385. def encode(self,tokens,begin_index,end_index,**kwargs):
  386. # return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=10),shape=(2,10,128))
  387. return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=20),shape=(2,20,128))
  388. def predict(self,x):
  389. x = np.transpose(np.array(x),(1,0,2,3))
  390. model_person = self.getModel()
  391. assert len(x)==len(model_person[0])
  392. feed_dict = {}
  393. for _x,_t in zip(x,model_person[0]):
  394. feed_dict[_t] = _x
  395. list_result = limitRun(self.sess_person,[model_person[1]],feed_dict)[0]
  396. return list_result
  397. #return self.sess_person.run(model_person[1],feed_dict=feed_dict)
  398. '''
  399. with self.graph.as_default():
  400. return self.getModel().predict([x[0],x[1]])
  401. '''
  402. class Model_form_line():
  403. def __init__(self,lazyLoad=getLazyLoad()):
  404. self.model_file = os.path.dirname(__file__)+"/../form/model/model_form.model - 副本.hdf5"
  405. self.model_form = None
  406. self.graph = tf.get_default_graph()
  407. if not lazyLoad:
  408. self.getModel()
  409. def getModel(self):
  410. if self.model_form is None:
  411. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  412. return self.model_form
  413. def encode(self,data,shape=(100,60),expand=False,**kwargs):
  414. embedding = np.zeros(shape)
  415. word_model = getModel_word()
  416. for i in range(len(data)):
  417. if i>=shape[0]:
  418. break
  419. if data[i] in word_model.vocab:
  420. embedding[i] = word_model[data[i]]
  421. if expand:
  422. embedding = np.expand_dims(embedding,0)
  423. return embedding
  424. def predict(self,x):
  425. with self.graph.as_default():
  426. return self.getModel().predict(x)
  427. class Model_form_item():
  428. def __init__(self,lazyLoad=getLazyLoad(),config=None):
  429. self.model_file = os.path.dirname(__file__)+"/../form/log/ep039-loss0.038-val_loss0.064-f10.9783.h5"
  430. self.model_form = None
  431. self.sess_form = tf.Session(graph=tf.Graph(),config=config)
  432. if not lazyLoad:
  433. self.getModel()
  434. def getModel(self):
  435. if self.model_form is None:
  436. with self.sess_form.as_default() as sess:
  437. with sess.graph.as_default():
  438. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir="%s/form_savedmodel"%(os.path.dirname(__file__)))
  439. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  440. signature_def = meta_graph_def.signature_def
  441. inputs = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
  442. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  443. self.model_form = [[inputs],output]
  444. return self.model_form
  445. '''
  446. if self.model_form is None:
  447. with self.graph.as_defalt():
  448. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  449. return self.model_form
  450. '''
  451. def encode(self,data,**kwargs):
  452. return encodeInput([data], word_len=50, word_flag=True,userFool=False)[0]
  453. return encodeInput_form(data)
  454. def predict(self,x):
  455. if USE_API:
  456. requests_result = requests.post(API_URL+"/predict_form_item",json={"inputs":x.tolist()}, verify=True)
  457. list_result = json.loads(requests_result.text)['result']
  458. else:
  459. model_form = self.getModel()
  460. list_result = limitRun(self.sess_form,[model_form[1]],feed_dict={model_form[0][0]:x})[0]
  461. return list_result
  462. # return self.sess_form.run(model_form[1],feed_dict={model_form[0][0]:x})
  463. '''
  464. with self.graph.as_default():
  465. return self.getModel().predict(x)
  466. '''
  467. class Model_form_context():
  468. def __init__(self,lazyLoad=getLazyLoad(),config=None):
  469. self.model_form = None
  470. self.sess_form = tf.Session(graph=tf.Graph(),config=config)
  471. if not lazyLoad:
  472. self.getModel()
  473. def getModel(self):
  474. if self.model_form is None:
  475. with self.sess_form.as_default() as sess:
  476. with sess.graph.as_default():
  477. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir="%s/form_context_savedmodel"%(os.path.dirname(__file__)))
  478. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  479. signature_def = meta_graph_def.signature_def
  480. inputs = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
  481. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  482. self.model_form = [[inputs],output]
  483. return self.model_form
  484. '''
  485. if self.model_form is None:
  486. with self.graph.as_defalt():
  487. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  488. return self.model_form
  489. '''
  490. def encode_table(self,inner_table,size=30):
  491. def encode_item(_table,i,j):
  492. _x = [_table[j-1][i-1],_table[j-1][i],_table[j-1][i+1],
  493. _table[j][i-1],_table[j][i],_table[j][i+1],
  494. _table[j+1][i-1],_table[j+1][i],_table[j+1][i+1]]
  495. e_x = [encodeInput_form(_temp[0],MAX_LEN=30) for _temp in _x]
  496. _label = _table[j][i][1]
  497. # print(_x)
  498. # print(_x[4],_label)
  499. return e_x,_label,_x
  500. def copytable(inner_table):
  501. table = []
  502. for line in inner_table:
  503. list_line = []
  504. for item in line:
  505. list_line.append([item[0][:size],item[1]])
  506. table.append(list_line)
  507. return table
  508. table = copytable(inner_table)
  509. padding = ["#"*30,0]
  510. width = len(table[0])
  511. height = len(table)
  512. table.insert(0,[padding for i in range(width)])
  513. table.append([padding for i in range(width)])
  514. for item in table:
  515. item.insert(0,padding.copy())
  516. item.append(padding.copy())
  517. data_x = []
  518. data_y = []
  519. data_text = []
  520. data_position = []
  521. for _i in range(1,width+1):
  522. for _j in range(1,height+1):
  523. _x,_y,_text = encode_item(table,_i,_j)
  524. data_x.append(_x)
  525. _label = [0,0]
  526. _label[_y] = 1
  527. data_y.append(_label)
  528. data_text.append(_text)
  529. data_position.append([_i-1,_j-1])
  530. # input = table[_j][_i][0]
  531. # item_y = [0,0]
  532. # item_y[table[_j][_i][1]] = 1
  533. # data_x.append(encodeInput([input], word_len=50, word_flag=True,userFool=False)[0])
  534. # data_y.append(item_y)
  535. return data_x,data_y,data_text,data_position
  536. def encode(self,inner_table,**kwargs):
  537. data_x,_,_,data_position = self.encode_table(inner_table)
  538. return data_x,data_position
  539. def predict(self,x):
  540. model_form = self.getModel()
  541. list_result = limitRun(self.sess_form,[model_form[1]],feed_dict={model_form[0][0]:x})[0]
  542. return list_result
  543. # class Model_form_item():
  544. # def __init__(self,lazyLoad=False):
  545. # self.model_file = os.path.dirname(__file__)+"/ep039-loss0.038-val_loss0.064-f10.9783.h5"
  546. # self.model_form = None
  547. #
  548. # if not lazyLoad:
  549. # self.getModel()
  550. # self.graph = tf.get_default_graph()
  551. #
  552. # def getModel(self):
  553. # if self.model_form is None:
  554. # self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  555. # return self.model_form
  556. #
  557. # def encode(self,data,**kwargs):
  558. #
  559. # return encodeInput_form(data)
  560. #
  561. # def predict(self,x):
  562. # with self.graph.as_default():
  563. # return self.getModel().predict(x)