modelFactory.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. '''
  2. Created on 2019年5月16日
  3. @author: User
  4. '''
  5. import os
  6. import sys
  7. sys.path.append(os.path.abspath("../.."))
  8. from keras import models
  9. from keras import layers
  10. from keras_contrib.layers import CRF
  11. from keras.preprocessing.sequence import pad_sequences
  12. from keras import optimizers,losses,metrics
  13. from BiddingKG.dl.common.Utils import *
  14. import tensorflow as tf
  15. class Model_role_classify():
  16. def __init__(self,lazyLoad=getLazyLoad()):
  17. #self.model_role_file = os.path.abspath("../role/models/model_role.model.hdf5")
  18. self.model_role_file = os.path.dirname(__file__)+"/../role/log/new_biLSTM-ep012-loss0.028-val_loss0.040-f10.954.h5"
  19. self.model_role = None
  20. self.graph = tf.get_default_graph()
  21. if not lazyLoad:
  22. self.getModel()
  23. def getModel(self):
  24. if self.model_role is None:
  25. self.model_role = models.load_model(self.model_role_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  26. return self.model_role
  27. def encode(self,tokens,begin_index,end_index,**kwargs):
  28. return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=10),shape=(2,10,128))
  29. def predict(self,x):
  30. x = np.transpose(np.array(x),(1,0,2,3))
  31. with self.graph.as_default():
  32. return self.getModel().predict([x[0],x[1]])
  33. class Model_role_classify_word():
  34. def __init__(self,lazyLoad=getLazyLoad()):
  35. if USE_PAI_EAS:
  36. lazyLoad = True
  37. #self.model_role_file = os.path.abspath("../role/log/ep071-loss0.107-val_loss0.122-f10.956.h5")
  38. self.model_role_file = os.path.dirname(__file__)+"/../role/models/ep038-loss0.140-val_loss0.149-f10.947.h5"
  39. #self.model_role_file = os.path.abspath("../role/log/textcnn_ep017-loss0.088-val_loss0.125-f10.955.h5")
  40. self.model_role = None
  41. self.sess_role = tf.Session(graph=tf.Graph())
  42. if not lazyLoad:
  43. self.getModel()
  44. def getModel(self):
  45. if self.model_role is None:
  46. with self.sess_role.as_default() as sess:
  47. with self.sess_role.graph.as_default():
  48. meta_graph_def = tf.saved_model.loader.load(sess=self.sess_role, tags=["serve"], export_dir=os.path.dirname(__file__)+"/role_savedmodel")
  49. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  50. signature_def = meta_graph_def.signature_def
  51. input0 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  52. input1 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  53. input2 = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)
  54. output = self.sess_role.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  55. self.model_role = [[input0,input1,input2],output]
  56. return self.model_role
  57. '''
  58. def load_weights(self):
  59. model = self.getModel()
  60. model.load_weights(self.model_role_file)
  61. '''
  62. def encode(self,tokens,begin_index,end_index,entity_text,**kwargs):
  63. _span = spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=12,center_include=True,word_flag=True,text=entity_text)
  64. # print(_span)
  65. _encode_span = encodeInput(_span, word_len=20, word_flag=True,userFool=False)
  66. # print(_encode_span)
  67. return _encode_span
  68. def predict(self,x):
  69. x = np.transpose(np.array(x),(1,0,2))
  70. model_role = self.getModel()
  71. assert len(x)==len(model_role[0])
  72. feed_dict = {}
  73. for _x,_t in zip(x,model_role[0]):
  74. feed_dict[_t] = _x
  75. list_result = limitRun(self.sess_role,[model_role[1]],feed_dict)[0]
  76. return list_result
  77. #return self.sess_role.run(model_role[1],feed_dict=feed_dict)
  78. class Model_money_classify():
  79. def __init__(self,lazyLoad=getLazyLoad()):
  80. if USE_PAI_EAS:
  81. lazyLoad = True
  82. self.model_money_file = os.path.dirname(__file__)+"/../money/models/model_money_word.h5"
  83. self.model_money = None
  84. self.sess_money = tf.Session(graph=tf.Graph())
  85. if not lazyLoad:
  86. self.getModel()
  87. def getModel(self):
  88. if self.model_money is None:
  89. with self.sess_money.as_default() as sess:
  90. with sess.graph.as_default():
  91. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/money_savedmodel")
  92. # meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/money_savedmodel_bilstmonly")
  93. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  94. signature_def = meta_graph_def.signature_def
  95. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  96. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  97. input2 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input2"].name)
  98. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  99. self.model_money = [[input0,input1,input2],output]
  100. return self.model_money
  101. '''
  102. if self.model_money is None:
  103. self.model_money = models.load_model(self.model_money_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  104. return self.model_money
  105. '''
  106. '''
  107. def load_weights(self):
  108. model = self.getModel()
  109. model.load_weights(self.model_money_file)
  110. '''
  111. def encode(self,tokens,begin_index,end_index,**kwargs):
  112. _span = spanWindow(tokens=tokens, begin_index=begin_index, end_index=end_index, size=10, center_include=True, word_flag=True)
  113. # print(_span)
  114. return encodeInput(_span, word_len=50, word_flag=True,userFool=False)
  115. return embedding_word(_span,shape=(3,100,60))
  116. def predict(self,x):
  117. # print("shape",np.shape(x))
  118. x = np.transpose(np.array(x),(1,0,2))
  119. model_money = self.getModel()
  120. assert len(x)==len(model_money[0])
  121. feed_dict = {}
  122. for _x,_t in zip(x,model_money[0]):
  123. feed_dict[_t] = _x
  124. list_result = limitRun(self.sess_money,[model_money[1]],feed_dict)[0]
  125. #return self.sess_money.run(model_money[1],feed_dict=feed_dict)
  126. return list_result
  127. '''
  128. with self.graph.as_default():
  129. return self.getModel().predict([x[0],x[1],x[2]])
  130. '''
  131. class Model_person_classify():
  132. def __init__(self,lazyLoad=getLazyLoad()):
  133. if USE_PAI_EAS:
  134. lazyLoad = True
  135. self.model_person_file = os.path.dirname(__file__)+"/../person/models/model_person.model.hdf5"
  136. self.model_person = None
  137. self.sess_person = tf.Session(graph=tf.Graph())
  138. if not lazyLoad:
  139. self.getModel()
  140. def getModel(self):
  141. if self.model_person is None:
  142. with self.sess_person.as_default() as sess:
  143. with sess.graph.as_default():
  144. # meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/person_savedmodel_new")
  145. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir=os.path.dirname(__file__)+"/person_savedmodel_new_znj")
  146. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  147. signature_def = meta_graph_def.signature_def
  148. input0 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input0"].name)
  149. input1 = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["input1"].name)
  150. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  151. self.model_person = [[input0,input1],output]
  152. return self.model_person
  153. '''
  154. if self.model_person is None:
  155. self.model_person = models.load_model(self.model_person_file,custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score})
  156. return self.model_person
  157. '''
  158. '''
  159. def load_weights(self):
  160. model = self.getModel()
  161. model.load_weights(self.model_person_file)
  162. '''
  163. def encode(self,tokens,begin_index,end_index,**kwargs):
  164. # return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=10),shape=(2,10,128))
  165. return embedding(spanWindow(tokens=tokens,begin_index=begin_index,end_index=end_index,size=20),shape=(2,20,128))
  166. def predict(self,x):
  167. x = np.transpose(np.array(x),(1,0,2,3))
  168. model_person = self.getModel()
  169. assert len(x)==len(model_person[0])
  170. feed_dict = {}
  171. for _x,_t in zip(x,model_person[0]):
  172. feed_dict[_t] = _x
  173. list_result = limitRun(self.sess_person,[model_person[1]],feed_dict)[0]
  174. return list_result
  175. #return self.sess_person.run(model_person[1],feed_dict=feed_dict)
  176. '''
  177. with self.graph.as_default():
  178. return self.getModel().predict([x[0],x[1]])
  179. '''
  180. class Model_form_line():
  181. def __init__(self,lazyLoad=getLazyLoad()):
  182. self.model_file = os.path.dirname(__file__)+"/../form/model/model_form.model - 副本.hdf5"
  183. self.model_form = None
  184. self.graph = tf.get_default_graph()
  185. if not lazyLoad:
  186. self.getModel()
  187. def getModel(self):
  188. if self.model_form is None:
  189. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  190. return self.model_form
  191. def encode(self,data,shape=(100,60),expand=False,**kwargs):
  192. embedding = np.zeros(shape)
  193. word_model = getModel_word()
  194. for i in range(len(data)):
  195. if i>=shape[0]:
  196. break
  197. if data[i] in word_model.vocab:
  198. embedding[i] = word_model[data[i]]
  199. if expand:
  200. embedding = np.expand_dims(embedding,0)
  201. return embedding
  202. def predict(self,x):
  203. with self.graph.as_default():
  204. return self.getModel().predict(x)
  205. class Model_form_item():
  206. def __init__(self,lazyLoad=getLazyLoad()):
  207. self.model_file = os.path.dirname(__file__)+"/../form/log/ep039-loss0.038-val_loss0.064-f10.9783.h5"
  208. self.model_form = None
  209. self.sess_form = tf.Session(graph=tf.Graph())
  210. if not lazyLoad:
  211. self.getModel()
  212. def getModel(self):
  213. if self.model_form is None:
  214. with self.sess_form.as_default() as sess:
  215. with sess.graph.as_default():
  216. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir="%s/form_savedmodel"%(os.path.dirname(__file__)))
  217. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  218. signature_def = meta_graph_def.signature_def
  219. inputs = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
  220. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  221. self.model_form = [[inputs],output]
  222. return self.model_form
  223. '''
  224. if self.model_form is None:
  225. with self.graph.as_defalt():
  226. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  227. return self.model_form
  228. '''
  229. def encode(self,data,**kwargs):
  230. return encodeInput([data], word_len=50, word_flag=True,userFool=False)[0]
  231. return encodeInput_form(data)
  232. def predict(self,x):
  233. model_form = self.getModel()
  234. list_result = limitRun(self.sess_form,[model_form[1]],feed_dict={model_form[0][0]:x})[0]
  235. return list_result
  236. #return self.sess_form.run(model_form[1],feed_dict={model_form[0][0]:x})
  237. '''
  238. with self.graph.as_default():
  239. return self.getModel().predict(x)
  240. '''
  241. class Model_form_context():
  242. def __init__(self,lazyLoad=getLazyLoad()):
  243. self.model_form = None
  244. self.sess_form = tf.Session(graph=tf.Graph())
  245. if not lazyLoad:
  246. self.getModel()
  247. def getModel(self):
  248. if self.model_form is None:
  249. with self.sess_form.as_default() as sess:
  250. with sess.graph.as_default():
  251. meta_graph_def = tf.saved_model.loader.load(sess,tags=["serve"],export_dir="%s/form_context_savedmodel"%(os.path.dirname(__file__)))
  252. signature_key = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
  253. signature_def = meta_graph_def.signature_def
  254. inputs = sess.graph.get_tensor_by_name(signature_def[signature_key].inputs["inputs"].name)
  255. output = sess.graph.get_tensor_by_name(signature_def[signature_key].outputs["outputs"].name)
  256. self.model_form = [[inputs],output]
  257. return self.model_form
  258. '''
  259. if self.model_form is None:
  260. with self.graph.as_defalt():
  261. self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  262. return self.model_form
  263. '''
  264. def encode_table(self,inner_table,size=30):
  265. def encode_item(_table,i,j):
  266. _x = [_table[j-1][i-1],_table[j-1][i],_table[j-1][i+1],
  267. _table[j][i-1],_table[j][i],_table[j][i+1],
  268. _table[j+1][i-1],_table[j+1][i],_table[j+1][i+1]]
  269. e_x = [encodeInput_form(_temp[0],MAX_LEN=30) for _temp in _x]
  270. _label = _table[j][i][1]
  271. # print(_x)
  272. # print(_x[4],_label)
  273. return e_x,_label,_x
  274. def copytable(inner_table):
  275. table = []
  276. for line in inner_table:
  277. list_line = []
  278. for item in line:
  279. list_line.append([item[0][:size],item[1]])
  280. table.append(list_line)
  281. return table
  282. table = copytable(inner_table)
  283. padding = ["#"*30,0]
  284. width = len(table[0])
  285. height = len(table)
  286. table.insert(0,[padding for i in range(width)])
  287. table.append([padding for i in range(width)])
  288. for item in table:
  289. item.insert(0,padding.copy())
  290. item.append(padding.copy())
  291. data_x = []
  292. data_y = []
  293. data_text = []
  294. data_position = []
  295. for _i in range(1,width+1):
  296. for _j in range(1,height+1):
  297. _x,_y,_text = encode_item(table,_i,_j)
  298. data_x.append(_x)
  299. _label = [0,0]
  300. _label[_y] = 1
  301. data_y.append(_label)
  302. data_text.append(_text)
  303. data_position.append([_i-1,_j-1])
  304. # input = table[_j][_i][0]
  305. # item_y = [0,0]
  306. # item_y[table[_j][_i][1]] = 1
  307. # data_x.append(encodeInput([input], word_len=50, word_flag=True,userFool=False)[0])
  308. # data_y.append(item_y)
  309. return data_x,data_y,data_text,data_position
  310. def encode(self,inner_table,**kwargs):
  311. data_x,_,_,data_position = self.encode_table(inner_table)
  312. return data_x,data_position
  313. def predict(self,x):
  314. model_form = self.getModel()
  315. list_result = limitRun(self.sess_form,[model_form[1]],feed_dict={model_form[0][0]:x})[0]
  316. return list_result
  317. # class Model_form_item():
  318. # def __init__(self,lazyLoad=False):
  319. # self.model_file = os.path.dirname(__file__)+"/ep039-loss0.038-val_loss0.064-f10.9783.h5"
  320. # self.model_form = None
  321. #
  322. # if not lazyLoad:
  323. # self.getModel()
  324. # self.graph = tf.get_default_graph()
  325. #
  326. # def getModel(self):
  327. # if self.model_form is None:
  328. # self.model_form = models.load_model(self.model_file,custom_objects={"precision":precision,"recall":recall,"f1_score":f1_score})
  329. # return self.model_form
  330. #
  331. # def encode(self,data,**kwargs):
  332. #
  333. # return encodeInput_form(data)
  334. #
  335. # def predict(self,x):
  336. # with self.graph.as_default():
  337. # return self.getModel().predict(x)