Utils.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824
  1. '''
  2. Created on 2018年12月20日
  3. @author: User
  4. '''
  5. import numpy as np
  6. import re
  7. import gensim
  8. from keras import backend as K
  9. import os
  10. import time
  11. from threading import RLock
  12. # from pai_tf_predict_proto import tf_predict_pb2
  13. import requests
  14. model_w2v = None
  15. lock_model_w2v = RLock()
  16. USE_PAI_EAS = False
  17. Lazy_load = False
  18. def getw2vfilepath():
  19. w2vfile = os.path.dirname(__file__)+"/../wiki_128_word_embedding_new.vector"
  20. if os.path.exists(w2vfile):
  21. return w2vfile
  22. return "wiki_128_word_embedding_new.vector"
  23. def getLazyLoad():
  24. global Lazy_load
  25. return Lazy_load
  26. model_word_file = os.path.dirname(__file__)+"/../singlew2v_model.vector"
  27. model_word = None
  28. lock_model_word = RLock()
  29. from decimal import Decimal
  30. import logging
  31. logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  32. logger = logging.getLogger(__name__)
  33. import pickle
  34. import os
  35. import json
  36. #自定义jsonEncoder
  37. class MyEncoder(json.JSONEncoder):
  38. def __init__(self):
  39. import numpy as np
  40. global np
  41. def default(self, obj):
  42. if isinstance(obj, np.ndarray):
  43. return obj.tolist()
  44. elif isinstance(obj, bytes):
  45. return str(obj, encoding='utf-8')
  46. elif isinstance(obj, (np.float_, np.float16, np.float32,
  47. np.float64)):
  48. return float(obj)
  49. elif isinstance(obj,(np.int64,np.int32)):
  50. return int(obj)
  51. return json.JSONEncoder.default(self, obj)
  52. vocab_word = None
  53. vocab_words = None
  54. file_vocab_word = "vocab_word.pk"
  55. file_vocab_words = "vocab_words.pk"
  56. selffool_authorization = "NjlhMWFjMjVmNWYyNzI0MjY1OGQ1M2Y0ZmY4ZGY0Mzg3Yjc2MTVjYg=="
  57. selffool_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/selffool_gpu"
  58. selffool_seg_authorization = "OWUwM2Q0ZmE3YjYxNzU4YzFiMjliNGVkMTA3MzJkNjQ2MzJiYzBhZg=="
  59. selffool_seg_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/selffool_seg_gpu"
  60. codename_authorization = "Y2M5MDUxMzU1MTU4OGM3ZDk2ZmEzYjkxYmYyYzJiZmUyYTgwYTg5NA=="
  61. codename_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/codename_gpu"
  62. form_item_authorization = "ODdkZWY1YWY0NmNhNjU2OTI2NWY4YmUyM2ZlMDg1NTZjOWRkYTVjMw=="
  63. form_item_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/form"
  64. person_authorization = "N2I2MDU2N2Q2MGQ0ZWZlZGM3NDkyNTA1Nzc4YmM5OTlhY2MxZGU1Mw=="
  65. person_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/person"
  66. role_authorization = "OWM1ZDg5ZDEwYTEwYWI4OGNjYmRlMmQ1NzYwNWNlZGZkZmRmMjE4OQ=="
  67. role_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/role"
  68. money_authorization = "MDQyNjc2ZDczYjBhYmM4Yzc4ZGI4YjRmMjc3NGI5NTdlNzJiY2IwZA=="
  69. money_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/money"
  70. codeclasses_authorization = "MmUyNWIxZjQ2NjAzMWJlMGIzYzkxMjMzNWY5OWI3NzJlMWQ1ZjY4Yw=="
  71. codeclasses_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/codeclasses"
  72. def viterbi_decode(score, transition_params):
  73. """Decode the highest scoring sequence of tags outside of TensorFlow.
  74. This should only be used at test time.
  75. Args:
  76. score: A [seq_len, num_tags] matrix of unary potentials.
  77. transition_params: A [num_tags, num_tags] matrix of binary potentials.
  78. Returns:
  79. viterbi: A [seq_len] list of integers containing the highest scoring tag
  80. indices.
  81. viterbi_score: A float containing the score for the Viterbi sequence.
  82. """
  83. trellis = np.zeros_like(score)
  84. backpointers = np.zeros_like(score, dtype=np.int32)
  85. trellis[0] = score[0]
  86. for t in range(1, score.shape[0]):
  87. v = np.expand_dims(trellis[t - 1], 1) + transition_params
  88. trellis[t] = score[t] + np.max(v, 0)
  89. backpointers[t] = np.argmax(v, 0)
  90. viterbi = [np.argmax(trellis[-1])]
  91. for bp in reversed(backpointers[1:]):
  92. viterbi.append(bp[viterbi[-1]])
  93. viterbi.reverse()
  94. viterbi_score = np.max(trellis[-1])
  95. return viterbi, viterbi_score
  96. def limitRun(sess,list_output,feed_dict,MAX_BATCH=1024):
  97. len_sample = 0
  98. if len(feed_dict.keys())>0:
  99. len_sample = len(feed_dict[list(feed_dict.keys())[0]])
  100. if len_sample>MAX_BATCH:
  101. list_result = [[] for _ in range(len(list_output))]
  102. _begin = 0
  103. while(_begin<len_sample):
  104. new_dict = dict()
  105. for _key in feed_dict.keys():
  106. new_dict[_key] = feed_dict[_key][_begin:_begin+MAX_BATCH]
  107. _output = sess.run(list_output,feed_dict=new_dict)
  108. for _index in range(len(list_output)):
  109. list_result[_index].extend(_output[_index])
  110. _begin += MAX_BATCH
  111. else:
  112. list_result = sess.run(list_output,feed_dict=feed_dict)
  113. return list_result
  114. def get_values(response,output_name):
  115. """
  116. Get the value of a specified output tensor
  117. :param output_name: name of the output tensor
  118. :return: the content of the output tensor
  119. """
  120. output = response.outputs[output_name]
  121. if output.dtype == tf_predict_pb2.DT_FLOAT:
  122. _value = output.float_val
  123. elif output.dtype == tf_predict_pb2.DT_INT8 or output.dtype == tf_predict_pb2.DT_INT16 or \
  124. output.dtype == tf_predict_pb2.DT_INT32:
  125. _value = output.int_val
  126. elif output.dtype == tf_predict_pb2.DT_INT64:
  127. _value = output.int64_val
  128. elif output.dtype == tf_predict_pb2.DT_DOUBLE:
  129. _value = output.double_val
  130. elif output.dtype == tf_predict_pb2.DT_STRING:
  131. _value = output.string_val
  132. elif output.dtype == tf_predict_pb2.DT_BOOL:
  133. _value = output.bool_val
  134. return np.array(_value).reshape(response.outputs[output_name].array_shape.dim)
  135. def vpc_requests(url,authorization,request_data,list_outputs):
  136. headers = {"Authorization": authorization}
  137. dict_outputs = dict()
  138. response = tf_predict_pb2.PredictResponse()
  139. resp = requests.post(url, data=request_data, headers=headers)
  140. if resp.status_code != 200:
  141. print(resp.status_code,resp.content)
  142. log("调用pai-eas接口出错,authorization:"+str(authorization))
  143. return None
  144. else:
  145. response = tf_predict_pb2.PredictResponse()
  146. response.ParseFromString(resp.content)
  147. for _output in list_outputs:
  148. dict_outputs[_output] = get_values(response, _output)
  149. return dict_outputs
  150. def encodeInput(data,word_len,word_flag=True,userFool=False):
  151. result = []
  152. out_index = 0
  153. for item in data:
  154. if out_index in [0]:
  155. list_word = item[-word_len:]
  156. else:
  157. list_word = item[:word_len]
  158. temp = []
  159. if word_flag:
  160. for word in list_word:
  161. if userFool:
  162. temp.append(getIndexOfWord_fool(word))
  163. else:
  164. temp.append(getIndexOfWord(word))
  165. list_append = []
  166. temp_len = len(temp)
  167. while(temp_len<word_len):
  168. if userFool:
  169. list_append.append(0)
  170. else:
  171. list_append.append(getIndexOfWord("<pad>"))
  172. temp_len += 1
  173. if out_index in [0]:
  174. temp = list_append+temp
  175. else:
  176. temp = temp+list_append
  177. else:
  178. for words in list_word:
  179. temp.append(getIndexOfWords(words))
  180. list_append = []
  181. temp_len = len(temp)
  182. while(temp_len<word_len):
  183. list_append.append(getIndexOfWords("<pad>"))
  184. temp_len += 1
  185. if out_index in [0,1]:
  186. temp = list_append+temp
  187. else:
  188. temp = temp+list_append
  189. result.append(temp)
  190. out_index += 1
  191. return result
  192. def encodeInput_form(input,MAX_LEN=30):
  193. x = np.zeros([MAX_LEN])
  194. for i in range(len(input)):
  195. if i>=MAX_LEN:
  196. break
  197. x[i] = getIndexOfWord(input[i])
  198. return x
  199. def getVocabAndMatrix(model,Embedding_size = 60):
  200. '''
  201. @summary:获取子向量的词典和子向量矩阵
  202. '''
  203. vocab = ["<pad>"]+model.index2word
  204. embedding_matrix = np.zeros((len(vocab),Embedding_size))
  205. for i in range(1,len(vocab)):
  206. embedding_matrix[i] = model[vocab[i]]
  207. return vocab,embedding_matrix
  208. def getIndexOfWord(word):
  209. global vocab_word,file_vocab_word
  210. if vocab_word is None:
  211. if os.path.exists(file_vocab_word):
  212. vocab = load(file_vocab_word)
  213. vocab_word = dict((w, i) for i, w in enumerate(np.array(vocab)))
  214. else:
  215. model = getModel_word()
  216. vocab,_ = getVocabAndMatrix(model, Embedding_size=60)
  217. vocab_word = dict((w, i) for i, w in enumerate(np.array(vocab)))
  218. save(vocab,file_vocab_word)
  219. if word in vocab_word.keys():
  220. return vocab_word[word]
  221. else:
  222. return vocab_word['<pad>']
  223. def changeIndexFromWordToWords(tokens,word_index):
  224. '''
  225. @summary:转换某个字的字偏移为词偏移
  226. '''
  227. before_index = 0
  228. after_index = 0
  229. for i in range(len(tokens)):
  230. after_index = after_index+len(tokens[i])
  231. if before_index<=word_index and after_index>=word_index:
  232. return i
  233. before_index = after_index
  234. def getIndexOfWords(words):
  235. global vocab_words,file_vocab_words
  236. if vocab_words is None:
  237. if os.path.exists(file_vocab_words):
  238. vocab = load(file_vocab_words)
  239. vocab_words = dict((w, i) for i, w in enumerate(np.array(vocab)))
  240. else:
  241. model = getModel_w2v()
  242. vocab,_ = getVocabAndMatrix(model, Embedding_size=128)
  243. vocab_words = dict((w, i) for i, w in enumerate(np.array(vocab)))
  244. save(vocab,file_vocab_words)
  245. if words in vocab_words.keys():
  246. return vocab_words[words]
  247. else:
  248. return vocab_words["<pad>"]
  249. def log(msg):
  250. '''
  251. @summary:打印信息
  252. '''
  253. logger.info(msg)
  254. def debug(msg):
  255. '''
  256. @summary:打印信息
  257. '''
  258. logger.debug(msg)
  259. def save(object_to_save, path):
  260. '''
  261. 保存对象
  262. @Arugs:
  263. object_to_save: 需要保存的对象
  264. @Return:
  265. 保存的路径
  266. '''
  267. with open(path, 'wb') as f:
  268. pickle.dump(object_to_save, f)
  269. def load(path):
  270. '''
  271. 读取对象
  272. @Arugs:
  273. path: 读取的路径
  274. @Return:
  275. 读取的对象
  276. '''
  277. with open(path, 'rb') as f:
  278. object1 = pickle.load(f)
  279. return object1
  280. fool_char_to_id = load(os.path.dirname(__file__)+"/fool_char_to_id.pk")
  281. def getIndexOfWord_fool(word):
  282. if word in fool_char_to_id.keys():
  283. return fool_char_to_id[word]
  284. else:
  285. return fool_char_to_id["[UNK]"]
  286. def find_index(list_tofind,text):
  287. '''
  288. @summary: 查找所有词汇在字符串中第一次出现的位置
  289. @param:
  290. list_tofind:待查找词汇
  291. text:字符串
  292. @return: list,每个词汇第一次出现的位置
  293. '''
  294. result = []
  295. for item in list_tofind:
  296. index = text.find(item)
  297. if index>=0:
  298. result.append(index)
  299. else:
  300. result.append(-1)
  301. return result
  302. def combine(list1,list2):
  303. '''
  304. @summary:将两个list中的字符串两两拼接
  305. @param:
  306. list1:字符串list
  307. list2:字符串list
  308. @return:拼接结果list
  309. '''
  310. result = []
  311. for item1 in list1:
  312. for item2 in list2:
  313. result.append(str(item1)+str(item2))
  314. return result
  315. def getDigitsDic(unit):
  316. '''
  317. @summary:拿到中文对应的数字
  318. '''
  319. DigitsDic = {"零":0, "壹":1, "贰":2, "叁":3, "肆":4, "伍":5, "陆":6, "柒":7, "捌":8, "玖":9,
  320. "〇":0, "一":1, "二":2, "三":3, "四":4, "五":5, "六":6, "七":7, "八":8, "九":9}
  321. return DigitsDic.get(unit)
  322. def getMultipleFactor(unit):
  323. '''
  324. @summary:拿到单位对应的值
  325. '''
  326. MultipleFactor = {"兆":Decimal(1000000000000),"亿":Decimal(100000000),"万":Decimal(10000),"仟":Decimal(1000),"千":Decimal(1000),"佰":Decimal(100),"百":Decimal(100),"拾":Decimal(10),"十":Decimal(10),"元":Decimal(1),"圆":Decimal(1),"角":round(Decimal(0.1),1),"分":round(Decimal(0.01),2)}
  327. return MultipleFactor.get(unit)
  328. def getUnifyMoney(money):
  329. '''
  330. @summary:将中文金额字符串转换为数字金额
  331. @param:
  332. money:中文金额字符串
  333. @return: decimal,数据金额
  334. '''
  335. MAX_MONEY = 1000000000000
  336. MAX_NUM = 12
  337. #去掉逗号
  338. money = re.sub("[,,]","",money)
  339. money = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",money)
  340. result = Decimal(0)
  341. chnDigits = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"]
  342. chnFactorUnits = ["兆", "亿", "万", "仟", "佰", "拾","圆","元","角","分"]
  343. LowMoneypattern = re.compile("^[\d,]+(\.\d+)?$")
  344. BigMoneypattern = re.compile("^零?(?P<BigMoney>[%s])$"%("".join(chnDigits)))
  345. try:
  346. if re.search(LowMoneypattern,money) is not None:
  347. return Decimal(money)
  348. elif re.search(BigMoneypattern,money) is not None:
  349. return getDigitsDic(re.search(BigMoneypattern,money).group("BigMoney"))
  350. for factorUnit in chnFactorUnits:
  351. if re.search(re.compile(".*%s.*"%(factorUnit)),money) is not None:
  352. subMoneys = re.split(re.compile("%s(?!.*%s.*)"%(factorUnit,factorUnit)),money)
  353. if re.search(re.compile("^(\d+)(\.\d+)?$"),subMoneys[0]) is not None:
  354. if MAX_MONEY/getMultipleFactor(factorUnit)<Decimal(subMoneys[0]):
  355. return Decimal(0)
  356. result += Decimal(subMoneys[0])*(getMultipleFactor(factorUnit))
  357. elif len(subMoneys[0])==1:
  358. if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[0]) is not None:
  359. result += Decimal(getDigitsDic(subMoneys[0]))*(getMultipleFactor(factorUnit))
  360. else:
  361. result += Decimal(getUnifyMoney(subMoneys[0]))*(getMultipleFactor(factorUnit))
  362. if len(subMoneys)>1:
  363. if re.search(re.compile("^(\d+(,)?)+(\.\d+)?[百千万亿]?\s?(元)?$"),subMoneys[1]) is not None:
  364. result += Decimal(subMoneys[1])
  365. elif len(subMoneys[1])==1:
  366. if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[1]) is not None:
  367. result += Decimal(getDigitsDic(subMoneys[1]))
  368. else:
  369. result += Decimal(getUnifyMoney(subMoneys[1]))
  370. break
  371. except Exception as e:
  372. return Decimal(0)
  373. return result
  374. def getModel_w2v():
  375. '''
  376. @summary:加载词向量
  377. '''
  378. global model_w2v,lock_model_w2v
  379. with lock_model_w2v:
  380. if model_w2v is None:
  381. model_w2v = gensim.models.KeyedVectors.load_word2vec_format(getw2vfilepath(),binary=True)
  382. return model_w2v
  383. def getModel_word():
  384. '''
  385. @summary:加载字向量
  386. '''
  387. global model_word,lock_model_w2v
  388. with lock_model_word:
  389. if model_word is None:
  390. model_word = gensim.models.KeyedVectors.load_word2vec_format(model_word_file,binary=True)
  391. return model_word
  392. # getModel_w2v()
  393. # getModel_word()
  394. def findAllIndex(substr,wholestr):
  395. '''
  396. @summary: 找到字符串的子串的所有begin_index
  397. @param:
  398. substr:子字符串
  399. wholestr:子串所在完整字符串
  400. @return: list,字符串的子串的所有begin_index
  401. '''
  402. copystr = wholestr
  403. result = []
  404. indexappend = 0
  405. while(True):
  406. index = copystr.find(substr)
  407. if index<0:
  408. break
  409. else:
  410. result.append(indexappend+index)
  411. indexappend += index+len(substr)
  412. copystr = copystr[index+len(substr):]
  413. return result
  414. def spanWindow(tokens,begin_index,end_index,size,center_include=False,word_flag = False,use_text = False,text = None):
  415. '''
  416. @summary:取得某个实体的上下文词汇
  417. @param:
  418. tokens:句子分词list
  419. begin_index:实体的开始index
  420. end_index:实体的结束index
  421. size:左右两边各取多少个词
  422. center_include:是否包含实体
  423. word_flag:词/字,默认是词
  424. @return: list,实体的上下文词汇
  425. '''
  426. if use_text:
  427. assert text is not None
  428. length_tokens = len(tokens)
  429. if begin_index>size:
  430. begin = begin_index-size
  431. else:
  432. begin = 0
  433. if end_index+size<length_tokens:
  434. end = end_index+size+1
  435. else:
  436. end = length_tokens
  437. result = []
  438. if not word_flag:
  439. result.append(tokens[begin:begin_index])
  440. if center_include:
  441. if use_text:
  442. result.append(text)
  443. else:
  444. result.append(tokens[begin_index:end_index+1])
  445. result.append(tokens[end_index+1:end])
  446. else:
  447. result.append("".join(tokens[begin:begin_index]))
  448. if center_include:
  449. if use_text:
  450. result.append(text)
  451. else:
  452. result.append("".join(tokens[begin_index:end_index+1]))
  453. result.append("".join(tokens[end_index+1:end]))
  454. #print(result)
  455. return result
  456. #根据规则补全编号或名称两边的符号
  457. def fitDataByRule(data):
  458. symbol_dict = {"(":")",
  459. "(":")",
  460. "[":"]",
  461. "【":"】",
  462. ")":"(",
  463. ")":"(",
  464. "]":"[",
  465. "】":"【"}
  466. leftSymbol_pattern = re.compile("[\((\[【]")
  467. rightSymbol_pattern = re.compile("[\))\]】]")
  468. leftfinds = re.findall(leftSymbol_pattern,data)
  469. rightfinds = re.findall(rightSymbol_pattern,data)
  470. result = data
  471. if len(leftfinds)+len(rightfinds)==0:
  472. return data
  473. elif len(leftfinds)==len(rightfinds):
  474. return data
  475. elif abs(len(leftfinds)-len(rightfinds))==1:
  476. if len(leftfinds)>len(rightfinds):
  477. if symbol_dict.get(data[0]) is not None:
  478. result = data[1:]
  479. else:
  480. #print(symbol_dict.get(leftfinds[0]))
  481. result = data+symbol_dict.get(leftfinds[0])
  482. else:
  483. if symbol_dict.get(data[-1]) is not None:
  484. result = data[:-1]
  485. else:
  486. result = symbol_dict.get(rightfinds[0])+data
  487. result = re.sub("[。]","",result)
  488. return result
  489. time_format_pattern = re.compile("((?P<year>\d{4}|\d{2})\s*[-\/年\.]\s*(?P<month>\d{1,2})\s*[-\/月\.]\s*(?P<day>\d{1,2}))")
  490. def timeFormat(_time):
  491. current_year = time.strftime("%Y",time.localtime())
  492. all_match = re.finditer(time_format_pattern,_time)
  493. for _match in all_match:
  494. if len(_match.group())>0:
  495. legal = True
  496. year = ""
  497. month = ""
  498. day = ""
  499. for k,v in _match.groupdict().items():
  500. if k=="year":
  501. year = v
  502. if k=="month":
  503. month = v
  504. if k=="day":
  505. day = v
  506. if year!="":
  507. if len(year)==2:
  508. year = "20"+year
  509. if int(year)>int(current_year):
  510. legal = False
  511. else:
  512. legal = False
  513. if month!="":
  514. if int(month)>12:
  515. legal = False
  516. else:
  517. legal = False
  518. if day!="":
  519. if int(day)>31:
  520. legal = False
  521. else:
  522. legal = False
  523. if legal:
  524. return "%s-%s-%s"%(year,month.rjust(2,"0"),day.rjust(2,"0"))
  525. return ""
  526. def embedding(datas,shape):
  527. '''
  528. @summary:查找词汇对应的词向量
  529. @param:
  530. datas:词汇的list
  531. shape:结果的shape
  532. @return: array,返回对应shape的词嵌入
  533. '''
  534. model_w2v = getModel_w2v()
  535. embed = np.zeros(shape)
  536. length = shape[1]
  537. out_index = 0
  538. #print(datas)
  539. for data in datas:
  540. index = 0
  541. for item in data:
  542. item_not_space = re.sub("\s*","",item)
  543. if index>=length:
  544. break
  545. if item_not_space in model_w2v.vocab:
  546. embed[out_index][index] = model_w2v[item_not_space]
  547. index += 1
  548. else:
  549. #embed[out_index][index] = model_w2v['unk']
  550. index += 1
  551. out_index += 1
  552. return embed
  553. def embedding_word(datas,shape):
  554. '''
  555. @summary:查找词汇对应的词向量
  556. @param:
  557. datas:词汇的list
  558. shape:结果的shape
  559. @return: array,返回对应shape的词嵌入
  560. '''
  561. model_w2v = getModel_word()
  562. embed = np.zeros(shape)
  563. length = shape[1]
  564. out_index = 0
  565. #print(datas)
  566. for data in datas:
  567. index = 0
  568. for item in str(data)[-shape[1]:]:
  569. if index>=length:
  570. break
  571. if item in model_w2v.vocab:
  572. embed[out_index][index] = model_w2v[item]
  573. index += 1
  574. else:
  575. # embed[out_index][index] = model_w2v['unk']
  576. index += 1
  577. out_index += 1
  578. return embed
  579. def formEncoding(text,shape=(100,60),expand=False):
  580. embedding = np.zeros(shape)
  581. word_model = getModel_word()
  582. for i in range(len(text)):
  583. if i>=shape[0]:
  584. break
  585. if text[i] in word_model.vocab:
  586. embedding[i] = word_model[text[i]]
  587. if expand:
  588. embedding = np.expand_dims(embedding,0)
  589. return embedding
  590. def partMoney(entity_text,input2_shape = [7]):
  591. '''
  592. @summary:对金额分段
  593. @param:
  594. entity_text:数值金额
  595. input2_shape:分类数
  596. @return: array,分段之后的独热编码
  597. '''
  598. money = float(entity_text)
  599. parts = np.zeros(input2_shape)
  600. if money<100:
  601. parts[0] = 1
  602. elif money<1000:
  603. parts[1] = 1
  604. elif money<10000:
  605. parts[2] = 1
  606. elif money<100000:
  607. parts[3] = 1
  608. elif money<1000000:
  609. parts[4] = 1
  610. elif money<10000000:
  611. parts[5] = 1
  612. else:
  613. parts[6] = 1
  614. return parts
  615. def recall(y_true, y_pred):
  616. '''
  617. 计算召回率
  618. @Argus:
  619. y_true: 正确的标签
  620. y_pred: 模型预测的标签
  621. @Return
  622. 召回率
  623. '''
  624. c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
  625. c3 = K.sum(K.round(K.clip(y_true, 0, 1)))
  626. if c3 == 0:
  627. return 0
  628. recall = c1 / c3
  629. return recall
  630. def f1_score(y_true, y_pred):
  631. '''
  632. 计算F1
  633. @Argus:
  634. y_true: 正确的标签
  635. y_pred: 模型预测的标签
  636. @Return
  637. F1值
  638. '''
  639. c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
  640. c2 = K.sum(K.round(K.clip(y_pred, 0, 1)))
  641. c3 = K.sum(K.round(K.clip(y_true, 0, 1)))
  642. precision = c1 / c2
  643. if c3 == 0:
  644. recall = 0
  645. else:
  646. recall = c1 / c3
  647. f1_score = 2 * (precision * recall) / (precision + recall)
  648. return f1_score
  649. def precision(y_true, y_pred):
  650. '''
  651. 计算精确率
  652. @Argus:
  653. y_true: 正确的标签
  654. y_pred: 模型预测的标签
  655. @Return
  656. 精确率
  657. '''
  658. c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
  659. c2 = K.sum(K.round(K.clip(y_pred, 0, 1)))
  660. precision = c1 / c2
  661. return precision
  662. # def print_metrics(history):
  663. # '''
  664. # 制作每次迭代的各metrics变化图片
  665. #
  666. # @Arugs:
  667. # history: 模型训练迭代的历史记录
  668. # '''
  669. # import matplotlib.pyplot as plt
  670. #
  671. # # loss图
  672. # loss = history.history['loss']
  673. # val_loss = history.history['val_loss']
  674. # epochs = range(1, len(loss) + 1)
  675. # plt.subplot(2, 2, 1)
  676. # plt.plot(epochs, loss, 'bo', label='Training loss')
  677. # plt.plot(epochs, val_loss, 'b', label='Validation loss')
  678. # plt.title('Training and validation loss')
  679. # plt.xlabel('Epochs')
  680. # plt.ylabel('Loss')
  681. # plt.legend()
  682. #
  683. # # f1图
  684. # f1 = history.history['f1_score']
  685. # val_f1 = history.history['val_f1_score']
  686. # plt.subplot(2, 2, 2)
  687. # plt.plot(epochs, f1, 'bo', label='Training f1')
  688. # plt.plot(epochs, val_f1, 'b', label='Validation f1')
  689. # plt.title('Training and validation f1')
  690. # plt.xlabel('Epochs')
  691. # plt.ylabel('F1')
  692. # plt.legend()
  693. #
  694. # # precision图
  695. # prec = history.history['precision']
  696. # val_prec = history.history['val_precision']
  697. # plt.subplot(2, 2, 3)
  698. # plt.plot(epochs, prec, 'bo', label='Training precision')
  699. # plt.plot(epochs, val_prec, 'b', label='Validation pecision')
  700. # plt.title('Training and validation precision')
  701. # plt.xlabel('Epochs')
  702. # plt.ylabel('Precision')
  703. # plt.legend()
  704. #
  705. # # recall图
  706. # recall = history.history['recall']
  707. # val_recall = history.history['val_recall']
  708. # plt.subplot(2, 2, 4)
  709. # plt.plot(epochs, recall, 'bo', label='Training recall')
  710. # plt.plot(epochs, val_recall, 'b', label='Validation recall')
  711. # plt.title('Training and validation recall')
  712. # plt.xlabel('Epochs')
  713. # plt.ylabel('Recall')
  714. # plt.legend()
  715. #
  716. # plt.show()
  717. if __name__=="__main__":
  718. print(fool_char_to_id[">"])
  719. # model = getModel_w2v()
  720. # vocab,matrix = getVocabAndMatrix(model, Embedding_size=128)
  721. # save([vocab,matrix],"vocabMatrix_words.pk")