extract.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. '''
  2. Created on 2019年1月4日
  3. @author: User
  4. '''
  5. import os
  6. from bs4 import BeautifulSoup, Comment
  7. import copy
  8. import re
  9. import sys
  10. import os
  11. import codecs
  12. import requests
  13. import time
  14. _time1 = time.time()
  15. sys.path.append(os.path.abspath("../.."))
  16. from BiddingKG.dl.common.Utils import *
  17. import BiddingKG.dl.entityLink.entityLink as entityLink
  18. import BiddingKG.dl.interface.predictor as predictor
  19. import BiddingKG.dl.interface.Preprocessing as Preprocessing
  20. import BiddingKG.dl.interface.getAttributes as getAttributes
  21. import BiddingKG.dl.complaint.punish_predictor as punish_rule
  22. import json
  23. #自定义jsonEncoder
  24. class MyEncoder(json.JSONEncoder):
  25. def default(self, obj):
  26. if isinstance(obj, np.ndarray):
  27. return obj.tolist()
  28. elif isinstance(obj, bytes):
  29. return str(obj, encoding='utf-8')
  30. elif isinstance(obj, (np.float_, np.float16, np.float32,
  31. np.float64)):
  32. return float(obj)
  33. elif isinstance(obj,str):
  34. return obj
  35. return json.JSONEncoder.default(self, obj)
  36. def predict(doc_id,text,title="",page_time="",**kwargs):
  37. cost_time = dict()
  38. start_time = time.time()
  39. log("start process doc %s"%(str(doc_id)))
  40. list_articles,list_sentences,list_entitys,_cost_time = Preprocessing.get_preprocessed([[doc_id,text,"","",title,page_time]],useselffool=True)
  41. log("get preprocessed done of doc_id%s"%(doc_id))
  42. cost_time["preprocess"] = round(time.time()-start_time,2)
  43. cost_time.update(_cost_time)
  44. start_time = time.time()
  45. codeName = predictor.getPredictor("codeName").predict(list_sentences,MAX_AREA=5000,list_entitys=list_entitys)
  46. log("get codename done of doc_id%s"%(doc_id))
  47. cost_time["codename"] = round(time.time()-start_time,2)
  48. start_time = time.time()
  49. predictor.getPredictor("prem").predict(list_sentences,list_entitys)
  50. log("get prem done of doc_id%s"%(doc_id))
  51. cost_time["prem"] = round(time.time()-start_time,2)
  52. start_time = time.time()
  53. predictor.getPredictor("product").predict(list_sentences,list_entitys)
  54. log("get product done of doc_id%s"%(doc_id))
  55. cost_time["product"] = round(time.time()-start_time,2)
  56. start_time = time.time()
  57. product_attrs = predictor.getPredictor("product_attrs").predict(doc_id, text)
  58. log("get product attributes done of doc_id%s"%(doc_id))
  59. cost_time["product_attrs"] = round(time.time()-start_time,2)
  60. start_time = time.time()
  61. predictor.getPredictor("roleRule").predict(list_articles,list_sentences, list_entitys,codeName)
  62. cost_time["rule"] = round(time.time()-start_time,2)
  63. start_time = time.time()
  64. predictor.getPredictor("epc").predict(list_sentences,list_entitys)
  65. log("get epc done of doc_id%s"%(doc_id))
  66. cost_time["person"] = round(time.time()-start_time,2)
  67. start_time = time.time()
  68. predictor.getPredictor("time").predict(list_sentences, list_entitys)
  69. log("get time done of doc_id%s"%(doc_id))
  70. cost_time["time"] = round(time.time()-start_time,2)
  71. #依赖句子顺序
  72. start_time = time.time()
  73. entityLink.link_entitys(list_entitys)
  74. prem = getAttributes.getPREMs(list_sentences,list_entitys,list_articles)
  75. log("get attributes done of doc_id%s"%(doc_id))
  76. cost_time["attrs"] = round(time.time()-start_time,2)
  77. #依赖句子顺序
  78. start_time = time.time()
  79. list_channel_dic = predictor.getPredictor("channel").predict(title=title, content=list_sentences[0])
  80. cost_time["channel"] = round(time.time()-start_time,2)
  81. start_time = time.time()
  82. list_punish_dic = predictor.getPredictor("punish").get_punish_extracts(list_articles,list_sentences, list_entitys)
  83. cost_time["punish"] = round(time.time()-start_time,2)
  84. #print(prem)
  85. # data_res = Preprocessing.union_result(Preprocessing.union_result(codeName, prem),list_punish_dic)[0]
  86. # data_res = Preprocessing.union_result(Preprocessing.union_result(Preprocessing.union_result(codeName, prem),list_punish_dic), list_channel_dic)[0]
  87. data_res = dict(codeName[0], **prem[0], **list_channel_dic[0], **product_attrs[0])
  88. data_res["cost_time"] = cost_time
  89. data_res["success"] = True
  90. #
  91. # for _article in list_articles:
  92. # log(_article.content)
  93. #
  94. # for list_entity in list_entitys:
  95. # for _entity in list_entity:
  96. # log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
  97. # (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
  98. # str(_entity.begin_index),str(_entity.end_index)))
  99. return json.dumps(data_res,cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)
  100. def test(name,content):
  101. user = {
  102. "content": content,
  103. "id":name
  104. }
  105. myheaders = {'Content-Type': 'application/json'}
  106. _resp = requests.post("http://192.168.2.102:15030" + '/article_extract', json=user, headers=myheaders, verify=True)
  107. resp_json = _resp.content.decode("utf-8")
  108. # print(resp_json)
  109. return resp_json
  110. if __name__=="__main__":
  111. import pandas as pd
  112. t1 = time.time()
  113. text = '中标人:广州中医药有限公司,招标人:广州市第一人民医院, 代理机构:希达招标代理有限公司。招标金额:100万元, 手续费:100元,总投资:1亿元。中标金额:50000元。合同金额:50000万元。'
  114. title = '合同公告'
  115. print(predict('',text,title))
  116. # df = pd.read_excel('G:\公告金额/170角色金额原模型预测错误数据_new3为新预测中标金额_predict0812.xlsx')[:20]
  117. # new_prem = []
  118. # for i in range(len(df)):
  119. # # i = 246
  120. # doc_id = df.loc[i, 'docid']
  121. # text = df.loc[i, 'dochtmlcon']
  122. # title = df.loc[i, 'doctitle']
  123. # rs = predict(doc_id,text,title)
  124. # # print(rs)
  125. # new_prem.append(rs)
  126. # df['new_prem'] = pd.Series(new_prem)
  127. # print('耗时:', time.time()-t1)
  128. # df.to_excel('G:\公告金额/170角色金额原模型预测错误数据_new3为新预测中标金额_predict0813.xlsx')
  129. pass