BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/test/测试整个要素提取流程.py

'''
Created on 2019年1月4日

@author: User
'''

from bs4 import BeautifulSoup, Comment
import copy
import re
import sys
import os
import codecs
import requests
import time

_time1 = time.time()
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath('../../'))
print('当前路径为:',os.getcwd())
print('sys.path',sys.path)
import fool
from BiddingKG.dl.interface.Connection import *
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Connection import getConnection
import BiddingKG.dl.interface.predictor as predictor
import BiddingKG.dl.interface.Preprocessing as Preprocessing
import BiddingKG.dl.interface.getAttributes as getAttributes
import BiddingKG.dl.entityLink.entityLink as entityLink
import json


'''
doc_id = '3537ed94-9c7e-11e8-aed5-44a84246dbba'

conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")

cursor = conn.cursor()

cursor.execute(" select content from articles where id='"+doc_id+"' ")

row = cursor.fetchall()[0]


#text = codecs.open("C:\\Users\\User\\Desktop\\1.html","r",encoding="utf8").read()

#content = segment(tableToText(BeautifulSoup(row[0],"lxml")))
'''

'''''' 
codeNamePredict = predictor.CodeNamePredict()
premPredict = predictor.PREMPredict()
epcPredict = predictor.EPCPredict()
roleRulePredict = predictor.RoleRulePredictor()

#自定义jsonEncoder
class MyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, bytes):
            return str(obj, encoding='utf-8')
        elif isinstance(obj, (np.float_, np.float16, np.float32, 
        np.float64)):
            return float(obj)
        elif isinstance(obj,str):
            return obj
        return json.JSONEncoder.default(self, obj)


def predict(doc_id,text):
    list_articles,list_sentences,list_entitys,_ = Preprocessing.get_preprocessed([[doc_id,text,"","",""]],useselffool=True)
    # for articles in list_articles:
    #     print('预处理后文本信息')
    #     print(articles.content)


    ''''''
        
    codeName = codeNamePredict.predict(list_sentences,list_entitys=list_entitys)
    # print(codeName)
    premPredict.predict(list_sentences,list_entitys)
    roleRulePredict.predict(list_articles,list_sentences, list_entitys,codeName)
    # print("epcPredict")
    epcPredict.predict(list_sentences,list_entitys)
    # print("entityLink")
    entityLink.link_entitys(list_entitys)
    # print("getPREMs")
    prem = getAttributes.getPREMs(list_sentences,list_entitys,list_articles)
    # print("getPREMs")
    
    
    ''''''
    
    entitys_all = [[[entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.begin_index,entity.end_index] for entity in entitys] for entitys in list_entitys]
    for entitys in entitys_all:
        # print(entitys)
        # en_types = set([it[1] for it in entitys])
        print([(it[0],it[1], it[2],it[3][it[2]],it[4],it[5],it[6]) for it in entitys if it[1] in ('org', 'company', 'person')])
        # print([it for it in entitys if it[1] in ('org','company','person')])
        # for en_type in en_types:
        #     print('***************************************')
        #     print(en_type)
        #     print([(it[0],it[2],it[3]) for it in entitys if it[1]==en_type])

    # for entitys in list_entitys:
    #     for entity in entitys:
    #         print('**********实体信息****************')
    #         print(entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.wordOffset_begin,entity.wordOffset_end)

    #print(prem)
    return json.dumps(Preprocessing.union_result(codeName, prem)[0][1],cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)

         
# def test(name,content):
#     user = {
#             "content": content,
#             "id":name
#             }
#     myheaders = {'Content-Type': 'application/json'}
#     _resp = requests.post("http://192.168.2.101:15015" + '/article_extract', json=user, headers=myheaders, verify=True)
#     resp_json = _resp.content.decode("utf-8")
#     print(resp_json)
#     return resp_json
def get_result_online(docid):
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    user = {
            "content": rows[0][1],
            "id":docid
            }
    myheaders = {'Content-Type': 'application/json'}
    _resp = requests.post("http://192.168.2.101:15030" + '/article_extract', json=user, headers=myheaders, verify=True)  # 15015  #最新模型15030
    resp_json = _resp.content.decode("utf-8")
    return json.loads(resp_json)

def get_result(docid):
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    return(json.loads(predict(docid, rows[0][1])))

def analys_person_phone():
    import pandas as pd
    import time
    t1 = time.time()
    df = pd.read_excel(r'E:\workspace\BiddingKG\BiddingKG\dl\person\实习生标注信息角色联系人电话.xlsx', encoding='utf-8')
    lab_num = pos_num = pre_num = 0
    lab_num2 = pos_num2 = pre_num2 = 0
    lab_person = pos_person = pre_person = 0
    lab_role = pos_role = pre_role = 0
    person_errors = []
    phone_errors = []
    join_errors = []
    person_name_errors =[]
    role_name_errors =[]
    for docid in set(df['doc_id']):
        print('开始处理 ： ',docid)
        df_tmp = df[df.loc[:, 'doc_id'] == docid]
        values = list(df_tmp['value'])
        a = [it.split() for it in values]
        rel_person = [it for it in a if it[1] == 'rel_person']
        rel_phone = [it for it in a if it[1] == 'rel_phone']
        r1 = get_result(str(docid))
        # r1 = get_result_online(str(docid))
        label_role_person = []  # 标注角色+联系人
        for rel in rel_person:
            role = [it for it in a if it[0] == rel[2].split(':')[-1]]
            person = [it for it in a if it[0] == rel[3].split(':')[-1]]
            if person != [] and role != []:
                label_role_person.append(role[0][-1] +'+'+ person[0][-1])
        label_person_phone = []  # 标注角色+联系人
        for rel in rel_phone:
            person = [it for it in a if it[0] == rel[2].split(':')[-1]]
            phone = [it for it in a if it[0] == rel[3].split(':')[-1]]
            if person != [] and phone != []:
                label_person_phone.append(person[0][-1] +'+'+ phone[0][-1])
        role_person = []
        person_phone = []
        if r1.get('success','')==False:
            print(docid, '接口返回失败 ')
        else:
            for v in r1['prem'].values():
                roleList = v['roleList']
                for role in roleList:
                    for it in role[3]:
                        role_person.append(role[1] +'+'+ it[0])
                for role in roleList:
                    for it in role[3]:
                        person_phone.append(it[0] +'+'+ it[1])
                    # print(set(label_person_phone))
            # print(set(person_phone))
        pos_num += len(set(role_person) & set(label_role_person))
        lab_num += len(set(label_role_person))
        pre_num += len(set(role_person))
        if set(role_person)&set(label_role_person) != set(label_role_person):
            person_errors.append([docid, set(label_role_person), set(role_person)])
            # 判断角色联系人是否正确逻辑：1、先看预测角色是否都在标签角色里，2判断预测联系人是否在标签联系人，
            # print(set(role_person))
            # print(set(label_role_person))
        if set(label_person_phone) & set(person_phone)!=set(label_person_phone):
            phone_errors.append([docid, set(label_person_phone), set(person_phone)])
        pos_num2 += len(set(label_person_phone) & set(person_phone))
        lab_num2 += len(set(label_person_phone))
        pre_num2 += len(set(person_phone))

        lab_person += len(set([it.split('+')[1] for it in label_role_person]))
        pos_person += len(set([it.split('+')[1] for it in label_role_person])&set([it.split('+')[1] for it in role_person]))
        pre_person += len(set([it.split('+')[1] for it in role_person]))

        lab_role += len(set([it.split('+')[0] for it in label_role_person]))
        pos_role += len(set([it.split('+')[0] for it in label_role_person])&set([it.split('+')[0] for it in role_person]))
        pre_role += len(set([it.split('+')[0] for it in role_person]))

        if set([it.split('+')[0] for it in label_role_person]) != set([it.split('+')[0] for it in role_person]):
            if set([it.split('+')[1] for it in label_role_person]) != set([it.split('+')[1] for it in role_person]):
                person_name_errors.append([docid,set(label_role_person), set(role_person)])
            else:
                role_name_errors.append([docid, set(label_role_person), set(role_person)])
        else:
            if set([it.split('+')[1] for it in label_role_person]) != set([it.split('+')[1] for it in role_person]):
                person_name_errors.append([docid, set(label_role_person), set(role_person)])
            elif set(label_role_person)!= set(role_person):
                print(docid,set(label_role_person), set(role_person))
                join_errors.append([docid,set(label_role_person), set(role_person)])
    print('单独角色召回率：%.4f，准确率：%.4f'%(pos_role/lab_role, pos_role/pre_role))
    print('单独联系人召回率：%.4f, 准确率：%.4f'%(pos_person/lab_person, pos_person/pre_person))
    print('联系人召回率：%.4f, 准确率：%.4f' % (pos_num / lab_num, pos_num / pre_num))
    print('电话召回率：%.4f，准确率：%.4f' % (pos_num2 / lab_num2, pos_num2 / pre_num2))
    print('总耗时：',time.time()-t1)
    return person_errors, phone_errors, join_errors, role_name_errors, person_name_errors

def predict_fromdb(docid, dbname="sys_document_23"):
    # import pymysql
    # conn = pymysql.Connect(host='rm-bp1quo50k0q2ok73gi.mysql.rds.aliyuncs.com', port=3306, db='bxkc', user='bxkc_read', passwd='bxkc_20RE18AD') #新账号密码
    # cursor = conn.cursor()
    # sql = "SELECT  docid as id, dochtmlcon as content  from {1} WHERE DOCID='{0}';".format(docid, dbname)
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier as id,sourcetext as content from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    doc_id = rows[0][0]
    text = rows[0][1]
    # text = '竟然很明显的表达没识别为代理，代理机构名称：国信国采(北京)招标咨询有限责任公司，代理机构地址：北京市海淀区首体南路22号国兴大厦11层，  1.采购人信息名 称：北京市植物园。'
    list_articles, list_sentences, list_entitys, _ = Preprocessing.get_preprocessed([[doc_id, text, "", "", ""]],useselffool=True)
    codeName = codeNamePredict.predict(list_sentences, list_entitys=list_entitys)
    # print(codeName)
    premPredict.predict(list_sentences, list_entitys)
    roleRulePredict.predict(list_articles, list_sentences, list_entitys, codeName)
    # print("epcPredict")
    epcPredict.predict(list_sentences, list_entitys)
    # print("entityLink")
    entityLink.link_entitys(list_entitys)
    # print("getPREMs")
    prem = getAttributes.getPREMs(list_sentences, list_entitys, list_articles)
    return list_articles, list_sentences, list_entitys, codeName, prem

if __name__=="__main__":
    # import pandas as pd
    # import math
    # import pymysql
    # conn = pymysql.Connect(host='rm-bp1quo50k0q2ok73gi.mysql.rds.aliyuncs.com', port=3306, db='bxkc', user='bxkc_read', passwd='bxkc_20RE18AD') #新账号密码
    # cursor = conn.cursor()
    # df = pd.read_excel('G:/大网站规则识别/1027统计入库top100编号.xlsx')
    # docs_list = []
    # for i in range(100):
    #     web_no = df.loc[i, '编号']
    #     # num = math.ceil(int(df.loc[i, '1019-1023入库公告数量']) * 0.01)
    #     num = 10
    #     sql = "SELECT DOCID,DOCCHANNEL,DOCHTMLCON,WEB_SOURCE_NO from sys_document_23 where WEB_SOURCE_NO='{0}' and DOCCHANNEL='101' and DOCID%9=1 limit {1}".format(
    #         web_no, num)
    #     #  rows = cursor.execute(sql) 此处代码错误 rows 需要用 cursor.fetchall方法获取
    #     cursor.execute(sql)
    #     rows = cursor.fetchall()
    #     docs_list.extend(list(rows))
    # df_doc = pd.DataFrame(docs_list, columns=['docid', 'channel', 'html', 'web_no'])
    # codenames = []
    # prems = []
    # for docid,text in zip(df_doc['docid'], df_doc['html']):
    #     list_articles, list_sentences, list_entitys, _ = Preprocessing.get_preprocessed([[docid, text, "", "", ""]],
    #                                                                                     useselffool=True)
    #     codeName = codeNamePredict.predict(list_sentences, list_entitys=list_entitys)
    #     # print(codeName)
    #     premPredict.predict(list_sentences, list_entitys)
    #     roleRulePredict.predict(list_articles, list_sentences, list_entitys, codeName)
    #     # print("epcPredict")
    #     epcPredict.predict(list_sentences, list_entitys)
    #     # print("entityLink")
    #     entityLink.link_entitys(list_entitys)
    #     # print("getPREMs")
    #     prem = getAttributes.getPREMs(list_sentences, list_entitys, list_articles)
    #     if codeName:
    #         codenames.append(codeName[0][1])
    #     else:
    #         codenames.append(" ")
    #     if prem:
    #         prems.append(prem[0][1])
    #     else:
    #         prems.append(" ")
    # df_doc['codename'] = pd.Series(codenames)
    # df_doc['prem'] = pd.Series(prems)
    # df_doc.to_excel('G:/大网站规则识别/大网站规则调整后预测结果20201124.xlsx', columns=['docid', 'channel', 'html', 'prem', 'codename', 'web_no'])


    list_articles, list_sentences, list_entitys, codeName, prem = predict_fromdb('100006370',dbname="sys_document_25")  #sys_document_23
    print(prem)
    print(codeName)
    entitys_all = [[[entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.begin_index,entity.end_index] for entity in entitys] for entitys in list_entitys]
    for entitys in entitys_all:
        # print(entitys)
        # en_types = set([it[1] for it in entitys])
        print([(it[0],it[1], it[2],it[3][it[2]],it[4],it[5],it[6]) for it in entitys if it[1] in ('org', 'company', 'person')])
    print(list_articles[0].content)

    # print(get_result('100000203'))

    # person_errors, phone_errors, join_errors, role_name_errors, person_name_errors = analys_person_phone()
    # import pickle
    # with open('phone_errors.pkl','wb') as f:
    #     pickle.dump(phone_errors, f)

    # filename = "比地_52_79929693.html"
    # #text = codecs.open("C:\\Users\\User\\Desktop\\数据20191014\\"+filename,"r",encoding="utf8").read()
    # # text = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # # text = codecs.open('F:/工作文档/实体识别实体对其/20190320/比地_101_58511386.html', encoding='utf-8').read()
    # docid = '100000203'
    # r1 = get_result(docid)
    # r2 = get_result_online(docid)
    # rolperson = []
    # person_phone = []
    # for v in r1['prem'].values():
    #     roleList = v['roleList']
    #     for role in roleList:
    #         for it in role[3]:
    #             rolperson.append(role[1] + it[0])
    #     for role in roleList:
    #         for it in role[3]:
    #             person_phone.append(it[0]+it[1])
    # print(r1['prem'])
    # print(r2['prem'])
    #
    # import psycopg2
    # conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    # cursor = conn.cursor()
    # sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('95008163');"""
    # cursor.execute(sql)
    # rows = cursor.fetchall()
    # # print(len(rows), rows)
    # content = rows[0][1]
    # # content = str(BeautifulSoup(text).find("div",id="pcontent"))
    # # content = text
    # # print('content: ',content)
    # #text = codecs.open("C:\\Users\\User\\Desktop\\a.html","r",encoding="utf8").read()
    # #text = "张家港保税区宏宇建设咨询有限公司受张家港市给排水公司委托，就二次供水泵房浊度仪进行国内组织公开招标采购，欢迎符合条件的供应商参加投标。"
    # a = time.time()
    # print("start")
    # # print(predict("12",content))
    # result = predict("12",content)
    # print(json.loads(result))
    # #test("12",text)
    # print("takes",time.time()-a)
    # _time2 = time.time()
    # print(predict("12",content))
    # _time3 = time.time()
    # print("init takes:%d"%((_time2-_time1)-(_time3-_time2)))
    # pass