BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/test/测试整个要素提取流程.py

'''
Created on 2019年1月4日

@author: User
'''

from bs4 import BeautifulSoup, Comment
import copy
import re
import sys
import os
import codecs
import requests
import time

_time1 = time.time()
sys.path.append(os.path.abspath("../.."))
sys.path.append(os.path.abspath('../../'))
print('当前路径为:',os.getcwd())
print('sys.path',sys.path)
import fool
from BiddingKG.dl.interface.Connection import *
from BiddingKG.dl.common.Utils import *
from BiddingKG.dl.interface.Connection import getConnection
import BiddingKG.dl.interface.predictor as predictor
import BiddingKG.dl.interface.Preprocessing as Preprocessing
import BiddingKG.dl.interface.getAttributes as getAttributes
import BiddingKG.dl.entityLink.entityLink as entityLink
import json


'''
doc_id = '3537ed94-9c7e-11e8-aed5-44a84246dbba'

conn = psycopg2.connect(dbname="BiddingKG",user="postgres",password="postgres",host="192.168.2.101")

cursor = conn.cursor()

cursor.execute(" select content from articles where id='"+doc_id+"' ")

row = cursor.fetchall()[0]


#text = codecs.open("C:\\Users\\User\\Desktop\\1.html","r",encoding="utf8").read()

#content = segment(tableToText(BeautifulSoup(row[0],"lxml")))
'''

'''''' 
codeNamePredict = predictor.CodeNamePredict()
premPredict = predictor.PREMPredict()
epcPredict = predictor.EPCPredict()
roleRulePredict = predictor.RoleRulePredictor()
timePredictor = predictor.TimePredictor()

#自定义jsonEncoder
class MyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, bytes):
            return str(obj, encoding='utf-8')
        elif isinstance(obj, (np.float_, np.float16, np.float32, 
        np.float64)):
            return float(obj)
        elif isinstance(obj,str):
            return obj
        return json.JSONEncoder.default(self, obj)


def predict(doc_id,text):
    list_articles,list_sentences,list_entitys,_ = Preprocessing.get_preprocessed([[doc_id,text,"","","",""]],useselffool=True)
    for articles in list_articles:
        print('预处理后文本信息')
        print(articles.content)
    for sentences in list_sentences:
        for sentence in sentences:
            print(sentence.tokens)


    ''''''
        
    codeName = codeNamePredict.predict(list_sentences,list_entitys=list_entitys)
    # print(codeName)
    premPredict.predict(list_sentences,list_entitys)
    # for entitys in list_entitys:
    #     for entity in entitys:
    #         if entity.entity_type in ['org','company']:
    #             print("公司->联系人2：", end=' ')
    #             print(entity.entity_text, entity.pointer_person,entity.label,entity.values)
    #             pass
    roleRulePredict.predict(list_articles,list_sentences, list_entitys,codeName)
    # print("epcPredict")
    epcPredict.predict(list_sentences,list_entitys)

    timePredictor.predict(list_sentences,list_entitys)
    # print("entityLink")
    entityLink.link_entitys(list_entitys)
    # print("getPREMs")
    prem = getAttributes.getPREMs(list_sentences,list_entitys,list_articles)
    # print("getPREMs")
    print("公司——联系人：", end=' ')
    print(prem[0])
    # print(prem[0]['prem']['Project']['roleList'])

    
    ''''''
    
    entitys_all = [[[entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.begin_index,entity.end_index] for entity in entitys] for entitys in list_entitys]
    # for entitys in entitys_all:
        # print(entitys)
        # en_types = set([it[1] for it in entitys])
        # print([(it[0],it[1], it[2],it[3][it[2]],it[4],it[5],it[6]) for it in entitys if it[1] in ('org', 'company', 'person')])
        # print([it for it in entitys if it[1] in ('org','company','person')])
        # for en_type in en_types:
        #     print('***************************************')
        #     print(en_type)
        #     print([(it[0],it[2],it[3]) for it in entitys if it[1]==en_type])
    for entitys in list_entitys:
        entitys = sorted(entitys, key=lambda x: (x.sentence_index, x.wordOffset_begin))
        for entity in entitys:
            # print('**********实体信息****************')
            if entity.entity_type=='person':
                print("联系人-电话：",end=' ')
                print(entity.entity_text,[i.entity_text for i in entity.person_phone] if entity.person_phone else None,entity.label,entity.values)
                if entity.pointer_email:
                    print("联系人-邮箱：",entity.entity_text,entity.pointer_email.entity_text)
                # print(entity.begin_index, entity.end_index)
                print(entity.sentence_index)
                pass
            elif entity.entity_type=="time":
                print("time:",end=" ")
                print(entity.entity_text, entity.label, entity.values)
            elif entity.entity_type=="email":
                print("email:",end=" ")
                print(entity.entity_text, entity.begin_index, entity.end_index)
            elif entity.entity_type in ['org','company']:
                _sentence = list_sentences[0][entity.sentence_index]
                print(entity.entity_type)
                if entity.pointer_person:
                    print("公司->联系人1：",end=' ')
                    print(entity.entity_text,[i.entity_text for i in entity.pointer_person],entity.label,entity.values)
                    # print(entity.entity_text,entity.label,entity.values)
                    print(_sentence.sentence_text,_sentence.tokens[entity.begin_index:entity.end_index+1])
                else:
                    print("公司->联系人2：", end=' ')
                    print(entity.entity_text, entity.pointer_person,entity.label,entity.values)
                    print(_sentence.sentence_text,_sentence.tokens[entity.begin_index:entity.end_index+1])
                    pass
                if entity.label in [2,3,4]:
                    if entity.pointer_money:
                        print("公司->中投标金额：", end=' ')
                        print(entity.entity_text, entity.pointer_money.entity_text)
                    if entity.pointer_serviceTime:
                        print("公司->工期：", end=' ')
                        print(entity.entity_text, entity.pointer_serviceTime.entity_text)
                    if entity.pointer_ratio:
                        print("公司->费率：", end=' ')
                        print(entity.entity_text, entity.pointer_ratio.entity_text)
                # print(entity.pointer_pack)
            # elif entity.entity_type =='serviceTime':
            #     print(entity.entity_text)
            #     if entity.pointer_pack:
            #         print('pointer_pack_name:',entity.pointer_pack.entity_text)
            # elif entity.entity_type =='money':
            #     print('money',entity.entity_text,entity.label)
            # elif entity.entity_type in ['package']:
            #     print('pack_entity:',entity.entity_text)
            # print(entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.wordOffset_begin,entity.wordOffset_end)

    #print(prem)
    return json.dumps(Preprocessing.union_result(codeName, prem)[0][1],cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)

         
# def test(name,content):
#     user = {
#             "content": content,
#             "id":name
#             }
#     myheaders = {'Content-Type': 'application/json'}
#     _resp = requests.post("http://192.168.2.101:15015" + '/article_extract', json=user, headers=myheaders, verify=True)
#     resp_json = _resp.content.decode("utf-8")
#     print(resp_json)
#     return resp_json
def get_result_online(docid):
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    user = {
            "content": rows[0][1],
            "id":docid
            }
    myheaders = {'Content-Type': 'application/json'}
    _resp = requests.post("http://192.168.2.101:15030" + '/article_extract', json=user, headers=myheaders, verify=True)  # 15015  #最新模型15030
    resp_json = _resp.content.decode("utf-8")
    return json.loads(resp_json)

def get_result(docid):
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    return(json.loads(predict(docid, rows[0][1])))

def analys_person_phone():
    import pandas as pd
    import time
    t1 = time.time()
    df = pd.read_excel(r'E:\workspace\BiddingKG\BiddingKG\dl\person\实习生标注信息角色联系人电话.xlsx', encoding='utf-8')
    lab_num = pos_num = pre_num = 0
    lab_num2 = pos_num2 = pre_num2 = 0
    lab_person = pos_person = pre_person = 0
    lab_role = pos_role = pre_role = 0
    person_errors = []
    phone_errors = []
    join_errors = []
    person_name_errors =[]
    role_name_errors =[]
    for docid in set(df['doc_id']):
        print('开始处理 ： ',docid)
        df_tmp = df[df.loc[:, 'doc_id'] == docid]
        values = list(df_tmp['value'])
        a = [it.split() for it in values]
        rel_person = [it for it in a if it[1] == 'rel_person']
        rel_phone = [it for it in a if it[1] == 'rel_phone']
        r1 = get_result(str(docid))
        # r1 = get_result_online(str(docid))
        label_role_person = []  # 标注角色+联系人
        for rel in rel_person:
            role = [it for it in a if it[0] == rel[2].split(':')[-1]]
            person = [it for it in a if it[0] == rel[3].split(':')[-1]]
            if person != [] and role != []:
                label_role_person.append(role[0][-1] +'+'+ person[0][-1])
        label_person_phone = []  # 标注角色+联系人
        for rel in rel_phone:
            person = [it for it in a if it[0] == rel[2].split(':')[-1]]
            phone = [it for it in a if it[0] == rel[3].split(':')[-1]]
            if person != [] and phone != []:
                label_person_phone.append(person[0][-1] +'+'+ phone[0][-1])
        role_person = []
        person_phone = []
        if r1.get('success','')==False:
            print(docid, '接口返回失败 ')
        else:
            for v in r1['prem'].values():
                roleList = v['roleList']
                for role in roleList:
                    for it in role[3]:
                        role_person.append(role[1] +'+'+ it[0])
                for role in roleList:
                    for it in role[3]:
                        person_phone.append(it[0] +'+'+ it[1])
                    # print(set(label_person_phone))
            # print(set(person_phone))
        pos_num += len(set(role_person) & set(label_role_person))
        lab_num += len(set(label_role_person))
        pre_num += len(set(role_person))
        if set(role_person)&set(label_role_person) != set(label_role_person):
            person_errors.append([docid, set(label_role_person), set(role_person)])
            # 判断角色联系人是否正确逻辑：1、先看预测角色是否都在标签角色里，2判断预测联系人是否在标签联系人，
            # print(set(role_person))
            # print(set(label_role_person))
        if set(label_person_phone) & set(person_phone)!=set(label_person_phone):
            phone_errors.append([docid, set(label_person_phone), set(person_phone)])
        pos_num2 += len(set(label_person_phone) & set(person_phone))
        lab_num2 += len(set(label_person_phone))
        pre_num2 += len(set(person_phone))

        lab_person += len(set([it.split('+')[1] for it in label_role_person]))
        pos_person += len(set([it.split('+')[1] for it in label_role_person])&set([it.split('+')[1] for it in role_person]))
        pre_person += len(set([it.split('+')[1] for it in role_person]))

        lab_role += len(set([it.split('+')[0] for it in label_role_person]))
        pos_role += len(set([it.split('+')[0] for it in label_role_person])&set([it.split('+')[0] for it in role_person]))
        pre_role += len(set([it.split('+')[0] for it in role_person]))

        if set([it.split('+')[0] for it in label_role_person]) != set([it.split('+')[0] for it in role_person]):
            if set([it.split('+')[1] for it in label_role_person]) != set([it.split('+')[1] for it in role_person]):
                person_name_errors.append([docid,set(label_role_person), set(role_person)])
            else:
                role_name_errors.append([docid, set(label_role_person), set(role_person)])
        else:
            if set([it.split('+')[1] for it in label_role_person]) != set([it.split('+')[1] for it in role_person]):
                person_name_errors.append([docid, set(label_role_person), set(role_person)])
            elif set(label_role_person)!= set(role_person):
                print(docid,set(label_role_person), set(role_person))
                join_errors.append([docid,set(label_role_person), set(role_person)])
    print('单独角色召回率：%.4f，准确率：%.4f'%(pos_role/lab_role, pos_role/pre_role))
    print('单独联系人召回率：%.4f, 准确率：%.4f'%(pos_person/lab_person, pos_person/pre_person))
    print('联系人召回率：%.4f, 准确率：%.4f' % (pos_num / lab_num, pos_num / pre_num))
    print('电话召回率：%.4f，准确率：%.4f' % (pos_num2 / lab_num2, pos_num2 / pre_num2))
    print('总耗时：',time.time()-t1)
    return person_errors, phone_errors, join_errors, role_name_errors, person_name_errors

def predict_fromdb(docid, dbname="sys_document_23"):
    # import pymysql
    # conn = pymysql.Connect(host='rm-bp1quo50k0q2ok73gi.mysql.rds.aliyuncs.com', port=3306, db='bxkc', user='bxkc_read', passwd='bxkc_20RE18AD') #新账号密码
    # cursor = conn.cursor()
    # sql = "SELECT  docid as id, dochtmlcon as content  from {1} WHERE DOCID='{0}';".format(docid, dbname)
    import psycopg2
    conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    cursor = conn.cursor()
    sql = """select human_identifier as id,sourcetext as content from corpus_iedocument where human_identifier in ('{0}');""".format(docid)
    cursor.execute(sql)
    rows = cursor.fetchall()
    doc_id = rows[0][0]
    text = rows[0][1]
    # text = '竟然很明显的表达没识别为代理，代理机构名称：国信国采(北京)招标咨询有限责任公司，代理机构地址：北京市海淀区首体南路22号国兴大厦11层，  1.采购人信息名 称：北京市植物园。'
    list_articles, list_sentences, list_entitys, _ = Preprocessing.get_preprocessed([[doc_id, text, "", "", ""]],useselffool=True)
    codeName = codeNamePredict.predict(list_sentences, list_entitys=list_entitys)
    # print(codeName)
    premPredict.predict(list_sentences, list_entitys)
    roleRulePredict.predict(list_articles, list_sentences, list_entitys, codeName)
    # print("epcPredict")
    epcPredict.predict(list_sentences, list_entitys)
    # print("entityLink")
    entityLink.link_entitys(list_entitys)
    # print("getPREMs")
    prem = getAttributes.getPREMs(list_sentences, list_entitys, list_articles)
    return list_articles, list_sentences, list_entitys, codeName, prem

if __name__=="__main__":
    # import pandas as pd
    # import math
    # import pymysql
    # conn = pymysql.Connect(host='rm-bp1quo50k0q2ok73gi.mysql.rds.aliyuncs.com', port=3306, db='bxkc', user='bxkc_read', passwd='bxkc_20RE18AD') #新账号密码
    # cursor = conn.cursor()
    # df = pd.read_excel('G:/大网站规则识别/1027统计入库top100编号.xlsx')
    # docs_list = []
    # for i in range(100):
    #     web_no = df.loc[i, '编号']
    #     # num = math.ceil(int(df.loc[i, '1019-1023入库公告数量']) * 0.01)
    #     num = 10
    #     sql = "SELECT DOCID,DOCCHANNEL,DOCHTMLCON,WEB_SOURCE_NO from sys_document_23 where WEB_SOURCE_NO='{0}' and DOCCHANNEL='101' and DOCID%9=1 limit {1}".format(
    #         web_no, num)
    #     #  rows = cursor.execute(sql) 此处代码错误 rows 需要用 cursor.fetchall方法获取
    #     cursor.execute(sql)
    #     rows = cursor.fetchall()
    #     docs_list.extend(list(rows))
    # df_doc = pd.DataFrame(docs_list, columns=['docid', 'channel', 'html', 'web_no'])
    # codenames = []
    # prems = []
    # for docid,text in zip(df_doc['docid'], df_doc['html']):
    #     list_articles, list_sentences, list_entitys, _ = Preprocessing.get_preprocessed([[docid, text, "", "", ""]],
    #                                                                                     useselffool=True)
    #     codeName = codeNamePredict.predict(list_sentences, list_entitys=list_entitys)
    #     # print(codeName)
    #     premPredict.predict(list_sentences, list_entitys)
    #     roleRulePredict.predict(list_articles, list_sentences, list_entitys, codeName)
    #     # print("epcPredict")
    #     epcPredict.predict(list_sentences, list_entitys)
    #     # print("entityLink")
    #     entityLink.link_entitys(list_entitys)
    #     # print("getPREMs")
    #     prem = getAttributes.getPREMs(list_sentences, list_entitys, list_articles)
    #     if codeName:
    #         codenames.append(codeName[0][1])
    #     else:
    #         codenames.append(" ")
    #     if prem:
    #         prems.append(prem[0][1])
    #     else:
    #         prems.append(" ")
    # df_doc['codename'] = pd.Series(codenames)
    # df_doc['prem'] = pd.Series(prems)
    # df_doc.to_excel('G:/大网站规则识别/大网站规则调整后预测结果20201124.xlsx', columns=['docid', 'channel', 'html', 'prem', 'codename', 'web_no'])


    # list_articles, list_sentences, list_entitys, codeName, prem = predict_fromdb('82862794',dbname="sys_document_25")  #sys_document_23
    # print(prem)
    # print(codeName)
    # entitys_all = [[[entity.entity_text,entity.entity_type,entity.label,entity.values,entity.sentence_index,entity.begin_index,entity.end_index] for entity in entitys] for entitys in list_entitys]
    # for entitys in entitys_all:
    #     # print(entitys)
    #     # en_types = set([it[1] for it in entitys])
    #     print([(it[0],it[1], it[2],it[3][it[2]],it[4],it[5],it[6]) for it in entitys if it[1] in ('org', 'company', 'person')])
    # print(list_articles[0].content)

    # print(get_result('100000203'))

    # person_errors, phone_errors, join_errors, role_name_errors, person_name_errors = analys_person_phone()
    # import pickle
    # with open('phone_errors.pkl','wb') as f:
    #     pickle.dump(phone_errors, f)

    # filename = "比地_52_79929693.html"
    # #text = codecs.open("C:\\Users\\User\\Desktop\\数据20191014\\"+filename,"r",encoding="utf8").read()
    # # text = codecs.open("C:\\Users\\User\\Desktop\\2.html","r",encoding="utf8").read()
    # # text = codecs.open('F:/工作文档/实体识别实体对其/20190320/比地_101_58511386.html', encoding='utf-8').read()
    # docid = '100000203'
    # r1 = get_result(docid)
    # r2 = get_result_online(docid)
    # rolperson = []
    # person_phone = []
    # for v in r1['prem'].values():
    #     roleList = v['roleList']
    #     for role in roleList:
    #         for it in role[3]:
    #             rolperson.append(role[1] + it[0])
    #     for role in roleList:
    #         for it in role[3]:
    #             person_phone.append(it[0]+it[1])
    # print(r1['prem'])
    # print(r2['prem'])
    #
    # import psycopg2
    # conn = psycopg2.connect(dbname='iepy', user='postgres', password='postgres', host='192.168.2.101')
    # cursor = conn.cursor()
    # sql = """select human_identifier,sourcetext from corpus_iedocument where human_identifier in ('95008163');"""
    # cursor.execute(sql)
    # rows = cursor.fetchall()
    # # print(len(rows), rows)
    # content = rows[0][1]
    # content = str(BeautifulSoup(text).find("div",id="pcontent"))
    # # content = text
    # # print('content: ',content)
    # #text = codecs.open("C:\\Users\\User\\Desktop\\a.html","r",encoding="utf8").read()
    # text = '''
    # 招标人：常州市金坛区直溪镇迪庄村村民委员会，地址：直溪镇迪庄村，电话：18798916198电话：17798916098，
    # '''
    text = '''
招标人：湖南财信金融控股集团有限公司，招标代理机构：湖南省招标有限责任公司。联系人：曾君联系人：夏女士，电话：0731－84556896电话：0731-85196661
    '''
    # text='''，东莞市粤隆招标有限公司受东莞市道滘镇教育管理中心的委托，于2020年05月20日就2020年道滘镇公办中小学增加多媒体教学平台
    # 采购项目(441900-10-202003-1606000137-0006)采用公开招标进行采购。现就本次采购的中标(成交)结果公告如下：
    # 一、采购项目编号：441900-10-202003-1606000137-0006二、采购项目名称：2020年道滘镇公办中小学增加多媒体教学平台采购项目三、
    # 采购项目预算金额(元)：3,505,265.00四、采购方式：公开招标五、中标供应商，1：A包中标供应商名称东莞市靖恩智能科技有限公司法人代表王地址
    # 东莞市高埗镇高埗大道93号301房，六、报价明细，备注。主要中标、成交标的名称：2020年道滘镇公办中小学增加多媒体教学平台采购项目，
    # 中标供应商：东莞市靖恩智能科技有限公司，包编号：A，规格型号：\，数量：1，预算(元):3,505,265.00，服务要求：详见招标文件，
    # 中标、成交金额(元):3,478,120.00。报价明细附件七、评审日期：2020-05-20评审地点：广东省东莞市道滘镇道厚路创新岛2号四楼，评审委员会
    # (谈判小组、询价小组、磋商小组或单一来源采购小组)：负责人：廖云峰，成员：叶润宁、莫立基、李松涛和吴树根，八、本项目代理收费标准：国家发展
    # 和改革委员会“发改价格[2011]534号文”及国家发展计划委员会“计价格[2002]1980号文”相关规定，收费金额：人民币肆万贰仟贰佰伍拾玖元整(￥42,259.00元)九、
    # 评审意见(非标采购方式或竞争性磋商采购方式采用书面推荐供应商参加采购活动的，还应当公告采购人和评审专家的推荐意见)，经评标委员会一致决定，
    # 推荐2020年道滘镇公办中小学增加多媒体教学平台采购项目为第一中标候选人，十、本公告期限1个工作日。十一、联系事项：
    # (一)，采购项目联系人(代理机构)：翟先生，联系电话：0769-88839111，采购项目联系人(采购人)：李先生，联系电话：0769-81332303。
    # (二)，采购代理机构：东莞市粤隆招标有限公司，地址：东莞市南城街道体育路2号鸿禧中心B902号，联系人：翟先生，联系电话：0769-88839111，
    # 传真：0769-81216222，邮编：523000。(三)，采购人：东莞市道滘镇教育管理中心，地址：广东省东莞市道滘镇花园街1号，联系人：李先生，联系电话：0769-81332303，传真：/，邮编：523000，各有关当事人对中标、成交结果有异议的，可以在中标、成交公告发布之日起7个工作日内以书面形式向(政府采购代理机构)(或采购人)提出质疑，逾期将依法不予受理，'''
    text = codecs.open("C:\\Users\\Administrator\\Desktop\\test12354.txt", "r", encoding="utf8").read()
    content = str(BeautifulSoup(text).find("div", id="pcontent"))
    from BiddingKG.dl.interface.Preprocessing import tableToText
    # print("tableToText:",tableToText(BeautifulSoup(re.sub("<html>|</html>|<body>|</body>","",content),"lxml")))
#     text = '''
# 采购代理机构：山东立行建设项目管理有限公司地址：山东省临沂市兰山县(区)柳青广州路与蒙河路交汇大官苑社区西沿街A区三楼南侧号，联系方式：17862288900，
# '''
#     text = '''
# (二)，采购人信息，名称：四川省绵阳市平武县卫生健康局，地址：东莞市南城街道体育路2号鸿禧中心B902号，(三)，采购代理机构信息，名称：四川省蜀汉硕创工程项目管理有限公司，地址：体育路2号鸿禧中心B，联系人：翟先生，联系电话：0769-88839111，
# 采购项目,联系电话：0769-81332303/0769-81332903,联系人姓名和电话：李先生/廖云峰\叶润宁，
# '''
    a = time.time()
    print("start")
    # print(predict("12",content))
    result = predict("12",text)
    # result = predict("12",content)
    # print(json.loads(result))
    #test("12",text)
    print("takes",time.time()-a)
    _time2 = time.time()
    # print(predict("12",content))
    _time3 = time.time()
    print("init takes:%d"%((_time2-_time1)-(_time3-_time2)))
    pass