BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/metrics/extractMetric.py

import psycopg2
from BiddingKG.dl.interface.extract import predict,test
from BiddingKG.dl.common.Utils import getUnifyMoney,timeFormat
from BiddingKG.dl.entityLink.entityLink import jaccard_score
import re
import json


bidway_dict = {'询价': '询价', '竞争性谈判': '竞争性谈判',
               '公开比选': '其他', '国内竞争性磋商': '竞争性磋商',
               '招标方式：t公开': '公开招标', '竞价': '竞价',
               '竞标': '竞价', '电子竞价': '竞价',
               '电子书面竞投': '竞价', '单一来源': '单一来源',
               '网上竞价': '竞价', '公开招标': '公开招标',
               '询比': '询价', '定点采购': '其他',
               '招标方式：■公开': '公开招标', '交易其他，付款其他': '其他',
               '竞争性评审': '竞争性磋商', '公开招租': '其他', '\\N': '',
               '比选': '其他', '比质比价': '其他', '分散采购': '其他',
               '内部邀标': '邀请招标', '邀请招标': '邀请招标',
               '网上招标': '公开招标', '非定向询价': '询价',
               '网络竞价': '竞价', '公开询价': '询价',
               '定点采购议价': '其他', '询单': '询价',
               '网上挂牌': '其他', '网上直购': '其他',
               '定向询价': '询价', '采购方式：公开': '公开招标',
               '磋商': '竞争性磋商', '公开招投标': '公开招标',
               '招标方式：√公开': '公开招标', '公开选取': '公开招标',
               '网上电子投标': '公开招标', '公开竞谈': '竞争性谈判',
               '竞争性磋商': '竞争性磋商', '采购方式：邀请': '邀请招标',
               '公开竞价': '竞价', '其他': '其他', '公开招募': '其他',
               '网上询价': '询价'}
# bidway名称统一规范
def bidway_integrate(bidway):
    integrate_name = bidway_dict.get(bidway,"其他")
    return integrate_name

class ExtractMetric():

    def __init__(self):
        self.conn1 = self.getConnection_postgres("iepy")
        self.conn2 = self.getConnection_postgres("iepy")

    def fitDataByRule(self,data):
        symbol_dict = {"(":")",
                       "（":"）",
                       "[":"]",
                       "【":"】",
                       ")":"(",
                       "）":"（",
                       "]":"[",
                       "】":"【"}
        leftSymbol_pattern = re.compile("[\(（\[【]")
        rightSymbol_pattern = re.compile("[\)）\]】]")
        leftfinds = re.findall(leftSymbol_pattern,data)
        rightfinds = re.findall(rightSymbol_pattern,data)
        result = data
        if len(leftfinds)+len(rightfinds)==0:
            return data
        elif len(leftfinds)==len(rightfinds):
            return data
        elif abs(len(leftfinds)-len(rightfinds))==1:
            if len(leftfinds)>len(rightfinds):
                if symbol_dict.get(data[0]) is not None:
                    result = data[1:]
                else:
                    #print(symbol_dict.get(leftfinds[0]))
                    result = data+symbol_dict.get(leftfinds[0])
            else:
                if symbol_dict.get(data[-1]) is not None:
                    result = data[:-1]
                else:
                    result = symbol_dict.get(rightfinds[0])+data
        return  result


    def getConnection_postgres(self,db):
        conn = psycopg2.connect(dbname=db,user="postgres",password="postgres",host="192.168.2.103")
        return conn

    def label2interface(self,list_anno,Htext):
        dict_result = {}
        dict_anno = {}
        for _anno in list_anno:
            value = _anno["value"]
            _split = value.split("\t")
            if _split[0][0]=="T":
                _type,_begin,_end = _split[1].split(" ")
                dict_anno[_split[0]] = {"id":_split[0],"type":_type,"text":_split[2],"begin":int(_begin),"end":int(_end)}
            elif _split[0][0]=="R":
                _type,arg1,arg2 = _split[1].split(" ")
                dict_anno[_split[0]] = {"id":_split[0],"type":_type,"arg1":arg1.split(":")[1],"arg2":arg2.split(":")[1]}
        dict_role = {}
        dict_money = {}
        dict_person2role = {}
        dict_name_freq_score = {}
        pattern_score = re.compile("工程|服务|采购|施工|项目|系统|招标|中标|公告|学校|[大中小]学校?|医院|公司|分公司|研究院|政府采购中心|学院|中心校?|办公室|政府|财[政务]局|办事处|委员会|[部总支]队|警卫局|幼儿园|党委|党校|银行|分行|解放军|发电厂|供电局|管理所|供电公司|卷烟厂|机务段|研究[院所]|油厂|调查局|调查中心|出版社|电视台|监狱|水厂|服务站|信用合作联社|信用社|交易所|交易中心|交易中心党校|科学院|测绘所|运输厅|管理处|局|中心|机关|部门?|处|科|厂|集团|图书馆|馆|所|厅|楼|区|酒店|场|基地|矿|餐厅|酒店")
        for k,v in dict_anno.items():
            if v.get("type") in ["code","product","person_review"]:
                if v.get("type") not in dict_result:
                    dict_result[v.get("type")] = []
                dict_result[v.get("type")].append(v.get("text"))
                dict_result[v.get("type")] = list(set(dict_result[v.get("type")]))
            if v.get("type") in ["name","bidway","moneysource","serviceTime","time_release","time_bidopen","time_bidclose"]:
                if v.get("type")=="name":
                    _name = self.fitDataByRule(v.get("text"))
                    w = 1 if re.search('(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]', _name)!=None else 0.5
                    if _name not in dict_name_freq_score:
                        # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
                        dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05)*w]
                    else:
                        dict_name_freq_score[_name][0] += 1
                    max_score = 0
                    for _k1,_v1 in dict_name_freq_score.items():
                        if _v1[0]*_v1[1]>max_score:
                            max_score = _v1[0]*_v1[1]
                            dict_result[v.get("type")] = _k1
                if v.get("type") not in dict_result:
                    if v.get("type") in ["time_release","time_bidopen","time_bidclose"]:
                        _t = timeFormat(v.get("text"))
                    else:
                        _t = v.get("text")
                    dict_result[v.get("type")] = _t
                if v.get("type")=="bidway":
                    dict_result[v.get("type")] = bidway_integrate(v.get("text"))
            _split = v.get("type").split("_")
            if len(_split)>1:
                if _split[1]=="tenderee":
                    dict_role["tenderee"] = {"subject":v.get("text")}
                if _split[1]=="agency":
                    dict_role["agency"] = {"subject":v.get("text")}
                if _split[1]=="tenderer":
                    dict_role["tenderer"] = {"subject":v.get("text")}
                if _split[1]=="secondTenderer":
                    dict_role["secondTenderer"] = {"subject":v.get("text")}
                if _split[1]=="thirdTenderer":
                    dict_role["thirdTenderer"] = {"subject":v.get("text")}
        tendereeMoney = 0
        for k,v in dict_anno.items():
            _split = v.get("type").split("_")
            if v.get("type") in ["money_tendereeMoney"]:
                _before_text = Htext[max(v["begin"]-10,0):v["begin"]]
                if re.search('总投资|投资总额|总预算|总概算|投资规模|投资|工程造价', _before_text):
                    continue
                if re.search("万",_before_text) is not None and re.search("整",_before_text) is None:
                    _unit = 10000
                else:
                    _unit = 1
                tendereeMoney = float(getUnifyMoney(v["text"])*_unit)
            if v.get("type") in ["rel_tendereeMoney","rel_tendererMoney"]:
                arg1 = v.get("arg1")
                arg2 = v.get("arg2")
                for _k,_v in dict_role.items():
                    if _v["subject"]==dict_anno[arg1]["text"]:
                        _before_text = Htext[max(dict_anno[arg2]["begin"]-10,0):dict_anno[arg2]["begin"]]
                        if re.search('总投资|投资总额|总预算|总概算|投资规模|投资|工程造价', _before_text):
                            continue
                        if re.search("万",_before_text) is not None and re.search("整",_before_text) is None:
                            _unit = 10000
                        else:
                            _unit = 1
                        _v["money"] = float(getUnifyMoney(dict_anno[arg2]["text"])*_unit)
            if v.get("type")=="person_tendereePerson":
                if "tenderee" in dict_role:
                    if "person" not in dict_role["tenderee"]:
                        dict_role["tenderee"]["person"] = []
                    dict_role["tenderee"]["person"].append({"person":v["text"]})
            if v.get("type")=="person_agencyPerson":
                if "agency" in dict_role:
                    if "person" not in dict_role["agency"]:
                        dict_role["agency"]["person"] = []
                    dict_role["agency"]["person"].append({"person":v["text"]})
            if v.get("type")=="rel_person":
                arg1 = v.get("arg1")
                arg2 = v.get("arg2")
                for _k,_v in dict_role.items():
                    if _v["subject"]==dict_anno[arg1]["text"]:
                        if "person" not in dict_role[_k]:
                            dict_role[_k]["person"] = []
                        dict_role[_k]["person"].append({"person":dict_anno[arg2]["text"]})
                        dict_person2role[dict_anno[arg2]["text"]] = _k
        for k,v in dict_anno.items():
            if v.get("type")=="rel_phone":
                arg1 = v.get("arg1")
                arg2 = v.get("arg2")
                _person = dict_anno[arg1]["text"]
                if _person in dict_person2role:
                    for item in dict_role[dict_person2role[_person]]["person"]:
                        if item["person"]==_person:
                            item["phone"] = dict_anno[arg2]["text"]


        roleList = []
        for k,v in dict_role.items():
            if k=="tenderee":
                _role = "tenderee"
            if k=="agency":
                _role = "agency"
            if k=="tenderer":
                _role = "win_tenderer"
            if k=="secondTenderer":
                _role = "second_tenderer"
            if k=="thirdTenderer":
                _role = "third_tenderer"
            list_person = []
            set_person = set()
            for item in v.get("person",[]):
                if item["person"] not in set_person:
                    list_person.append([item["person"],item.get("phone","")])
                    set_person.add(item["person"])
            roleList.append([_role,v.get("subject","").replace("(","（").replace(")","）"),v.get("money",0),list_person,""])

        dict_result["prem"] = {"Project":{"roleList":roleList,"tendereeMoney":tendereeMoney}}
        return dict_result


    def culExtractMetrics(self):
        conn = self.conn1
        cursor = conn.cursor()
        sql = ' select begin_time,end_time,"user",doc_count from corpus_payroll where end_time<=\'2021-07-25\' order by end_time desc limit 20'
        cursor.execute(sql)
        list_diff = []
        rows_payroll = cursor.fetchall()
        for _payroll in rows_payroll:
            _begin_time = _payroll[0]
            _end_time = _payroll[1]
            _user = _payroll[2]
            doc_count = _payroll[3]
            print(_user,_begin_time,_end_time,doc_count)
            _sql = "select document_id,value from brat_bratannotation where document_id in (select human_identifier from corpus_iedocument where edituser='%s' and to_char(edittime,'yyyy-mm-dd')>='%s' and to_char(edittime,'yyyy-mm-dd')<='%s' limit 100)  order by document_id"%(_user,_begin_time,_end_time)
            cursor.execute(_sql)
            rows = cursor.fetchall()
            if len(rows)>0:
                current_docid = rows[0][0]
                _index = -1
                list_values = []
                while _index<len(rows)-1:
                    _index += 1
                    row = rows[_index]
                    document_id = row[0]
                    value = row[1]
                    if document_id!=current_docid:
                        print(current_docid)
                        sql = "select text from corpus_iedocument where human_identifier='%s'"%(str(current_docid))
                        cursor.execute(sql)
                        content = cursor.fetchall()[0][0]
                        _inter = self.label2interface(list_values,content)
                        _inter2 = self.extractFromInterface(content)
                        if not len(_inter2.get("prem").keys())>1:
                            _diff = self.getDiff(_inter,_inter2)
                            list_diff.append(_diff)
                        _index -= 1
                        current_docid = document_id
                        list_values = []
                    else:
                        list_values.append({"document_id":document_id,"value":value})
        metrics = self.getMetrics(list_diff)
        print(metrics)

    def extractFromInterface(self,content):
        _json = test("",content)
        return json.loads(_json)

    def getDiff(self,_inter,_inter2):
        _dict = {}
        for k in ["code","product","person_review"]:
            set_k1 = _inter.get(k,set())
            set_k2 = _inter2.get(k,set())
            _dict["%s_inter"%k] = len(set_k1)
            _dict["%s_inter2"%k] = len(set_k2)
            _dict["%s_union"%k] = len(set(set_k1)&set(set_k2))

        for k in ["name","bidway","moneysource","serviceTime","time_release","time_bidopen","time_bidclose"]:
            _k1 = _inter.get(k,"")
            _k2 = _inter2.get(k,"")
            len_k1 = 0 if _k1=="" else 1
            len_k2 = 0 if _k2=="" else 1
            if k in ["name","serviceTime"]:
                _score = jaccard_score(_k1,_k2)
                if len_k1 and len_k2 and _score>0.9:
                    len_union = 1
                else:
                    len_union = 0
            else:
                len_union = 1 if _k1==_k2 and len_k1==1 else 0
            _dict["%s_inter"%k] = len_k1
            _dict["%s_inter2"%k] = len_k2
            _dict["%s_union"%k] = len_union


        dict_project = {}
        for k,v in _inter.get("prem",{}).items():
            if float(v.get("tendereeMoney",0))>0:
                dict_project["%s_inter"%("tendereeMoney")] = [float(v.get("tendereeMoney"))]
            for _role in v.get("roleList",[]):
                dict_project["%s_inter"%_role[0]] = [_role[1]]
                if _role[0] in ["win_tenderer","second_tenderer","third_tenderer"]:
                    if float(_role[2])>0:
                        dict_project["%s_money_inter"%_role[0]] = [float(_role[2])]
                for item in _role[3]:
                    _person = item[0]
                    _phone = item[1]
                    if _person=="" or _phone=="":
                        continue
                    if "%s_person_inter"%_role[0] not in dict_project:
                        dict_project["%s_person_inter"%_role[0]] = []
                    dict_project["%s_person_inter"%_role[0]].append("%s-%s"%(_role[1],_person))
                    if "person_phone_inter" not in dict_project:
                        dict_project["person_phone_inter"] = []
                    dict_project["person_phone_inter"].append("%s-%s"%(_person,_phone))
        for k,v in _inter2.get("prem",{}).items():
            if float(v.get("tendereeMoney",0))>0:
                dict_project["%s_inter2"%("tendereeMoney")] = [float(v.get("tendereeMoney"))]
            for _role in v.get("roleList",[]):
                dict_project["%s_inter2"%_role.get("role_type")] = [_role.get("role_text")]
                if _role.get("role_type") in ["win_tenderer","second_tenderer","third_tenderer"]:
                    if float(_role.get("role_money").get("money",0))>0:
                        dict_project["%s_money_inter2"%_role.get("role_type")] = [float(_role.get("role_money").get("money",0))]
                for item in _role.get("linklist"):
                    _person = item[0]
                    _phone = item[1]
                    if _person=="" or _phone=="":
                        continue
                    if "%s_person_inter2"%_role.get("role_type") not in dict_project:
                        dict_project["%s_person_inter2"%_role.get("role_type")] = []
                    dict_project["%s_person_inter2"%_role.get("role_type")].append("%s-%s"%(_role.get("role_text"),_person))
                    if "person_phone_inter2" not in dict_project:
                        dict_project["person_phone_inter2"] = []
                    dict_project["person_phone_inter2"].append("%s-%s"%(_person,_phone))
        set_k = set()
        for k,v in dict_project.items():
            k_split = k.split("_")
            base_key = "_".join(k_split[:-1])
            if k_split[-1]=="inter":
                k2 = "inter2"
            else:
                k2 = "inter"
            if base_key in set_k:
                continue
            k_other = "%s_%s"%(base_key,k2)
            _dict[k] = len(v)
            _dict[k_other] = len(dict_project.get(k_other,[]))
            if base_key=="tenderee":
                _dict["%s_union"%base_key] = 0
                if _dict[k]>0 and _dict[k_other]>0:
                    _score = jaccard_score(dict_project.get(k),dict_project.get(k_other))
                    if _score>0.9:
                        _dict["%s_union"%base_key] = 1
                else:
                    #通过规则召回的也算
                    if dict_project.get("%s_%s"%(base_key,"inter")) is None and dict_project.get("%s_%s"%(base_key,"inter2")) is not None:
                        _dict[k] = 1
                        _dict[k_other] = 1
                        _dict["%s_union"%base_key] = 1

            else:
                _dict["%s_union"%base_key] = len(set(v)&set(dict_project.get(k_other,[])))
            set_k.add(base_key)
        print("=========================")
        print(_inter)
        print("-----")
        print(_inter2)
        print("|||||")
        print(_dict)
        return _dict



    def getMetrics(self,list_diff):
        dict_key_count = {}
        # print("all_count:",list_diff)
        for _diff in list_diff:
            for k,v in _diff.items():
                if k not in dict_key_count:
                    dict_key_count[k] = 0
                dict_key_count[k] += v
        set_k = set()
        for k,v in dict_key_count.items():
            k_split = k.split("_")
            base_k = "_".join(k_split[:-1])
            if base_k in set_k:
                continue
            set_k.add(base_k)
            _count_inter = max(dict_key_count.get("%s_inter"%base_k,-1),1)
            _count_inter2 = max(dict_key_count.get("%s_inter2"%base_k,-1),1)
            _count_union = dict_key_count.get("%s_union"%base_k,0)
            _precision = _count_union/_count_inter2
            _recall = _count_union/_count_inter
            _f1 = 2*(_precision*_recall)/(_precision+_recall)
            print("%s: recall:%.3f,precision:%.3f,f1_score:%.3f"%(base_k,_recall,_precision,_f1))
            print(base_k)
            print("%.3f"%_f1)
            print("%.3f"%_precision)
            print("%.3f"%_recall)



if __name__=="__main__":
    em = ExtractMetric()
    em.culExtractMetrics()