各要素指标

.idea/sonarlint-state.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SonarLintProjectState">
+    <lastEventPolling>1631004326176</lastEventPolling>
+  </component>
+</project>

.idea/sonarlint.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SonarLintProjectSettings">
+    <option name="bindingEnabled" value="true" />
+    <option name="projectKey" value="BiddingKG" />
+    <option name="serverId" value="sonarqube" />
+  </component>
+</project>

.scannerwork/report-task.txt

@@ -0,0 +1,6 @@
+projectKey=BiddingKG
+serverUrl=http://192.168.2.102:9000
+serverVersion=9.0.1.46107
+dashboardUrl=http://192.168.2.102:9000/dashboard?id=BiddingKG
+ceTaskId=AXu_QxDamj0G_RX-Yx74
+ceTaskUrl=http://192.168.2.102:9000/api/ce/task?id=AXu_QxDamj0G_RX-Yx74

BiddingKG/dl/interface/extract.py

@@ -104,14 +104,14 @@ def predict(doc_id,text,title=""):
     data_res["success"] = True
 
 
-    for _article in list_articles:
-        log(_article.content)
-
-    for list_entity in list_entitys:
-        for _entity in list_entity:
-            log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
-                  (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
-                   str(_entity.begin_index),str(_entity.end_index)))
+    # for _article in list_articles:
+    #     log(_article.content)
+    #
+    # for list_entity in list_entitys:
+    #     for _entity in list_entity:
+    #         log("type:%s,text:%s,label:%s,values:%s,sentence:%s,begin_index:%s,end_index:%s"%
+    #               (str(_entity.entity_type),str(_entity.entity_text),str(_entity.label),str(_entity.values),str(_entity.sentence_index),
+    #                str(_entity.begin_index),str(_entity.end_index)))
 
     return json.dumps(data_res,cls=MyEncoder,sort_keys=True,indent=4,ensure_ascii=False)
 
@@ -122,9 +122,9 @@ def test(name,content):
         "id":name
     }
     myheaders = {'Content-Type': 'application/json'}
-    _resp = requests.post("http://192.168.2.101:15015" + '/article_extract', json=user, headers=myheaders, verify=True)
+    _resp = requests.post("http://192.168.2.102:15030" + '/article_extract', json=user, headers=myheaders, verify=True)
     resp_json = _resp.content.decode("utf-8")
-    print(resp_json)
+    # print(resp_json)
     return resp_json
 
 

BiddingKG/dl/metrics/extractMetric.py

@@ -0,0 +1,343 @@
+
+import psycopg2
+from BiddingKG.dl.interface.extract import predict,test
+from BiddingKG.dl.common.Utils import getUnifyMoney,timeFormat
+import re
+import json
+
+class ExtractMetric():
+
+    def __init__(self):
+        self.conn1 = self.getConnection_postgres("iepy")
+        self.conn2 = self.getConnection_postgres("iepy")
+
+    def fitDataByRule(self,data):
+        symbol_dict = {"(":")",
+                       "（":"）",
+                       "[":"]",
+                       "【":"】",
+                       ")":"(",
+                       "）":"（",
+                       "]":"[",
+                       "】":"【"}
+        leftSymbol_pattern = re.compile("[\(（\[【]")
+        rightSymbol_pattern = re.compile("[\)）\]】]")
+        leftfinds = re.findall(leftSymbol_pattern,data)
+        rightfinds = re.findall(rightSymbol_pattern,data)
+        result = data
+        if len(leftfinds)+len(rightfinds)==0:
+            return data
+        elif len(leftfinds)==len(rightfinds):
+            return data
+        elif abs(len(leftfinds)-len(rightfinds))==1:
+            if len(leftfinds)>len(rightfinds):
+                if symbol_dict.get(data[0]) is not None:
+                    result = data[1:]
+                else:
+                    #print(symbol_dict.get(leftfinds[0]))
+                    result = data+symbol_dict.get(leftfinds[0])
+            else:
+                if symbol_dict.get(data[-1]) is not None:
+                    result = data[:-1]
+                else:
+                    result = symbol_dict.get(rightfinds[0])+data
+        return  result
+
+
+    def getConnection_postgres(self,db):
+        conn = psycopg2.connect(dbname=db,user="postgres",password="postgres",host="192.168.2.103")
+        return conn
+
+    def label2interface(self,list_anno,Htext):
+        dict_result = {}
+        dict_anno = {}
+        for _anno in list_anno:
+            value = _anno["value"]
+            _split = value.split("\t")
+            if _split[0][0]=="T":
+                _type,_begin,_end = _split[1].split(" ")
+                dict_anno[_split[0]] = {"id":_split[0],"type":_type,"text":_split[2],"begin":int(_begin),"end":int(_end)}
+            elif _split[0][0]=="R":
+                _type,arg1,arg2 = _split[1].split(" ")
+                dict_anno[_split[0]] = {"id":_split[0],"type":_type,"arg1":arg1.split(":")[1],"arg2":arg2.split(":")[1]}
+        dict_role = {}
+        dict_money = {}
+        dict_person2role = {}
+        dict_name_freq_score = {}
+        pattern_score = re.compile("工程|服务|采购|施工|项目|系统|招标|中标|公告|学校|[大中小]学校?|医院|公司|分公司|研究院|政府采购中心|学院|中心校?|办公室|政府|财[政务]局|办事处|委员会|[部总支]队|警卫局|幼儿园|党委|党校|银行|分行|解放军|发电厂|供电局|管理所|供电公司|卷烟厂|机务段|研究[院所]|油厂|调查局|调查中心|出版社|电视台|监狱|水厂|服务站|信用合作联社|信用社|交易所|交易中心|交易中心党校|科学院|测绘所|运输厅|管理处|局|中心|机关|部门?|处|科|厂|集团|图书馆|馆|所|厅|楼|区|酒店|场|基地|矿|餐厅|酒店")
+        for k,v in dict_anno.items():
+            if v.get("type") in ["code","product","person_review"]:
+                if v.get("type") not in dict_result:
+                    dict_result[v.get("type")] = []
+                dict_result[v.get("type")].append(v.get("text"))
+                dict_result[v.get("type")] = list(set(dict_result[v.get("type")]))
+            if v.get("type") in ["name","bidway","moneysource","serviceTime","time_release","time_bidopen","time_bidclose"]:
+                if v.get("type")=="name":
+                    _name = self.fitDataByRule(v.get("text"))
+                    w = 1 if re.search('(项目|工程|招标|合同|标项|标的|计划|询价|询价单|询价通知书|申购)(名称|标题|主题)[:：\s]', _name)!=None else 0.5
+                    if _name not in dict_name_freq_score:
+                        # dict_name_freq_score[_name] = [1,len(re.findall(pattern_score,_name))+len(_name)*0.1]
+                        dict_name_freq_score[_name] = [1, (len(re.findall(pattern_score, _name)) + len(_name) * 0.05)*w]
+                    else:
+                        dict_name_freq_score[_name][0] += 1
+                    max_score = 0
+                    for _k1,_v1 in dict_name_freq_score.items():
+                        if _v1[0]*_v1[1]>max_score:
+                            max_score = _v1[0]*_v1[1]
+                            dict_result[v.get("type")] = _k1
+                if v.get("type") not in dict_result:
+                    if v.get("type") in ["time_release","time_bidopen","time_bidclose"]:
+                        _t = timeFormat(v.get("text"))
+                    else:
+                        _t = v.get("text")
+                    dict_result[v.get("type")] = _t
+            _split = v.get("type").split("_")
+            if len(_split)>1:
+                if _split[1]=="tenderee":
+                    dict_role["tenderee"] = {"subject":v.get("text")}
+                if _split[1]=="agency":
+                    dict_role["agency"] = {"subject":v.get("text")}
+                if _split[1]=="tenderer":
+                    dict_role["tenderer"] = {"subject":v.get("text")}
+                if _split[1]=="secondTenderer":
+                    dict_role["secondTenderer"] = {"subject":v.get("text")}
+                if _split[1]=="thirdTenderer":
+                    dict_role["thirdTenderer"] = {"subject":v.get("text")}
+        tendereeMoney = 0
+        for k,v in dict_anno.items():
+            _split = v.get("type").split("_")
+            if v.get("type") in ["money_tendereeMoney"]:
+                _before_text = Htext[max(v["begin"]-10,0):v["begin"]]
+                if re.search("万",_before_text) is not None and re.search("整",_before_text) is None:
+                    _unit = 10000
+                else:
+                    _unit = 1
+                tendereeMoney = float(getUnifyMoney(v["text"])*_unit)
+            if v.get("type") in ["rel_tendereeMoney","rel_tendererMoney"]:
+                arg1 = v.get("arg1")
+                arg2 = v.get("arg2")
+                for _k,_v in dict_role.items():
+                    if _v["subject"]==dict_anno[arg1]["text"]:
+                        _before_text = Htext[max(dict_anno[arg2]["begin"]-10,0):dict_anno[arg2]["begin"]]
+                        if re.search("万",_before_text) is not None and re.search("整",_before_text) is None:
+                            _unit = 10000
+                        else:
+                            _unit = 1
+                        _v["money"] = float(getUnifyMoney(dict_anno[arg2]["text"])*_unit)
+            if v.get("type")=="person_tendereePerson":
+                if "tenderee" in dict_role:
+                    if "person" not in dict_role["tenderee"]:
+                        dict_role["tenderee"]["person"] = []
+                    dict_role["tenderee"]["person"].append({"person":v["text"]})
+            if v.get("type")=="person_agencyPerson":
+                if "agency" in dict_role:
+                    if "person" not in dict_role["agency"]:
+                        dict_role["agency"]["person"] = []
+                    dict_role["agency"]["person"].append({"person":v["text"]})
+            if v.get("type")=="rel_person":
+                arg1 = v.get("arg1")
+                arg2 = v.get("arg2")
+                for _k,_v in dict_role.items():
+                    if _v["subject"]==dict_anno[arg1]["text"]:
+                        if "person" not in dict_role[_k]:
+                            dict_role[_k]["person"] = []
+                        dict_role[_k]["person"].append({"person":dict_anno[arg2]["text"]})
+                        dict_person2role[dict_anno[arg2]["text"]] = _k
+        for k,v in dict_anno.items():
+            if v.get("type")=="rel_phone":
+                arg1 = v.get("arg1")
+                arg2 = v.get("arg2")
+                _person = dict_anno[arg1]["text"]
+                if _person in dict_person2role:
+                    for item in dict_role[dict_person2role[_person]]["person"]:
+                        if item["person"]==_person:
+                            item["phone"] = dict_anno[arg2]["text"]
+
+
+        roleList = []
+        for k,v in dict_role.items():
+            if k=="tenderee":
+                _role = "tenderee"
+            if k=="agency":
+                _role = "agency"
+            if k=="tenderer":
+                _role = "win_tenderer"
+            if k=="secondTenderer":
+                _role = "second_tenderer"
+            if k=="thirdTenderer":
+                _role = "third_tenderer"
+            list_person = []
+            set_person = set()
+            for item in v.get("person",[]):
+                if item["person"] not in set_person:
+                    list_person.append([item["person"],item.get("phone","")])
+                    set_person.add(item["person"])
+            roleList.append([_role,v.get("subject","").replace("(","（").replace(")","）"),v.get("money",0),list_person,""])
+
+        dict_result["prem"] = {"Project":{"roleList":roleList,"tendereeMoney":tendereeMoney}}
+        return dict_result
+
+
+    def culExtractMetrics(self):
+        conn = self.conn1
+        cursor = conn.cursor()
+        sql = ' select begin_time,end_time,"user",doc_count from corpus_payroll where end_time<=\'2021-07-25\' order by end_time desc limit 20'
+        cursor.execute(sql)
+        list_diff = []
+        rows_payroll = cursor.fetchall()
+        for _payroll in rows_payroll:
+            _begin_time = _payroll[0]
+            _end_time = _payroll[1]
+            _user = _payroll[2]
+            doc_count = _payroll[3]
+            print(_user,_begin_time,_end_time,doc_count)
+            _sql = "select document_id,value from brat_bratannotation where document_id in (select human_identifier from corpus_iedocument where edituser='%s' and to_char(edittime,'yyyy-mm-dd')>='%s' and to_char(edittime,'yyyy-mm-dd')<='%s' limit 100)  order by document_id"%(_user,_begin_time,_end_time)
+            cursor.execute(_sql)
+            rows = cursor.fetchall()
+            if len(rows)>0:
+                current_docid = rows[0][0]
+                _index = -1
+                list_values = []
+                while _index<len(rows)-1:
+                    _index += 1
+                    row = rows[_index]
+                    document_id = row[0]
+                    value = row[1]
+                    if document_id!=current_docid:
+                        print(current_docid)
+                        sql = "select text from corpus_iedocument where human_identifier='%s'"%(str(current_docid))
+                        cursor.execute(sql)
+                        content = cursor.fetchall()[0][0]
+                        _inter = self.label2interface(list_values,content)
+                        _inter2 = self.extractFromInterface(content)
+                        if not len(_inter2.get("prem").keys())>1:
+                            _diff = self.getDiff(_inter,_inter2)
+                            list_diff.append(_diff)
+                        _index -= 1
+                        current_docid = document_id
+                        list_values = []
+                    else:
+                        list_values.append({"document_id":document_id,"value":value})
+        metrics = self.getMetrics(list_diff)
+        print(metrics)
+
+    def extractFromInterface(self,content):
+        return json.loads(test("",content))
+
+    def getDiff(self,_inter,_inter2):
+        _dict = {}
+        for k in ["code","product","person_review"]:
+            set_k1 = _inter.get(k,set())
+            set_k2 = _inter2.get(k,set())
+            _dict["%s_inter"%k] = len(set_k1)
+            _dict["%s_inter2"%k] = len(set_k2)
+            _dict["%s_union"%k] = len(set(set_k1)&set(set_k2))
+
+        for k in ["name","bidway","moneysource","serviceTime","time_release","time_bidopen","time_bidclose"]:
+            _k1 = _inter.get(k,"")
+            _k2 = _inter2.get(k,"")
+            len_k1 = 0 if _k1=="" else 1
+            len_k2 = 0 if _k2=="" else 1
+            len_union = 1 if _k1==_k2 and len_k1==1 else 0
+            _dict["%s_inter"%k] = len_k1
+            _dict["%s_inter2"%k] = len_k2
+            _dict["%s_union"%k] = len_union
+
+
+        dict_project = {}
+        for k,v in _inter.get("prem",{}).items():
+            if float(v.get("tendereeMoney",0))>0:
+                dict_project["%s_inter"%("tendereeMoney")] = [float(v.get("tendereeMoney"))]
+            for _role in v.get("roleList",[]):
+                dict_project["%s_inter"%_role[0]] = [_role[1]]
+                if _role[0] in ["win_tenderer","second_tenderer","third_tenderer"]:
+                    if float(_role[2])>0:
+                        dict_project["%s_money_inter"%_role[0]] = [float(_role[2])]
+                for item in _role[3]:
+                    _person = item[0]
+                    _phone = item[1]
+                    if _person=="" or _phone=="":
+                        continue
+                    if "%s_person_inter"%_role[0] not in dict_project:
+                        dict_project["%s_person_inter"%_role[0]] = []
+                    dict_project["%s_person_inter"%_role[0]].append("%s-%s"%(_role[1],_person))
+                    if "person_phone_inter" not in dict_project:
+                        dict_project["person_phone_inter"] = []
+                    dict_project["person_phone_inter"].append("%s-%s"%(_person,_phone))
+        for k,v in _inter2.get("prem",{}).items():
+            if float(v.get("tendereeMoney",0))>0:
+                dict_project["%s_inter2"%("tendereeMoney")] = [float(v.get("tendereeMoney"))]
+            for _role in v.get("roleList",[]):
+                dict_project["%s_inter2"%_role[0]] = [_role[1]]
+                if _role[0] in ["win_tenderer","second_tenderer","third_tenderer"]:
+                    if float(_role[2])>0:
+                        dict_project["%s_money_inter2"%_role[0]] = [float(_role[2])]
+                for item in _role[3]:
+                    _person = item[0]
+                    _phone = item[1]
+                    if _person=="" or _phone=="":
+                        continue
+                    if "%s_person_inter2"%_role[0] not in dict_project:
+                        dict_project["%s_person_inter2"%_role[0]] = []
+                    dict_project["%s_person_inter2"%_role[0]].append("%s-%s"%(_role[1],_person))
+                    if "person_phone_inter2" not in dict_project:
+                        dict_project["person_phone_inter2"] = []
+                    dict_project["person_phone_inter2"].append("%s-%s"%(_person,_phone))
+        set_k = set()
+        for k,v in dict_project.items():
+            k_split = k.split("_")
+            base_key = "_".join(k_split[:-1])
+            if k_split[-1]=="inter":
+                k2 = "inter2"
+            else:
+                k2 = "inter"
+            if base_key in set_k:
+                continue
+            k_other = "%s_%s"%(base_key,k2)
+            _dict[k] = len(v)
+            _dict[k_other] = len(dict_project.get(k_other,[]))
+            _dict["%s_union"%base_key] = len(set(v)&set(dict_project.get(k_other,[])))
+            set_k.add(base_key)
+        print("=========================")
+        print(_inter)
+        print("-----")
+        print(_inter2)
+        print("|||||")
+        print(_dict)
+        return _dict
+
+
+
+    def getMetrics(self,list_diff):
+        dict_key_count = {}
+        print("all_count:",list_diff)
+        for _diff in list_diff:
+            for k,v in _diff.items():
+                if k not in dict_key_count:
+                    dict_key_count[k] = 0
+                dict_key_count[k] += v
+        set_k = set()
+        for k,v in dict_key_count.items():
+            k_split = k.split("_")
+            base_k = "_".join(k_split[:-1])
+            if base_k in set_k:
+                continue
+            set_k.add(base_k)
+            _count_inter = max(dict_key_count.get("%s_inter"%base_k,-1),1)
+            _count_inter2 = max(dict_key_count.get("%s_inter2"%base_k,-1),1)
+            _count_union = dict_key_count.get("%s_union"%base_k,0)
+            _precision = _count_union/_count_inter2
+            _recall = _count_union/_count_inter
+            _f1 = 2*(_precision*_recall)/(_precision+_recall)
+            print("%s: recall:%.3f,precision:%.3f,f1_score:%.3f"%(base_k,_recall,_precision,_f1))
+            print(base_k)
+            print("%.3f"%_f1)
+            print("%.3f"%_precision)
+            print("%.3f"%_recall)
+
+
+
+if __name__=="__main__":
+    em = ExtractMetric()
+    em.culExtractMetrics()
+

BiddingKG/dl/relation_extraction/model.py

@@ -425,6 +425,11 @@ def save_model():
 
 if __name__ == '__main__':
     test_model = Relation_extraction()
+    test_model.train_model.summary()
+    print("object_model=====================")
+    test_model.object_model.summary()
+    print("subject_model=======================")
+    test_model.subject_model.summary()
     # save_model()
 
     # ['<pad>','<unk>','<company/org>','<location>','<phone>','<contact_person>']

BiddingKG/dl/test/11.py

@@ -0,0 +1,3 @@
+a = "%d-"
+
+print(a%2)

BiddingKG/maxcompute/evaluates.py

@@ -113,10 +113,11 @@ class Extract(BaseUDTF):
         log("import done")
         while True:
             try:
-                item = task_queue.get(True)
+                item = task_queue.get(True,timeout=10)
                 result_json = predict(item.get("docid",""),item.get("content",""),item.get("title",""))
                 result_queue.put(result_json)
             except:
+                log("get data time out")
                 pass
 
     def __init__(self):
@@ -154,6 +155,7 @@ class Extract(BaseUDTF):
         self.result_queue = Queue()
         self.deal_process = Process(target=self.f_queue_process,args=(self.task_queue,self.result_queue))
         self.deal_process.start()
+        time.sleep(60*4)
         import numpy as np
 
 
@@ -180,18 +182,33 @@ class Extract(BaseUDTF):
 
 
         if content is not None and _doc_id not in [105677700,126694044,126795572,126951461,71708072,137850637]:
+            #清除队列中的数据
+            try:
+                while(self.task_queue.qsize()>0):
+                    self.task_queue.get(timeout=5)
+            except Exception as e:
+                pass
+            try:
+                while(self.result_queue.qsize()>0):
+                    self.result_queue.get(timeout=5)
+            except Exception as e:
+                pass
+
             _item = {"docid":_doc_id,"content":content,"title":_title}
-            self.task_queue.put(_item)
+
 
             try:
+                _timeout = 60*4
                 if not self.deal_process.is_alive():
                     log("deal process is down")
+                    self.task_queue = Queue()
                     self.deal_process = Process(target=self.f_queue_process,args=(self.task_queue,self.result_queue))
                     self.deal_process.start()
-                result_json = self.result_queue.get(timeout=60*4)
+                    _timeout += 60*4
+                log("putting item to task_queue with docid:%s"%(str(_doc_id)))
+                self.task_queue.put(_item)
+                result_json = self.result_queue.get(timeout=_timeout)
                 self.forward(page_time,int(_doc_id),result_json)
             except Exception as e:
                 log("dealing docid %s failed by timeout"%(str(_doc_id)))
                 self.deal_process.kill()
-                self.deal_process = Process(target=self.f_queue_process,args=(self.task_queue,self.result_queue))
-                self.deal_process.start()