Merge remote-tracking branch 'origin/master'

# Conflicts:
#	BaseDataMaintenance/common/multiThread.py

BaseDataMaintenance/common/ERNIE_utils.py

@@ -8,6 +8,7 @@ def get_access_token():
     """
 
     url = "https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=gnwVXv96An9qMYqq9eWbeNqk&client_secret=mDsRQbCPsV4N7x28LbwkhTAaLmrrDnXk"
+    url = "https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=Ok8QMe4qIQOAex0F9Gf1uns0&client_secret=6DjGGDdvhnBaEOMdSXAg02KxZnQhWpbd"
 
     payload = json.dumps("")
     headers = {
@@ -19,17 +20,18 @@ def get_access_token():
     return response.json().get("access_token")
 
 def main():
-    # _token = get_access_token()
-    _token = "24.93c9d66ffc94ffaef6c6c9d35770a5f5.2592000.1701242081.282335-37357318"
+    _token = get_access_token()
+    # _token = "24.93c9d66ffc94ffaef6c6c9d35770a5f5.2592000.1701242081.282335-37357318"
     url = "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token=" + _token
 
+    # url = "https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/xuanyuan_70b_chat?access_token=" + _token
+
     payload = json.dumps({
         "messages": [
             {
                 "role": "user",
                 "content": '''
-                假设分类是建筑建材-建筑涂料的相关产品词“面漆”
-                请拓展其相关行业产品词，列举30个
+               今天是几号
                 '''
             }
         ]

BaseDataMaintenance/common/activateMQUtils.py

@@ -18,7 +18,10 @@ def send_msg_toacmq(pool_conn,msg,dest,retry_times=5):
             return True
         except Exception as e:
             traceback.print_exc()
-            conn.disconnect()
+            try:
+                conn.disconnect()
+            except Exception as e:
+                pass
         finally:
             if conn.is_connected():
                 pool_conn.putConnector(conn)

BaseDataMaintenance/common/multiThread.py

@@ -6,7 +6,7 @@ import traceback
 
 import ctypes
 import inspect
-import sys
+import sys,os
 from multiprocessing import Process,Queue
 
 import logging
@@ -46,7 +46,7 @@ class _taskHandler(threading.Thread):
     def run(self):
         while(True):
             try:
-                logging.info("%s handler task queue size is %d need_stop %s thread_id:%d"%(self.task_handler.__name__, self.task_queue.qsize(),str(self.need_stop),threading.get_ident()))
+                logging.info("handler task queue size is %d need_stop %s thread_id:%d-%d"%(self.task_queue.qsize(),str(self.need_stop),os.getpid(),threading.get_ident()))
                 item = self.task_queue.get(True,timeout=5)
 
                 self.task_handler(item,self.result_queue,*self.args,**self.kwargs)

BaseDataMaintenance/dataSource/interface.py

@@ -20,6 +20,9 @@ DEFAULT_TIMEOUT = 3000
 import traceback
 import base64
 
+
+from BaseDataMaintenance.common.Utils import log
+
 def getAttachDealInterface(_data,_type,path="",restry=1,kwargs={},url=interface_url,timeout=DEFAULT_TIMEOUT,session=None):
     _succeed = False
     _html = ""
@@ -34,9 +37,11 @@ def getAttachDealInterface(_data,_type,path="",restry=1,kwargs={},url=interface_
                         "type":_type}
             if len(kwargs.keys())>0:
                 _json.update(kwargs)
-            headers = {"Content-Type":"application/json"}
-            with requests.Session() as session:
-                _resp = session.post(url,data=_json,timeout=timeout)
+
+
+            _json["timeout"] = 10000
+            with requests.Session() as sess:
+                _resp = sess.post(url,data=_json,timeout=timeout)
 
             if _resp.status_code==200:
                 _result = json.loads(_resp.content.decode())

BaseDataMaintenance/maintenance/2.py

@@ -1,35 +1,14 @@
 
-import time
+from multiprocessing import Queue
 
-import gunicorn
 
-import requests
-from bs4 import BeautifulSoup
+a = Queue()
 
-from BaseDataMaintenance.common.Utils import article_limit
+a.put("1")
 
-import codecs
-
-def getAttachPath(filemd5,_dochtmlcon):
-    _soup = BeautifulSoup(_dochtmlcon,"lxml")
-
-    list_mark = ["data","filelink"]
-    for _mark in list_mark:
-        _find = _soup.find("a",attrs={_mark:filemd5})
-        filelink = ""
-        if _find is None:
-            _find = _soup.find("img",attrs={_mark:filemd5})
-            if _find is not None:
-                filelink = _find.attrs.get("src","")
-        else:
-            filelink = _find.attrs.get("href","")
-        if filelink.find("bidizhaobiao")>=0:
-            _path = filelink.split("/file")
-            if len(_path)>1:
-                return _path[1]
-
-if __name__ == '__main__':
-
-    text = codecs.open("C:\\Users\\\Administrator\\Desktop\\2.html","r",encoding="utf8").read()
-    filemd5='61393b5ef3d460b3714eb9667682144f'
-    print(getAttachPath(filemd5,text))
+while 1:
+    try:
+        b = a.get(block=False)
+        print(b)
+    except Exception as e:
+        break

BaseDataMaintenance/maintenance/check_log.py

@@ -0,0 +1,45 @@
+
+
+import re
+
+def test_speed(logfile):
+    a = open(logfile,"r",encoding="utf8").read()
+    set_a = set()
+    _c = 0
+    for a in re.split("\n",s):
+        a = a.strip()
+        if a=="":
+            continue
+        b = a.split(":")[-1]
+        _c += 1
+        print(b)
+        set_a.add(b)
+    print(len(set_a),_c)
+
+def check_start_end(logfile):
+    list_start = []
+    set_end = set()
+    log_str = open(logfile,"r",encoding="utf8").read()
+    for a in re.split("\n",log_str):
+        a = a.strip()
+        if a=="":
+            continue
+        docid = a.split(":")[-1]
+        before = ":".join(a.split(":")[:-1])
+        if before.find("start on")>=0:
+            list_start.append(docid)
+        elif before.find("end on")>=0:
+            set_end.add(docid)
+        elif before.find("error on")>=0:
+            set_end.add(docid)
+
+    check_list = list_start[:-1000]
+    print("check_list length:",len(check_list))
+    for docid in check_list:
+        if docid not in set_end:
+            print("docid no end",docid)
+
+if __name__ == '__main__':
+    logfile = "log.txt"
+    # test_speed(logfile)
+    check_start_end(logfile)

BaseDataMaintenance/maintenance/dataflow.py

@@ -108,6 +108,7 @@ class Dataflow():
         self.queue_extract = Queue()
         self.list_extract = []
         self.queue_dumplicate = PQueue()
+        self.queue_dumplicate_processed = PQueue()
         self.dumplicate_set = set()
         self.queue_merge = Queue()
         self.queue_syncho = Queue()
@@ -2269,7 +2270,6 @@ class Dataflow_dumplicate(Dataflow):
         return []
 
     def dumplicate_check(self,_dict1,_dict2,min_counts,b_log=False):
-        b_log=True
         document_less = _dict1
         docid_less = _dict1["docid"]
         docchannel_less = document_less["docchannel"]
@@ -2814,6 +2814,16 @@ class Dataflow_dumplicate(Dataflow):
     def producer_flow_dumplicate(self,process_count,status_from,columns=[document_tmp_status,document_tmp_save,document_tmp_page_time,document_tmp_docchannel,document_tmp_tenderee,document_tmp_agency,document_tmp_doctitle,document_tmp_sub_docs_json,document_tmp_extract_json,document_attachment_extract_status,document_update_document,document_province,document_city,document_district]):
         q_size = self.queue_dumplicate.qsize()
         log("dumplicate queue size %d"%(q_size))
+
+        while 1:
+            try:
+                docid = self.queue_dumplicate_processed.get(block=False)
+                if docid in self.dumplicate_set:
+                    self.dumplicate_set.remove(docid)
+            except Exception as e:
+                break
+
+
         if q_size>process_count//3:
             return
         bool_query = BoolQuery(must_queries=[
@@ -2845,9 +2855,9 @@ class Dataflow_dumplicate(Dataflow):
                 self.queue_dumplicate.put(_dict)
             _count += len(list_dict)
 
-        _l = list(self.dumplicate_set)
-        _l.sort(key=lambda x:x,reverse=True)
-        self.dumplicate_set = set(_l[:flow_process_count*2])
+        # _l = list(self.dumplicate_set)
+        # _l.sort(key=lambda x:x,reverse=True)
+        # self.dumplicate_set = set(_l[:flow_process_count*2])
 
     def comsumer_flow_dumplicate(self):
         mt = MultiThreadHandler(self.queue_dumplicate,self.dumplicate_comsumer_handle,None,60,1,ots_client=self.ots_client)
@@ -2861,7 +2871,7 @@ class Dataflow_dumplicate(Dataflow):
     def flow_dumpcate_comsumer(self):
         from multiprocessing import Process
         process_count = 3
-        thread_count = 20
+        thread_count = 12
         list_process = []
         def start_thread():
             mt = MultiThreadHandler(self.queue_dumplicate,self.dumplicate_comsumer_handle,None,thread_count,1,need_stop=False,restart=True,timeout=600,ots_client=self.ots_client)
@@ -3916,15 +3926,20 @@ class Dataflow_dumplicate(Dataflow):
             start_time = time.time()
             self.post_extract(item)
 
+
+            log("dumplicate start on:%s"%(str(item.get(document_tmp_docid))))
             base_list = []
             set_docid = set()
 
             list_rules,table_name,table_index = self.translate_dumplicate_rules(flow_dumplicate_status_from,item,get_all=get_all,to_log=False)
             # print("len_rules",len(list_rules),table_name,table_index)
             list_rules.sort(key=lambda x:x["confidence"],reverse=True)
+            list_rules = list_rules[:30]
             _i = 0
             step = 5
 
+            print("here 1")
+
             item["confidence"] = 999
             if item.get(document_tmp_docid) not in set_docid:
                 base_list.append(item)
@@ -3945,7 +3960,7 @@ class Dataflow_dumplicate(Dataflow):
                 _i += step
 
 
-
+            print("here 2")
             b_log = False if upgrade else True
             _time = time.time()
             # log("%d start final check with length:%d"%(item["docid"],len(base_list)))
@@ -4031,12 +4046,12 @@ class Dataflow_dumplicate(Dataflow):
                     self.changeSaveStatus(list_merge_dump)
 
 
-
-
-            # log("dump takes %.2f"%(time.time()-start_time))
+            log("dumplicate end on:%s"%(str(item.get(document_tmp_docid))))
         except Exception as e:
             traceback.print_exc()
-            log("error on dumplicate of %s"%(str(item.get(document_tmp_docid))))
+            log("dumplicate error on:%s"%(str(item.get(document_tmp_docid))))
+        finally:
+            self.queue_dumplicate_processed.put(item.get(document_tmp_docid))
 
 
     def fix_doc_which_not_in_project(self):
@@ -4237,7 +4252,7 @@ if __name__ == '__main__':
     df_dump = Dataflow_dumplicate(start_delete_listener=False)
     # df_dump.start_flow_dumplicate()
     a = time.time()
-    df_dump.test_dumplicate(393550944
+    df_dump.test_dumplicate(397656324
                             )
     # df_dump.test_merge([385521167
     #                     ],[385521113])

BaseDataMaintenance/maintenance/dataflow_mq.py

@@ -77,7 +77,7 @@ class Dataflow_ActivteMQ_attachment(Dataflow_attachment):
 
         self.queue_attachment_ocr = Queue()
         self.queue_attachment_not_ocr = Queue()
-        self.comsumer_count = 18
+        self.comsumer_count = 90
         self.comsumer_process_count = 5
         self.retry_comsumer_count = 10
         self.retry_times = 5
@@ -95,9 +95,12 @@ class Dataflow_ActivteMQ_attachment(Dataflow_attachment):
 
         self.session = None
 
-        for _ in range(self.comsumer_process_count):
-            listener_p = Process(target=self.start_attachment_listener)
-            listener_p.start()
+        # for _ in range(self.comsumer_process_count):
+        #     listener_p = Process(target=self.start_attachment_listener)
+        #     listener_p.start()
+
+        listener_p = Process(target=self.start_attachment_listener)
+        listener_p.start()
 
 
 

BaseDataMaintenance/maintenance/test_speed.py

@@ -1,368 +0,0 @@
-
-
-import re
-
-
-s = '''
-2023-11-22 15:44:41,364 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:44:41,369 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:44:41,567 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:41,599 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:41,641 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:41,686 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:41,777 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:44:41,814 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:41,890 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:41,902 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:42,482 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:42,244 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:44:42,596 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:44:42,639 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:44:43,052 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:44:43,340 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:44:43,353 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:44:43,450 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:44:43,563 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:43,596 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:43,617 - BaseDataMaintenance.common.Utils - INFO - get message of idx:23
-2023-11-22 15:44:43,655 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:44:43,711 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:44,064 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:44:44,066 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:44:44,080 - BaseDataMaintenance.common.Utils - INFO - get message of idx:32
-2023-11-22 15:44:44,318 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:44,329 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:44,513 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:44:44,681 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:44,772 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:44,870 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:44:44,964 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:44:44,986 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:44,990 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:45,017 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:44:45,142 - BaseDataMaintenance.common.Utils - INFO - get message of idx:43
-2023-11-22 15:44:45,183 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:45,285 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:44:45,691 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:45,983 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:44:46,100 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:46,159 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:44:46,193 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:44:46,242 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:44:46,468 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:46,539 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:46,621 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:44:46,660 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:46,703 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:44:46,775 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:46,820 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:46,876 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:46,894 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:47,161 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:44:47,180 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:47,209 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:47,480 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:47,543 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:47,649 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:47,664 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:44:48,101 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:44:48,003 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:44:48,302 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:48,339 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:44:48,432 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:44:48,472 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:48,479 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:48,511 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:48,534 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:44:48,544 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:48,562 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:44:48,620 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:44:48,661 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:48,789 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:48,854 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:48,913 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:44:49,277 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:49,288 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:44:49,302 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:49,306 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:44:49,331 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:49,494 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:44:49,513 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:49,579 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:49,884 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:44:50,070 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:44:50,199 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:44:50,292 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:50,406 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:44:50,490 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:44:50,514 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:50,541 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:50,642 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:44:50,880 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:50,991 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:51,049 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:51,066 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:44:51,127 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:44:51,175 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:51,597 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:51,691 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:51,734 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:44:51,884 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:51,952 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:52,141 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:44:51,991 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:52,098 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:44:52,113 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:52,155 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:44:52,171 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:52,174 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:52,209 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:52,431 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:44:52,574 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:44:52,713 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:44:52,727 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:44:52,777 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:44:52,798 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:44:52,810 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:44:52,820 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:44:52,824 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:52,906 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:44:52,926 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:44:52,927 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:44:53,000 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:53,171 - BaseDataMaintenance.common.Utils - INFO - get message of idx:42
-2023-11-22 15:44:53,189 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:44:53,392 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:44:53,555 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:53,623 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:53,826 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:44:53,953 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:53,956 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:53,997 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:44:54,083 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:44:54,042 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:44:54,222 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:54,225 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:54,284 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:54,341 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:44:54,347 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:54,393 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:54,394 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:54,466 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:44:54,486 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:44:54,520 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:54,689 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:54,620 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:44:54,687 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:44:54,701 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:54,736 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:54,752 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:54,813 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:44:54,854 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:44:54,921 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:55,107 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:44:55,210 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:44:55,256 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:44:55,299 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:44:55,778 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:44:55,906 - BaseDataMaintenance.common.Utils - INFO - get message of idx:23
-2023-11-22 15:44:55,959 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:44:56,038 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:44:56,065 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:44:56,072 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:44:56,189 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:56,205 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:44:56,207 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:44:56,364 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:56,275 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:56,341 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:56,357 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:56,379 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:56,406 - BaseDataMaintenance.common.Utils - INFO - get message of idx:42
-2023-11-22 15:44:56,417 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:56,424 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:44:56,440 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:44:56,444 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:56,498 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:44:56,512 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:44:56,596 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:56,626 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:44:56,707 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:56,712 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:44:56,836 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:57,058 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:44:57,086 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:44:57,136 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:57,347 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:44:57,375 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:57,451 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:44:57,609 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:44:57,645 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:44:57,724 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:44:57,726 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:44:57,764 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:44:57,816 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:44:57,893 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:44:57,917 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:44:58,044 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:44:58,128 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:44:58,240 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:44:58,258 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:44:58,459 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:44:58,497 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:44:58,498 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:44:58,709 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:44:58,593 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:44:58,603 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:44:58,612 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:44:58,627 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:44:58,648 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:44:58,712 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:44:58,763 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:44:58,821 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:44:58,825 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:44:58,949 - BaseDataMaintenance.common.Utils - INFO - get message of idx:23
-2023-11-22 15:44:58,972 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:44:58,973 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:44:59,185 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:44:59,231 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:44:59,261 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:44:59,278 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:44:59,381 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:44:59,410 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:44:59,489 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:44:59,504 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:44:59,759 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:44:59,762 - BaseDataMaintenance.common.Utils - INFO - get message of idx:42
-2023-11-22 15:44:59,787 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:44:59,821 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:44:59,919 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:44:59,956 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:45:00,021 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:45:00,063 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:45:00,161 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:45:00,189 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:45:00,525 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:45:00,724 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:45:00,727 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:45:00,728 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:45:00,745 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:45:00,847 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:45:00,861 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:45:00,876 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:45:00,884 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:45:00,970 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:45:01,228 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:45:01,238 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:45:01,254 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:45:01,267 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:45:01,539 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:45:01,560 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:45:01,636 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:45:01,667 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:45:01,673 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:45:01,685 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:45:01,686 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:45:01,689 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:45:01,720 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:45:01,739 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:45:01,740 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:45:01,921 - BaseDataMaintenance.common.Utils - INFO - get message of idx:23
-2023-11-22 15:45:01,995 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:45:02,082 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:45:02,090 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:45:02,193 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:45:02,310 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:45:02,440 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:45:02,458 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:45:02,496 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:45:02,531 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:45:02,588 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:45:02,608 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:45:02,717 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:45:02,763 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:45:02,817 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:45:02,841 - BaseDataMaintenance.common.Utils - INFO - get message of idx:42
-2023-11-22 15:45:02,878 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:45:02,931 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:45:03,051 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:45:03,123 - BaseDataMaintenance.common.Utils - INFO - get message of idx:32
-2023-11-22 15:45:03,127 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:45:03,078 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:45:03,229 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:45:03,302 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:45:03,358 - BaseDataMaintenance.common.Utils - INFO - get message of idx:26
-2023-11-22 15:45:03,417 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:45:03,545 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:45:03,808 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:45:03,897 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:45:03,925 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:45:03,955 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:45:04,033 - BaseDataMaintenance.common.Utils - INFO - get message of idx:36
-2023-11-22 15:45:04,065 - BaseDataMaintenance.common.Utils - INFO - get message of idx:24
-2023-11-22 15:45:04,072 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:45:04,072 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:45:04,096 - BaseDataMaintenance.common.Utils - INFO - get message of idx:1
-2023-11-22 15:45:04,099 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:45:04,102 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:45:04,134 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:45:04,148 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-2023-11-22 15:45:04,208 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:45:04,330 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:45:04,335 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:45:04,392 - BaseDataMaintenance.common.Utils - INFO - get message of idx:3
-2023-11-22 15:45:04,614 - BaseDataMaintenance.common.Utils - INFO - get message of idx:10
-2023-11-22 15:45:04,827 - BaseDataMaintenance.common.Utils - INFO - get message of idx:22
-2023-11-22 15:45:04,992 - BaseDataMaintenance.common.Utils - INFO - get message of idx:34
-2023-11-22 15:45:05,001 - BaseDataMaintenance.common.Utils - INFO - get message of idx:40
-2023-11-22 15:45:05,090 - BaseDataMaintenance.common.Utils - INFO - get message of idx:39
-2023-11-22 15:45:05,091 - BaseDataMaintenance.common.Utils - INFO - get message of idx:16
-2023-11-22 15:45:05,190 - BaseDataMaintenance.common.Utils - INFO - get message of idx:28
-2023-11-22 15:45:05,214 - BaseDataMaintenance.common.Utils - INFO - get message of idx:49
-2023-11-22 15:45:05,229 - BaseDataMaintenance.common.Utils - INFO - get message of idx:12
-2023-11-22 15:45:05,400 - BaseDataMaintenance.common.Utils - INFO - get message of idx:11
-2023-11-22 15:45:05,259 - BaseDataMaintenance.common.Utils - INFO - get message of idx:7
-2023-11-22 15:45:05,320 - BaseDataMaintenance.common.Utils - INFO - get message of idx:38
-2023-11-22 15:45:05,370 - BaseDataMaintenance.common.Utils - INFO - get message of idx:13
-2023-11-22 15:45:05,566 - BaseDataMaintenance.common.Utils - INFO - get message of idx:17
-2023-11-22 15:45:05,540 - BaseDataMaintenance.common.Utils - INFO - get message of idx:8
-2023-11-22 15:45:05,585 - BaseDataMaintenance.common.Utils - INFO - get message of idx:14
-2023-11-22 15:45:05,620 - BaseDataMaintenance.common.Utils - INFO - get message of idx:47
-2023-11-22 15:45:05,646 - BaseDataMaintenance.common.Utils - INFO - get message of idx:42
-2023-11-22 15:45:06,029 - BaseDataMaintenance.common.Utils - INFO - get message of idx:35
-2023-11-22 15:45:06,087 - BaseDataMaintenance.common.Utils - INFO - get message of idx:9
-2023-11-22 15:45:06,125 - BaseDataMaintenance.common.Utils - INFO - get message of idx:29
-2023-11-22 15:45:06,175 - BaseDataMaintenance.common.Utils - INFO - get message of idx:21
-2023-11-22 15:45:06,325 - BaseDataMaintenance.common.Utils - INFO - get message of idx:20
-2023-11-22 15:45:06,326 - BaseDataMaintenance.common.Utils - INFO - get message of idx:25
-2023-11-22 15:45:06,343 - BaseDataMaintenance.common.Utils - INFO - get message of idx:32
-2023-11-22 15:45:06,447 - BaseDataMaintenance.common.Utils - INFO - get message of idx:5
-2023-11-22 15:45:06,451 - BaseDataMaintenance.common.Utils - INFO - get message of idx:23
-2023-11-22 15:45:06,596 - BaseDataMaintenance.common.Utils - INFO - get message of idx:48
-2023-11-22 15:45:06,597 - BaseDataMaintenance.common.Utils - INFO - get message of idx:2
-2023-11-22 15:45:06,616 - BaseDataMaintenance.common.Utils - INFO - get message of idx:37
-2023-11-22 15:45:06,655 - BaseDataMaintenance.common.Utils - INFO - get message of idx:18
-2023-11-22 15:45:06,795 - BaseDataMaintenance.common.Utils - INFO - get message of idx:33
-2023-11-22 15:45:06,873 - BaseDataMaintenance.common.Utils - INFO - get message of idx:41
-2023-11-22 15:45:06,895 - BaseDataMaintenance.common.Utils - INFO - get message of idx:44
-2023-11-22 15:45:06,949 - BaseDataMaintenance.common.Utils - INFO - get message of idx:27
-2023-11-22 15:45:06,961 - BaseDataMaintenance.common.Utils - INFO - get message of idx:0
-2023-11-22 15:45:06,968 - BaseDataMaintenance.common.Utils - INFO - get message of idx:30
-2023-11-22 15:45:06,981 - BaseDataMaintenance.common.Utils - INFO - get message of idx:19
-2023-11-22 15:45:07,061 - BaseDataMaintenance.common.Utils - INFO - get message of idx:45
-2023-11-22 15:45:07,139 - BaseDataMaintenance.common.Utils - INFO - get message of idx:31
-2023-11-22 15:45:07,172 - BaseDataMaintenance.common.Utils - INFO - get message of idx:4
-
-'''
-
-set_a = set()
-_c = 0
-for a in re.split("\n",s):
-    a = a.strip()
-    if a=="":
-        continue
-    b = a.split(":")[-1]
-    _c += 1
-    print(b)
-    set_a.add(b)
-print(len(set_a),_c)

BaseDataMaintenance/maxcompute/documentMerge.py

@@ -2129,7 +2129,7 @@ def dumplicate_projects(list_projects,b_log=False):
     appendKeyvalueCount(list_projects)
     list_projects.sort(key=lambda x:str(x.get(project_page_time,"")))
     list_projects.sort(key=lambda x:x.get("keyvaluecount",0),reverse=True)
-    cluster_projects = list_projects[:50]
+    cluster_projects = list_projects[:10]
     _count = 10
     print("dumplicate projects rest",len(cluster_projects))
     while _count>0:
@@ -3113,6 +3113,53 @@ class f_generate_project_with_attrs_json(BaseUDTF):
             project_json = to_project_json([_group])
             self.forward(project_json)
 
+@annotate('string,string -> string,string,bigint,bigint')
+class f_fix_merge_filter(BaseUDTF):
+    '''
+    将多个组拆解成多条记录
+    '''
+
+    def __init__(self):
+        import logging
+        import json
+        global json,logging
+        logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    def process(self,uuid,docids):
+        if docids is not None:
+            list_docid = [int(a) for a in docids.split(",")]
+            new_list_docid = []
+
+            for docid in list_docid:
+                if docid< 39400549900:
+                    new_list_docid.append(str(docid))
+            if len(new_list_docid)==0:
+                self.forward(uuid,docids,0,500)
+            else:
+                if len(list_docid)!=len(new_list_docid):
+                    self.forward(uuid,",".join(new_list_docid),len(new_list_docid),201)
+
+@annotate('string -> bigint,bigint')
+class f_fix_merge_rerun_document(BaseUDTF):
+    '''
+    将多个组拆解成多条记录
+    '''
+
+    def __init__(self):
+        import logging
+        import json
+        global json,logging
+        logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    def process(self,docids):
+        if docids is not None:
+            list_docid = [int(a) for a in docids.split(",")]
+            docid = list_docid[0]
+            partitionkey = docid%500+1
+
+            self.forward(partitionkey,docid)
+
+
 @annotate('string -> string')
 class f_generate_project_with_delete_uuid(BaseUDTF):
     '''

BaseDataMaintenance/model/ots/document.py

@@ -303,35 +303,39 @@ def turn_document_status():
     def producer(task_queue,ots_client):
 
 
-        # bool_query = BoolQuery(
-        #     must_queries=[
-        #         MatchPhraseQuery("doctitle","珠海城市职业技术学院2022年05月至2022年06月政府采购意向"),
-        #         # BoolQuery(should_queries=[
-        #         #                           # TermQuery("tenderee","山西利民工业有限责任公司"),
-        #         #                           # MatchPhraseQuery("doctitle","中国电信"),
-        #         #                           # MatchPhraseQuery("doctextcon","中国电信"),
-        #         #                           # MatchPhraseQuery("attachmenttextcon","中国电信")]),
-        #         #                           # RangeQuery(document_status,88,120,True,True),
-        #         #                           RangeQuery("page_time","2022-03-24","2022-03-25",True,False),
-        #         #                           ExistsQuery
-        #         #                                  #,TermQuery(document_docid,171146519)
-        #         #                                  ]
-        #         # )
-        #     ],
-        #     # must_not_queries=[WildcardQuery("DX004354*")]
-        # )
         bool_query = BoolQuery(
-            # must_queries=[
-            #     RangeQuery("crtime","2023-08-30 15:00:00","2023-08-30 23:59:59"),
-            #     NestedQuery("page_attachments",ExistsQuery("page_attachments.fileMd5"))
-            # ],
-            # must_not_queries=[WildcardQuery("attachmenttextcon","*")],
-            should_queries=[
-                NestedQuery("sub_docs_json",TermQuery("sub_docs_json.win_tenderer","个体工商户")),
-                NestedQuery("sub_docs_json",TermQuery("sub_docs_json.win_tenderer","机械设备")),
-            ]
-
+            must_queries=[
+                # MatchPhraseQuery("doctitle","珠海城市职业技术学院2022年05月至2022年06月政府采购意向"),
+                # WildcardQuery("web_source_no","14763*"),
+                RangeQuery("status",0,1),
+                # NestedQuery("page_attachments",ExistsQuery("page_attachments.fileMd5")),
+                # TermQuery("docid",397656324)
+                # BoolQuery(should_queries=[
+                #                           # TermQuery("tenderee","山西利民工业有限责任公司"),
+                #                           # MatchPhraseQuery("doctitle","中国电信"),
+                #                           # MatchPhraseQuery("doctextcon","中国电信"),
+                #                           # MatchPhraseQuery("attachmenttextcon","中国电信")]),
+                #                           # RangeQuery(document_status,88,120,True,True),
+                #                           RangeQuery("page_time","2022-03-24","2022-03-25",True,False),
+                #                           ExistsQuery
+                #                                  #,TermQuery(document_docid,171146519)
+                #                                  ]
+                # )
+            ],
+            # must_not_queries=[WildcardQuery("DX004354*")]
         )
+        # bool_query = BoolQuery(
+        #     # must_queries=[
+        #     #     RangeQuery("crtime","2023-08-30 15:00:00","2023-08-30 23:59:59"),
+        #     #     NestedQuery("page_attachments",ExistsQuery("page_attachments.fileMd5"))
+        #     # ],
+        #     # must_not_queries=[WildcardQuery("attachmenttextcon","*")],
+        #     should_queries=[
+        #         NestedQuery("sub_docs_json",TermQuery("sub_docs_json.win_tenderer","个体工商户")),
+        #         NestedQuery("sub_docs_json",TermQuery("sub_docs_json.win_tenderer","机械设备")),
+        #     ]
+        #
+        # )
 
         rows,next_token,total_count,is_all_succeed = ots_client.search("document","document_index",
                                                                        SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("docid",SortOrder.DESC)]),limit=100,get_total_count=True),
@@ -722,7 +726,7 @@ def turn_document_docchannel():
 
 if __name__=="__main__":
     # turn_extract_status()
-    # turn_document_status()
+    turn_document_status()
     # drop_extract2()
     # fixDocumentHtml()
-    turn_document_docchannel()
+    # turn_document_docchannel()