Kaynağa Gözat

修复去重有时卡死的问题,产品匹配优化,周期项目补充联系人

luojiehua 1 yıl önce
ebeveyn
işleme
961eb05d32

+ 22 - 21
BaseDataMaintenance/maintenance/preproject/fillColumns.py

@@ -16,27 +16,6 @@ class PreprojectFill():
         self.task_queue = Queue(3000)
         self.fill_concat_queue = Queue(10000)
 
-    def fill_producer(self):
-        q1 = BoolQuery(should_queries=[WildcardQuery("uuid","*"),
-                                          RangeQuery("has_bidfile",0)])
-        columns = ["uuid","has_bidfile","json_docids"]
-        query = BoolQuery(must_not_queries=[q1])
-        rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
-                                                                            SearchQuery(query,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
-                                                                            ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
-        while True or self.task_queue.full():
-            if len(rows)>0:
-                dict_rows = getRow_ots(rows)
-                for _row in dict_rows:
-                    self.task_queue.put(_row)
-            else:
-                break
-            rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
-                                                                                SearchQuery(query,next_token=next_token,get_total_count=True,limit=100),
-                                                                                ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
-
-
-
     def fill_comsumer(self):
 
         def comsumer_handle(_row,result_queue):
@@ -65,6 +44,27 @@ class PreprojectFill():
         _mul = MultiThreadHandler(self.task_queue,comsumer_handle,None,10)
         _mul.run()
 
+
+
+    def fill_producer(self):
+        q1 = BoolQuery(should_queries=[WildcardQuery("uuid","*"),
+                                          RangeQuery("has_bidfile",0)])
+        columns = ["uuid","has_bidfile","json_docids"]
+        query = BoolQuery(must_not_queries=[q1])
+        rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
+                                                                            SearchQuery(query,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
+                                                                            ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
+        while True or self.task_queue.full():
+            if len(rows)>0:
+                dict_rows = getRow_ots(rows)
+                for _row in dict_rows:
+                    self.task_queue.put(_row)
+            else:
+                break
+            rows,next_token,total_count,is_all_succeed = self.ots_client.search("preproject","preproject_index",
+                                                                                SearchQuery(query,next_token=next_token,get_total_count=True,limit=100),
+                                                                                ColumnsToGet(columns,ColumnReturnType.SPECIFIED))
+
     def fill_contact_producer(self):
         q1 = BoolQuery(must_queries=[TermQuery("status",1),
                                        ])
@@ -178,6 +178,7 @@ class PreprojectFill():
 
 
             _preproject = Preproject(_row)
+            _preproject.setValue("status",2,True)
             _preproject.update_row(self.ots_client)
 
         _mul = MultiThreadHandler(self.fill_concat_queue,comsumer_handle,None,10)