修复附件上传可能造成中断的问题

BaseDataMaintenance/common/ossUtils.py

@@ -34,18 +34,19 @@ def getMDFFromFile(path):
         traceback.print_exc()
         return None,_length
 
-def uploadFileByPath(bucket,filepath,uploadpath,headers=None):
-    try:
-        start_time = time.time()
-        log("uploading file of %s"%filepath)
-        with open(filepath,"rb") as f:
-            bucket.put_object(uploadpath,f,headers=headers)
-        log("upload file of %s takes %ds"%(filepath,time.time()-start_time))
-        return True
-    except Exception as e:
-        traceback.print_exc()
-        log("upload object failed of %s"%(filepath))
-        return False
+def uploadFileByPath(bucket,filepath,uploadpath,headers=None,trytimes=3):
+    for _i in range(trytimes):
+        try:
+            start_time = time.time()
+            log("uploading file of %s"%filepath)
+            with open(filepath,"rb") as f:
+                bucket.put_object(uploadpath,f,headers=headers)
+            log("upload file of %s takes %ds"%(filepath,time.time()-start_time))
+            return True
+        except Exception as e:
+            traceback.print_exc()
+            log("upload object failed of %s"%(filepath))
+    return False
 
 def deleteObject(bucket,objectName):
     try:
@@ -104,7 +105,7 @@ def test_download(filemd5):
 
 
 if __name__=="__main__":
-    # print(getMDFFromFile('1623894475151.pdf'))
+    print(getMDFFromFile('8a9c96a68803c2ad01881d0ee93618e5.pdf'))
     # test_download("0852ca62c6e3da56a89a02ed4af87724")
-    print(bucket.sign_url("GET","0015//20220623/2022-06-22/WGH001018/1655926900020.png",86500*30))
-    print(time.strftime("%Y-%m-%d",time.localtime(1658655178)))
+    # print(bucket.sign_url("GET","0015//20220623/2022-06-22/WGH001018/1655926900020.png",86500*30))
+    # print(time.strftime("%Y-%m-%d",time.localtime(1658655178)))

BaseDataMaintenance/maintenance/dataflow.py

@@ -3515,7 +3515,6 @@ class Dataflow_dumplicate(Dataflow):
                       TermQuery(project_agency,agency)]
             list_query.append([_query,1])
 
-        print(bidding_budget,"bidding_budget",type(bidding_budget))
         if tenderee!="" and float(bidding_budget)>0:
             _query = [TermQuery(project_tenderee,tenderee),
                                              TermQuery(project_bidding_budget,bidding_budget)]
@@ -3920,7 +3919,6 @@ class Dataflow_dumplicate(Dataflow):
                 dtmp.setValue(document_tmp_projects,"[]",True)
             else:
                 dtmp.setValue(document_tmp_projects,self.merge_document_real(item,list_docids,table_name,dtmp.getProperties().get(document_tmp_save),flow_dumplicate_status_to,b_log),True)
-            log(dtmp.getProperties().get(document_tmp_projects))
             log("upgrate %s save:%s:docid:%d,final_list:%d,rules:%d,best_docid:%s,dmp_docid:%s"%(str(upgrade),dtmp.getProperties().get(document_tmp_save),item.get(document_tmp_docid),len(final_list),len(list_rules),str(best_docid),dmp_docid))
             if upgrade:
                 if table_name=="document_tmp":
@@ -4132,7 +4130,7 @@ if __name__ == '__main__':
     df_dump = Dataflow_dumplicate(start_delete_listener=False)
     # df_dump.start_flow_dumplicate()
     a = time.time()
-    df_dump.test_dumplicate(316399675)
+    df_dump.test_dumplicate(324785921)
     # df_dump.test_merge([292315564],[287890754])
     # df_dump.flow_remove_project_tmp()
     print("takes",time.time()-a)

BaseDataMaintenance/maintenance/dataflow_mq.py

@@ -375,7 +375,7 @@ class Dataflow_ActivteMQ_attachment(Dataflow_attachment):
                 if local_exists:
                     upload_status = uploadFileByPath(self.bucket,localpath,objectPath)
                 try:
-                    if os.exists(localpath):
+                    if upload_status and os.exists(localpath):
                         os.remove(localpath)
                 except Exception as e:
                     pass

BaseDataMaintenance/maintenance/preproject/fillColumns.py

@@ -129,8 +129,24 @@ def delete_wrong_data():
     mt.run()
 
 
+def fixProductName():
+    from BaseDataMaintenance.dataSource.source import getConnect_ots
+    ots_client = getConnect_ots()
+    bool_query = BoolQuery(must_queries=[MatchPhraseQuery("product","酒水降尘")])
+    rows,next_token,total_count,is_all_succeed = ots_client.search("preproject","preproject_index",
+                                                                   SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("uuid")]),get_total_count=True,limit=100),
+                                                                   ColumnsToGet(return_type=ColumnReturnType.ALL))
+    dict_rows = getRow_ots(rows)
+    for _row in dict_rows:
+        preproject1 = Preproject(_row)
+        preproject1.delete_row(ots_client)
+        _row["product"] = _row["product"].replace("酒水降尘","洒水降尘")
+        preproject2 = Preproject(_row)
+        preproject2.update_row(ots_client)
+    print(dict_rows)
 
 if __name__=="__main__":
     # preprojectFill = PreprojectFill()
     # preprojectFill.schedule()
-    delete_wrong_data()
+    # delete_wrong_data()
+    fixProductName()

BaseDataMaintenance/maintenance/proposedBuilding/DataSynchronization.py

@@ -69,8 +69,10 @@ class DataSynchronization():
                 _time = time.time()
                 if _project_dict is not None:
                     #更新数据
+                    # log("project_dict:"+json.dumps(_project_dict))
                     _designed_project = designed_project(_project_dict)
                     _designed_project.update_project(ots_client)
+                    log("update desined takes %.2fs"%(time.time()-_time))
 
                 #删除tmp
                 _proposed.delete_row(ots_client)
@@ -118,7 +120,7 @@ class DataSynchronization():
         ots_client = self.ots_client
         current_date = getCurrent_date("%Y-%m-%d")
 
-        bool_query = BoolQuery(must_queries=[RangeQuery("latest_service_time",range_to=current_date)],
+        bool_query = BoolQuery(must_queries=[RangeQuery("latest_service_time",range_from="2000-01-01",range_to=current_date)],
                                must_not_queries=[TermQuery("progress","竣工阶段")])
 
         columns = ["progress"]