首頁 探索 說明
註冊 登入
luojiehua
/
BaseDataMaintenance
1
0
複刻 0
Files 問題管理 0 合併請求 0 Wiki
瀏覽代碼

修复配置提取效率问题；附件识别完之后删除比地链接

luojiehua 1 年之前
父節點
dd4d106141
當前提交
0f8e58fa24
共有 2 個文件被更改，包括 2 次插入 和 1 次删除
統一視圖 顯示文件統計
  1. 1 0
      BaseDataMaintenance/maintenance/dataflow_mq.py
  2. 1 1
      BaseDataMaintenance/model/ots/document_html.py

+ 1 - 0
BaseDataMaintenance/maintenance/dataflow_mq.py
查看文件 

@@ -210,6 +210,7 @@ class Dataflow_ActivteMQ_attachment(Dataflow_attachment):
 
                 ackMsg(conn,message_id)
 
                 ackMsg(conn,message_id)
 
             log("document:%d get attachments with result:%s %s retry_times:%d"%(item.get("docid"),str(_succeed),str(_to_ack),_retry_times))
 
             log("document:%d get attachments with result:%s %s retry_times:%d"%(item.get("docid"),str(_succeed),str(_to_ack),_retry_times))
 
         except Exception as e:
 
         except Exception as e:
 
+            traceback.print_exc()
 
             if time.time()-start_time<10:
 
             if time.time()-start_time<10:
 
                 item["retry_times"] -= 1
 
                 item["retry_times"] -= 1
 
             if send_msg_toacmq(self.pool_mq,json.dumps(item,cls=MyEncoder,ensure_ascii=False),self.mq_attachment):
 
             if send_msg_toacmq(self.pool_mq,json.dumps(item,cls=MyEncoder,ensure_ascii=False),self.mq_attachment):

+ 1 - 1
BaseDataMaintenance/model/ots/document_html.py
查看文件 

@@ -55,7 +55,7 @@ class Document_html(BaseModel):
 
         _dochtmlcon = re.sub("<html>|</html>|<body>|</body>","",_dochtmlcon)
 
         _dochtmlcon = re.sub("<html>|</html>|<body>|</body>","",_dochtmlcon)
 
         _soup = BeautifulSoup(_dochtmlcon,"lxml")
 
         _soup = BeautifulSoup(_dochtmlcon,"lxml")
 
         for a in _soup.find_all("a"):
 
         for a in _soup.find_all("a"):
 
-            if a.attrs["href"].startswith("http://www.bidizhaobiao.com"):
 
+            if a.attrs.get("href","").startswith("http://www.bidizhaobiao.com"):
 
                 a.decompose()
 
                 a.decompose()
 
         self.setValue(document_dochtmlcon,re.sub("<html>|</html>|<body>|</body>","",str(_soup)),True)
 
         self.setValue(document_dochtmlcon,re.sub("<html>|</html>|<body>|</body>","",str(_soup)),True)