首页 发现 帮助
注册 登录
luojiehua
/
BaseDataMaintenance
1
0
派生 0
文件 工单管理 0 合并请求 0 Wiki
浏览代码

修复配置提取效率问题；附件识别完之后删除比地链接

luojiehua 1 年之前
父节点
dd4d106141
当前提交
0f8e58fa24
共有 2 个文件被更改，包括 2 次插入 和 1 次删除
合并视图 显示文件统计
  1. 1 0
      BaseDataMaintenance/maintenance/dataflow_mq.py
  2. 1 1
      BaseDataMaintenance/model/ots/document_html.py

+ 1 - 0
BaseDataMaintenance/maintenance/dataflow_mq.py
查看文件 

@@ -210,6 +210,7 @@ class Dataflow_ActivteMQ_attachment(Dataflow_attachment):
 
                 ackMsg(conn,message_id)
 
                 ackMsg(conn,message_id)
 
             log("document:%d get attachments with result:%s %s retry_times:%d"%(item.get("docid"),str(_succeed),str(_to_ack),_retry_times))
 
             log("document:%d get attachments with result:%s %s retry_times:%d"%(item.get("docid"),str(_succeed),str(_to_ack),_retry_times))
 
         except Exception as e:
 
         except Exception as e:
 
+            traceback.print_exc()
 
             if time.time()-start_time<10:
 
             if time.time()-start_time<10:
 
                 item["retry_times"] -= 1
 
                 item["retry_times"] -= 1
 
             if send_msg_toacmq(self.pool_mq,json.dumps(item,cls=MyEncoder,ensure_ascii=False),self.mq_attachment):
 
             if send_msg_toacmq(self.pool_mq,json.dumps(item,cls=MyEncoder,ensure_ascii=False),self.mq_attachment):

+ 1 - 1
BaseDataMaintenance/model/ots/document_html.py
查看文件 

@@ -55,7 +55,7 @@ class Document_html(BaseModel):
 
         _dochtmlcon = re.sub("<html>|</html>|<body>|</body>","",_dochtmlcon)
 
         _dochtmlcon = re.sub("<html>|</html>|<body>|</body>","",_dochtmlcon)
 
         _soup = BeautifulSoup(_dochtmlcon,"lxml")
 
         _soup = BeautifulSoup(_dochtmlcon,"lxml")
 
         for a in _soup.find_all("a"):
 
         for a in _soup.find_all("a"):
 
-            if a.attrs["href"].startswith("http://www.bidizhaobiao.com"):
 
+            if a.attrs.get("href","").startswith("http://www.bidizhaobiao.com"):
 
                 a.decompose()
 
                 a.decompose()
 
         self.setValue(document_dochtmlcon,re.sub("<html>|</html>|<body>|</body>","",str(_soup)),True)
 
         self.setValue(document_dochtmlcon,re.sub("<html>|</html>|<body>|</body>","",str(_soup)),True)