|
@@ -137,6 +137,7 @@ class BaseDataMonitor():
|
|
|
|
|
|
|
|
|
|
def monitor_attachment(self):
|
|
def monitor_attachment(self):
|
|
|
|
+
|
|
try:
|
|
try:
|
|
# query = BoolQuery(must_queries=[
|
|
# query = BoolQuery(must_queries=[
|
|
# RangeQuery("status",0,11),
|
|
# RangeQuery("status",0,11),
|
|
@@ -165,21 +166,57 @@ class BaseDataMonitor():
|
|
# SearchQuery(query,None,True),
|
|
# SearchQuery(query,None,True),
|
|
# columns_to_get=ColumnsToGet(return_type=ColumnReturnType.NONE))
|
|
# columns_to_get=ColumnsToGet(return_type=ColumnReturnType.NONE))
|
|
|
|
|
|
- _cmd = 'cat %s | grep -c "%s.*process filemd5"'%(flow_attachment_log_path,self.get_last_tenmin_time())
|
|
|
|
- log(_cmd)
|
|
|
|
- process_count = self.cmd_execute(_cmd)
|
|
|
|
|
|
|
|
- _cmd = 'cat %s | grep -c "%s.*process filemd5.*True"'%(flow_attachment_log_path,self.get_last_tenmin_time())
|
|
|
|
- log(_cmd)
|
|
|
|
- process_succeed_count = self.cmd_execute(_cmd)
|
|
|
|
|
|
|
|
- _cmd = 'cat %s | grep -c "%s.*delete sql"'%(flow_init_path,self.get_last_tenmin_time())
|
|
|
|
- log(_cmd)
|
|
|
|
- init_count = self.cmd_execute(_cmd)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+ #通过命令行获取日志情况
|
|
|
|
+ # _cmd = 'cat %s | grep -c "%s.*process filemd5"'%(flow_attachment_log_path,self.get_last_tenmin_time())
|
|
|
|
+ # log(_cmd)
|
|
|
|
+ # process_count = self.cmd_execute(_cmd)
|
|
|
|
+ #
|
|
|
|
+ # _cmd = 'cat %s | grep -c "%s.*process filemd5.*True"'%(flow_attachment_log_path,self.get_last_tenmin_time())
|
|
|
|
+ # log(_cmd)
|
|
|
|
+ # process_succeed_count = self.cmd_execute(_cmd)
|
|
|
|
+ #
|
|
|
|
+ # _cmd = 'cat %s | grep -c "%s.*delete sql"'%(flow_init_path,self.get_last_tenmin_time())
|
|
|
|
+ # log(_cmd)
|
|
|
|
+ # init_count = self.cmd_execute(_cmd)
|
|
|
|
+
|
|
|
|
+ # _msg = "附件提取队列报警:队列堆积%s条公告,最近十分钟处理公告附件数:%s,处理成功数:%s"%(str(total_count_todeal),str(process_count),str(process_succeed_count))
|
|
|
|
+
|
|
|
|
+ #通过读取文件获取日志情况
|
|
|
|
+ dict_type = {}
|
|
|
|
+ _pattern = "%s.*process filemd5\:[^\s]* (?P<result>(True|False)) of type\:(?P<type>[^\s]*).*recognize takes (?P<costtime>\d+)s"%(re.escape(self.get_last_tenmin_time()))
|
|
|
|
+ with open(flow_attachment_log_path,"r",encoding="utf8") as f:
|
|
|
|
+ while True:
|
|
|
|
+ line = f.readline()
|
|
|
|
+ if not line:
|
|
|
|
+ break
|
|
|
|
+ _match = re.search(_pattern,str(line))
|
|
|
|
+ if _match is not None:
|
|
|
|
+ _type = _match.groupdict().get("type")
|
|
|
|
+ _result = _match.groupdict().get("result")
|
|
|
|
+ _costtime = _match.groupdict().get("costtime")
|
|
|
|
+ if _type not in dict_type:
|
|
|
|
+ dict_type[_type] = {"success":0,"fail":0,"success_costtime":0,"fail_costtime":0}
|
|
|
|
+ if _result=="True":
|
|
|
|
+ dict_type[_type]["success"] += 1
|
|
|
|
+ dict_type[_type]["success_costtime"] += int(_costtime)
|
|
|
|
+ else:
|
|
|
|
+ dict_type[_type]["fail"] += 1
|
|
|
|
+ dict_type[_type]["fail_costtime"] += int(_costtime)
|
|
|
|
+
|
|
|
|
+ process_count = 0
|
|
|
|
+ process_succeed_count = 0
|
|
|
|
+ _msg_type = ""
|
|
|
|
+ for k,v in dict_type.items():
|
|
|
|
+ process_count += v.get("success",0)+v.get("fail",0)
|
|
|
|
+ process_succeed_count += v.get("success",0)
|
|
|
|
+ _msg_type += "\n类型%s\n\t成功%s,消耗%s秒,%s秒/个,\n\t失败%s,消耗%s秒,%s秒/个"%(k,str(v.get("success")),str(v.get("success_costtime")),str(v.get("success_costtime")/max(1,v.get("success"))),str(v.get("fail")),str(v.get("fail_costtime")),str(v.get("fail_costtime")/max(1,v.get("fail"))))
|
|
|
|
+
|
|
_msg = "附件提取队列报警:队列堆积%s条公告,最近十分钟处理公告附件数:%s,处理成功数:%s"%(str(total_count_todeal),str(process_count),str(process_succeed_count))
|
|
_msg = "附件提取队列报警:队列堆积%s条公告,最近十分钟处理公告附件数:%s,处理成功数:%s"%(str(total_count_todeal),str(process_count),str(process_succeed_count))
|
|
- sentMsgToDD(_msg,ACCESS_TOKEN_DATAWORKS)
|
|
|
|
|
|
+ sentMsgToDD(_msg+_msg_type,ACCESS_TOKEN_DATAWORKS)
|
|
# sendEmail(smtp_host,smtp_username,smtp_password,self.recieviers,_msg)
|
|
# sendEmail(smtp_host,smtp_username,smtp_password,self.recieviers,_msg)
|
|
except Exception as e:
|
|
except Exception as e:
|
|
traceback.print_exc()
|
|
traceback.print_exc()
|