import datetime import logging import os import re import sys import time import psutil sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../") from format_convert.utils import get_ip_port, get_intranet_ip, get_args_from_config # ip_port_dict = get_ip_port() # ip = "http://" + get_intranet_ip() # ip = "http://127.0.0.1" # convert_port_list = ip_port_dict.get(ip).get("convert") # ocr_port_list = ip_port_dict.get(ip).get("ocr") # otr_port_list = ip_port_dict.get(ip).get("otr") # soffice_port_list = ip_port_dict.get(ip).get("office") # if not convert_port_list: # convert_port_list = [] # if not ocr_port_list: # ocr_port_list = [] # if not otr_port_list: # otr_port_list = [] # if not soffice_port_list: # soffice_port_list = [] # # # schedule_port_list = ip_port_dict.get(ip).get("schedule") # # python_path = ip_port_dict.get(ip).get("python_path") # # project_path = ip_port_dict.get(ip).get("project_path") # # interface_path = project_path[:-1] # std_out = " >>/convert.out 2>&1 &" # std_out_gpu = " >>/gpu.out 2>&1 &" # std_out_schedule = " >>/schedule.out 2>&1 &" # # convert_comm = "nohup " + python_path + " " + interface_path + "/format_convert/convert.py #" + std_out # # ocr_comm = "nohup " + python_path + " " + interface_path + "/ocr/ocr_gpu_interface.py # 0" + std_out_gpu # # otr_comm = "nohup " + python_path + " " + interface_path + "/otr/otr_gpu_interface.py # 0" + std_out_gpu # schedule_comm = "nohup " + python_path + " " + interface_path + "/format_convert/schedule_interface.py #" + std_out_schedule # soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash" # # # gunicorn_path = python_path # # print("convert_port_list", len(convert_port_list)) # convert_comm = "nohup " + gunicorn_path + " -w " + str(len(convert_port_list)) + " -t 300 -b 0.0.0.0:# --chdir " \ # + interface_path + "/format_convert convert:app" + std_out # ocr_comm = "nohup " + gunicorn_path + " -w " + str(len(ocr_port_list)) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " \ # + interface_path + "/ocr ocr_gpu_interface:app" + std_out_gpu # otr_comm = "nohup " + gunicorn_path + " -w " + str(len(otr_port_list)) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " \ # + interface_path + "/otr otr_gpu_interface:app" + std_out_gpu ip_port_dict = get_ip_port() ip = "http://127.0.0.1" convert_port_list = get_args_from_config(ip_port_dict, ip, "convert", "MASTER")[0] ocr_port_list = get_args_from_config(ip_port_dict, ip, "ocr") otr_port_list = get_args_from_config(ip_port_dict, ip, "otr") soffice_port_list = get_args_from_config(ip_port_dict, ip, "office", "MASTER")[0] python_path_list = get_args_from_config(ip_port_dict, ip, "python_path") project_path_list = get_args_from_config(ip_port_dict, ip, "project_path") gunicorn_path_list = get_args_from_config(ip_port_dict, ip, "gunicorn_path") std_out = " >>/convert.out 2>&1 &" std_out_gpu = " >>/gpu.out 2>&1 &" std_out_schedule = " >>/schedule.out 2>&1 &" print("ocr_port_list", ocr_port_list) print("otr_port_list", otr_port_list) print("soffice_port_list", soffice_port_list) ocr_comm_list = [] otr_comm_list = [] for i in range(len(ocr_port_list)): ocr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(ocr_port_list[i])) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " + project_path_list[i] + "/ocr ocr_interface:app" + std_out_gpu) for i in range(len(otr_port_list)): otr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(otr_port_list[i])) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " + project_path_list[i] + "/otr otr_interface:app" + std_out_gpu) convert_comm = "nohup " + gunicorn_path_list[0] + " -w " + str(len(convert_port_list)) + " -t 300 -b 0.0.0.0:# --chdir " \ + project_path_list[0] + "/format_convert convert:app" + std_out schedule_comm = "nohup " + python_path_list[0] + " " + project_path_list[0] + "/format_convert/schedule_interface.py #" + std_out_schedule soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash" def get_port(): net_conn = psutil.net_connections() current_port_list = [] for conn in net_conn: current_port_list.append(str(conn.laddr.port)) current_port_list = list(set(current_port_list)) current_port_list.sort(key=lambda x: x) return current_port_list def restart(process_type, port, index=0): if process_type == "convert": _comm = re.sub("#", port, convert_comm) elif process_type == "ocr": _comm = re.sub("#", port, ocr_comm_list[index]) elif process_type == "otr": _comm = re.sub("#", port, otr_comm_list[index]) elif process_type == "soffice": _comm = re.sub("#", port, soffice_comm) elif process_type == "schedule": _comm = re.sub("#", port, schedule_comm) else: _comm = "netstat -nltp" print("no process_type", process_type) # os.system("echo $(date +%F%n%T)") print(datetime.datetime.now(), "restart comm", _comm) os.system(_comm) def kill_soffice(limit_sec=30): pid_list = psutil.pids() for pid in pid_list: process = psutil.Process(pid) process_cmd = '' for c in process.cmdline(): process_cmd += c + " " if process_cmd.strip() == "": continue if process.status() == "zombie": print("zombie cmd", process_cmd) if re.search("soffice", process.exe()): start_time = process.create_time() now_time = time.time() run_time = now_time-start_time if run_time >= limit_sec: comm = "kill -9 " + str(pid) print(datetime.datetime.now(), "kill process ", str(pid), str(process.exe()), str(run_time), ">", limit_sec) os.system(comm) def kill_nested_timeout_process(): pid_list = psutil.pids() suspect_pid_list = [] for pid in pid_list: process = psutil.Process(pid) process_cmd = '' for c in process.cmdline(): process_cmd += c + " " if process_cmd.strip() == "": continue if re.search("convert:app", process_cmd): ppid = process.ppid() start_time = process.create_time() now_time = time.time() run_time = now_time-start_time if str(ppid) == "1": suspect_pid_list.append([str(pid), float(run_time)]) # 时间最久的父进程为1的不能杀，是接口主进程 if len(suspect_pid_list) <= 1: return else: suspect_pid_list.sort(key=lambda x: x[1], reverse=True) for pid, run_time in suspect_pid_list[1:]: # print("pid", pid, run_time) comm = "kill -9 " + str(pid) print(datetime.datetime.now(), "kill process ", str(pid), "father is 1", process_cmd) os.system(comm) def monitor(): current_port_list = get_port() if convert_port_list: for p in convert_port_list[:1]: if p not in current_port_list: restart("convert", p) if ocr_port_list: for j in range(len(ocr_port_list)): for p in ocr_port_list[j][:1]: if p not in current_port_list: restart("ocr", p, index=j) if otr_port_list: for j in range(len(otr_port_list)): for p in otr_port_list[j][:1]: if p not in current_port_list: restart("otr", p, index=j) if soffice_port_list: for p in soffice_port_list: if p not in current_port_list: restart("soffice", p) kill_soffice() kill_nested_timeout_process() # if schedule_port_list: # for p in schedule_port_list: # if p not in current_port_list: # restart("schedule", p) if __name__ == "__main__": for i in range(6): # os.system("echo $(date +%F%n%T)") monitor() time.sleep(10)