import logging import os import re import time import psutil convert_port_list = ["15010"] # ocr_port_list = ["15011", "15013", "15015"] # ocr_port_list = ["15011", "15013", "15015", "15017", "15019"] # otr_port_list = ["15012", "15014", "15016", "15018", "15020"] ocr_port_list = ["15011", "15013", "15015", "15017", "15019", "15021"] otr_port_list = ["15012", "15014", "15016", "15018", "15020", "15022"] soffice_port_list = ["16000", "16001", "16002", "16003", "16004", "16005", "16006", "16007", "16008", "16009"] python_path = "/root/miniconda3/bin/python" interface_path = "/data/format_conversion_maxcompute" std_out = " >>/convert.out 2>&1 &" std_out_gpu = " >>/gpu.out 2>&1 &" convert_comm = "nohup " + python_path + " " + interface_path + "/format_convert/convert.py #" + std_out ocr_comm = "nohup " + python_path + " " + interface_path + "/ocr/ocr_interface.py # 0" + std_out_gpu otr_comm = "nohup " + python_path + " " + interface_path + "/otr/otr_interface.py # 0" + std_out_gpu soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash" def get_port(): net_conn = psutil.net_connections() current_port_list = [] for conn in net_conn: current_port_list.append(str(conn.laddr.port)) current_port_list = list(set(current_port_list)) current_port_list.sort(key=lambda x: x) # print(current_port_list) return current_port_list def restart(process_type, port): if process_type == "convert": _comm = re.sub("#", port, convert_comm) elif process_type == "ocr": _comm = re.sub("#", port, ocr_comm) elif process_type == "otr": _comm = re.sub("#", port, otr_comm) elif process_type == "soffice": _comm = re.sub("#", port, soffice_comm) else: _comm = "netstat -nltp" print("no process_type", process_type) # os.system("netstat -nltp") os.system("echo $(date +%F%n%T)") print("restart comm", _comm) os.system(_comm) def kill_soffice(limit_sec=20): pid_list = psutil.pids() for pid in pid_list: process = psutil.Process(pid) process_cmd = '' for c in process.cmdline(): process_cmd += c + " " if process_cmd.strip() == "": continue if process.status() == "zombie": print("zombie cmd", process_cmd) if re.search("soffice", process.exe()): if process.status() == "zombie": ppid = process.ppid comm = "kill -9 " + str(ppid) print("kill defunct process ", str(ppid), str(process.exe())) os.system("echo $(date +%F%n%T)") os.system(comm) start_time = process.create_time() now_time = time.time() run_time = now_time-start_time if run_time >= limit_sec: comm = "kill -9 " + str(pid) print("kill process ", str(pid), str(process.exe()), str(run_time), ">", limit_sec) os.system("echo $(date +%F%n%T)") os.system(comm) def kill_defunct(): pid_list = psutil.pids() for pid in pid_list: process = psutil.Process(pid) if process.status() == "zombie": ppid = process.ppid process = psutil.Process(ppid) process.kill() process.send_signal(9) break # comm = "kill -9 " + str(ppid) # print("kill process ", str(ppid)) # os.system("echo $(date +%F%n%T)") # os.system(comm) def monitor(): current_port_list = get_port() for p in convert_port_list: if p not in current_port_list: restart("convert", p) for p in ocr_port_list: if p not in current_port_list: restart("ocr", p) for p in otr_port_list: if p not in current_port_list: restart("otr", p) for p in soffice_port_list: if p not in current_port_list: restart("soffice", p) kill_soffice() if __name__ == "__main__": for i in range(6): # os.system("echo $(date +%F%n%T)") monitor() time.sleep(10) # kill_defunct()