FORMAT_CONVERSION_MAXCOMPUTE/format_convert/monitor_process_config.py

import datetime
import logging
import os
import re
import sys
import time
import psutil
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
from format_convert.utils import get_ip_port, get_intranet_ip, get_args_from_config, get_all_ip, get_using_ip

# ip_port_dict = get_ip_port()
# ip = "http://" + get_intranet_ip()
# ip = "http://127.0.0.1"
# convert_port_list = ip_port_dict.get(ip).get("convert")
# ocr_port_list = ip_port_dict.get(ip).get("ocr")
# otr_port_list = ip_port_dict.get(ip).get("otr")
# soffice_port_list = ip_port_dict.get(ip).get("office")
# if not convert_port_list:
#     convert_port_list = []
# if not ocr_port_list:
#     ocr_port_list = []
# if not otr_port_list:
#     otr_port_list = []
# if not soffice_port_list:
#     soffice_port_list = []
#
# # schedule_port_list = ip_port_dict.get(ip).get("schedule")
# # python_path = ip_port_dict.get(ip).get("python_path")
# # project_path = ip_port_dict.get(ip).get("project_path")
#
# interface_path = project_path[:-1]
# std_out = " >>/convert.out 2>&1 &"
# std_out_gpu = " >>/gpu.out 2>&1 &"
# std_out_schedule = " >>/schedule.out 2>&1 &"
# # convert_comm = "nohup " + python_path + " " + interface_path + "/format_convert/convert.py #" + std_out
# # ocr_comm = "nohup " + python_path + " " + interface_path + "/ocr/ocr_gpu_interface.py # 0" + std_out_gpu
# # otr_comm = "nohup " + python_path + " " + interface_path + "/otr/otr_gpu_interface.py # 0" + std_out_gpu
# schedule_comm = "nohup " + python_path + " " + interface_path + "/format_convert/schedule_interface.py #" + std_out_schedule
# soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash"
#
#
# gunicorn_path = python_path
# # print("convert_port_list", len(convert_port_list))
# convert_comm = "nohup " + gunicorn_path + " -w " + str(len(convert_port_list)) + " -t 300 -b 0.0.0.0:# --chdir " \
#                + interface_path + "/format_convert convert:app" + std_out
# ocr_comm = "nohup " + gunicorn_path + " -w " + str(len(ocr_port_list)) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " \
#            + interface_path + "/ocr ocr_gpu_interface:app" + std_out_gpu
# otr_comm = "nohup " + gunicorn_path + " -w " + str(len(otr_port_list)) + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir " \
#            + interface_path + "/otr otr_gpu_interface:app" + std_out_gpu


ip_port_dict = get_ip_port()
ip = get_using_ip()
print("local ip:", ip)

convert_port_list = get_args_from_config(ip_port_dict, ip, "convert", "MASTER")
if convert_port_list:
    convert_port_list = convert_port_list[0]
ocr_port_list = get_args_from_config(ip_port_dict, ip, "ocr")
otr_port_list = get_args_from_config(ip_port_dict, ip, "otr")
soffice_port_list = get_args_from_config(ip_port_dict, ip, "office", "MASTER")
if soffice_port_list:
    soffice_port_list = soffice_port_list[0]
python_path_list = get_args_from_config(ip_port_dict, ip, "python_path")
project_path_list = get_args_from_config(ip_port_dict, ip, "project_path")
gunicorn_path_list = get_args_from_config(ip_port_dict, ip, "gunicorn_path")
std_out = " >>/convert.out 2>&1 &"
std_out_gpu = " >>/gpu.out 2>&1 &"
std_out_schedule = " >>/schedule.out 2>&1 &"

print("convert_port_list", convert_port_list)
print("ocr_port_list", ocr_port_list)
print("otr_port_list", otr_port_list)
print("soffice_port_list", soffice_port_list)


ocr_comm_list = []
otr_comm_list = []
for i in range(len(ocr_port_list)):
    ocr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(ocr_port_list[i]))
                         + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
                         + project_path_list[i] + "/ocr ocr_interface:app" + std_out_gpu)
for i in range(len(otr_port_list)):
    otr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(otr_port_list[i]))
                         + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
                         + project_path_list[i] + "/otr otr_interface:app" + std_out_gpu)
convert_comm = "nohup " + gunicorn_path_list[0] + " -w " + str(len(convert_port_list)) + " -t 300 -b 0.0.0.0:# --chdir " \
               + project_path_list[0] + "/format_convert convert:app" + std_out
schedule_comm = "nohup " + python_path_list[0] + " " + project_path_list[0] + "/format_convert/schedule_interface.py #" + std_out_schedule
soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash"


def get_port():
    net_conn = psutil.net_connections()
    current_port_list = []
    for conn in net_conn:
        current_port_list.append(str(conn.laddr.port))
    current_port_list = list(set(current_port_list))
    current_port_list.sort(key=lambda x: x)
    return current_port_list


def restart(process_type, port, index=0):
    if process_type == "convert":
        _comm = re.sub("#", port, convert_comm)
    elif process_type == "ocr":
        _comm = re.sub("#", port, ocr_comm_list[index])
    elif process_type == "otr":
        _comm = re.sub("#", port, otr_comm_list[index])
    elif process_type == "soffice":
        _comm = re.sub("#", port, soffice_comm)
    elif process_type == "schedule":
        _comm = re.sub("#", port, schedule_comm)
    else:
        _comm = "netstat -nltp"
        print("no process_type", process_type)
    # os.system("echo $(date +%F%n%T)")
    print(datetime.datetime.now(), "restart comm", _comm)
    os.system(_comm)


def kill_soffice(limit_sec=30):
    pid_list = psutil.pids()
    for pid in pid_list:
        process = psutil.Process(pid)

        process_cmd = ''
        for c in process.cmdline():
            process_cmd += c + " "
        if process_cmd.strip() == "":
            continue

        if process.status() == "zombie":
            print("zombie cmd", process_cmd)

        if re.search("soffice", process.exe()):
            start_time = process.create_time()
            now_time = time.time()
            run_time = now_time-start_time
            if run_time >= limit_sec:
                comm = "kill -9 " + str(pid)
                print(datetime.datetime.now(), "kill process ", str(pid), str(process.exe()), str(run_time), ">", limit_sec)
                os.system(comm)


def kill_nested_timeout_process():
    pid_list = psutil.pids()
    suspect_pid_list = []
    for pid in pid_list:
        process = psutil.Process(pid)

        process_cmd = ''
        for c in process.cmdline():
            process_cmd += c + " "
        if process_cmd.strip() == "":
            continue

        if re.search("convert:app", process_cmd):
            ppid = process.ppid()
            start_time = process.create_time()
            now_time = time.time()
            run_time = now_time-start_time
            if str(ppid) == "1":
                suspect_pid_list.append([str(pid), float(run_time)])

    # 时间最久的父进程为1的不能杀，是接口主进程
    if len(suspect_pid_list) <= 1:
        return
    else:
        suspect_pid_list.sort(key=lambda x: x[1], reverse=True)
        for pid, run_time in suspect_pid_list[1:]:
            # print("pid", pid, run_time)
            comm = "kill -9 " + str(pid)
            print(datetime.datetime.now(), "kill process ", str(pid), "father is 1", process_cmd)
            os.system(comm)


def monitor():
    current_port_list = get_port()

    if convert_port_list:
        for p in convert_port_list[:1]:
            if p not in current_port_list:
                restart("convert", p)

    if ocr_port_list:
        for j in range(len(ocr_port_list)):
            for p in ocr_port_list[j][:1]:
                if p not in current_port_list:
                    restart("ocr", p, index=j)

    if otr_port_list:
        for j in range(len(otr_port_list)):
            for p in otr_port_list[j][:1]:
                if p not in current_port_list:
                    restart("otr", p, index=j)

    if soffice_port_list:
        for p in soffice_port_list:
            if p not in current_port_list:
                restart("soffice", p)

    kill_soffice()

    kill_nested_timeout_process()

    # if schedule_port_list:
    #     for p in schedule_port_list:
    #         if p not in current_port_list:
    #             restart("schedule", p)


if __name__ == "__main__":
    for i in range(6):
        # os.system("echo $(date +%F%n%T)")
        monitor()
        time.sleep(10)