FORMAT_CONVERSION_MAXCOMPUTE/format_convert/monitor_process2.py

import logging
import os
import re
import time

import psutil


convert_port_list = ["15010"]
# ocr_port_list = ["15011", "15013", "15015"]
# ocr_port_list = ["15011", "15013", "15015", "15017", "15019"]
# otr_port_list = ["15012", "15014", "15016", "15018", "15020"]
ocr_port_list = ["15011", "15013", "15015", "15017", "15019", "15021"]
otr_port_list = ["15012", "15014", "15016", "15018", "15020", "15022"]
soffice_port_list = ["16000", "16001", "16002", "16003", "16004", "16005",
                     "16006", "16007", "16008", "16009"]


python_path = "/root/miniconda3/bin/python"
interface_path = "/data/format_conversion_maxcompute"
std_out = " >>/convert.out 2>&1 &"
std_out_gpu = " >>/gpu.out 2>&1 &"
convert_comm = "nohup " + python_path + " " + interface_path + "/format_convert/convert.py #" + std_out
ocr_comm = "nohup " + python_path + " " + interface_path + "/ocr/ocr_interface.py # 0" + std_out_gpu
otr_comm = "nohup " + python_path + " " + interface_path + "/otr/otr_interface.py # 0" + std_out_gpu
soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash"


def get_port():
    net_conn = psutil.net_connections()
    current_port_list = []
    for conn in net_conn:
        current_port_list.append(str(conn.laddr.port))
    current_port_list = list(set(current_port_list))
    current_port_list.sort(key=lambda x: x)
    # print(current_port_list)
    return current_port_list


def restart(process_type, port):
    if process_type == "convert":
        _comm = re.sub("#", port, convert_comm)
    elif process_type == "ocr":
        _comm = re.sub("#", port, ocr_comm)
    elif process_type == "otr":
        _comm = re.sub("#", port, otr_comm)
    elif process_type == "soffice":
        _comm = re.sub("#", port, soffice_comm)
    else:
        _comm = "netstat -nltp"
        print("no process_type", process_type)

    # os.system("netstat -nltp")
    os.system("echo $(date +%F%n%T)")
    print("restart comm", _comm)
    os.system(_comm)


def kill_soffice(limit_sec=20):
    pid_list = psutil.pids()
    for pid in pid_list:
        process = psutil.Process(pid)

        process_cmd = ''
        for c in process.cmdline():
            process_cmd += c + " "
        if process_cmd.strip() == "":
            continue

        if process.status() == "zombie":
            print("zombie cmd", process_cmd)

        if re.search("soffice", process.exe()):
            if process.status() == "zombie":
                ppid = process.ppid
                comm = "kill -9 " + str(ppid)
                print("kill defunct process ", str(ppid), str(process.exe()))
                os.system("echo $(date +%F%n%T)")
                os.system(comm)

            start_time = process.create_time()
            now_time = time.time()
            run_time = now_time-start_time
            if run_time >= limit_sec:
                comm = "kill -9 " + str(pid)
                print("kill process ", str(pid), str(process.exe()), str(run_time), ">", limit_sec)
                os.system("echo $(date +%F%n%T)")
                os.system(comm)


def kill_defunct():
    pid_list = psutil.pids()
    for pid in pid_list:
        process = psutil.Process(pid)
        if process.status() == "zombie":
            ppid = process.ppid
            process = psutil.Process(ppid)
            process.kill()
            process.send_signal(9)
            break
            # comm = "kill -9 " + str(ppid)
            # print("kill process ", str(ppid))
            # os.system("echo $(date +%F%n%T)")
            # os.system(comm)


def monitor():
    current_port_list = get_port()

    for p in convert_port_list:
        if p not in current_port_list:
            restart("convert", p)

    for p in ocr_port_list:
        if p not in current_port_list:
            restart("ocr", p)

    for p in otr_port_list:
        if p not in current_port_list:
            restart("otr", p)

    for p in soffice_port_list:
        if p not in current_port_list:
            restart("soffice", p)

    kill_soffice()


if __name__ == "__main__":
    for i in range(6):
        # os.system("echo $(date +%F%n%T)")
        monitor()
        time.sleep(10)
    # kill_defunct()