monitor_process_config.py 9.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242
  1. import datetime
  2. import logging
  3. import os
  4. import re
  5. import sys
  6. import time
  7. import psutil
  8. sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../")
  9. from format_convert.utils import get_ip_port, get_intranet_ip, get_args_from_config, get_all_ip, get_using_ip
  10. ip_port_dict = get_ip_port()
  11. ip = get_using_ip()
  12. print("local ip:", ip)
  13. # 获取各个参数
  14. convert_port_list = get_args_from_config(ip_port_dict, ip, "convert", "MASTER")
  15. if convert_port_list:
  16. convert_port_list = convert_port_list[0]
  17. ocr_port_list = get_args_from_config(ip_port_dict, ip, "ocr")
  18. otr_port_list = get_args_from_config(ip_port_dict, ip, "otr")
  19. idc_port_list = get_args_from_config(ip_port_dict, ip, "idc")
  20. isr_port_list = get_args_from_config(ip_port_dict, ip, "isr")
  21. atc_port_list = get_args_from_config(ip_port_dict, ip, "atc")
  22. yolo_port_list = get_args_from_config(ip_port_dict, ip, "yolo")
  23. soffice_port_list = get_args_from_config(ip_port_dict, ip, "office", "MASTER")
  24. if soffice_port_list:
  25. soffice_port_list = soffice_port_list[0]
  26. python_path_list = get_args_from_config(ip_port_dict, ip, "python_path")
  27. project_path_list = get_args_from_config(ip_port_dict, ip, "project_path")
  28. gunicorn_path_list = get_args_from_config(ip_port_dict, ip, "gunicorn_path")
  29. std_out = " >>/convert.out 2>&1 &"
  30. std_out_gpu = " >>/gpu.out 2>&1 &"
  31. std_out_schedule = " >>/schedule.out 2>&1 &"
  32. print("convert_port_list", convert_port_list)
  33. print("ocr_port_list", ocr_port_list)
  34. print("otr_port_list", otr_port_list)
  35. print("idc_port_list", idc_port_list)
  36. print("isr_port_list", isr_port_list)
  37. print("atc_port_list", atc_port_list)
  38. print("yolo_port_list", yolo_port_list)
  39. print("soffice_port_list", soffice_port_list)
  40. # 根据port生成gunicorn语句
  41. ocr_comm_list = []
  42. otr_comm_list = []
  43. isr_comm_list = []
  44. idc_comm_list = []
  45. atc_comm_list = []
  46. yolo_comm_list = []
  47. for i in range(len(ocr_port_list)):
  48. ocr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(ocr_port_list[i]))
  49. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  50. + project_path_list[i] + "/ocr ocr_interface:app" + std_out_gpu)
  51. for i in range(len(otr_port_list)):
  52. otr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(otr_port_list[i]))
  53. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  54. + project_path_list[i] + "/otr otr_interface:app" + std_out_gpu)
  55. for i in range(len(idc_port_list)):
  56. idc_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(idc_port_list[i]))
  57. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  58. + project_path_list[i] + "/idc idc_interface:app" + std_out_gpu)
  59. for i in range(len(isr_port_list)):
  60. isr_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(isr_port_list[i]))
  61. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  62. + project_path_list[i] + "/isr isr_interface:app" + std_out_gpu)
  63. for i in range(len(atc_port_list)):
  64. atc_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(atc_port_list[i]))
  65. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  66. + project_path_list[i] + "/atc atc_interface:app" + std_out_gpu)
  67. for i in range(len(yolo_port_list)):
  68. yolo_comm_list.append("nohup " + gunicorn_path_list[i] + " -w " + str(len(yolo_port_list[i]))
  69. + " -t 300 --keep-alive 600 -b 0.0.0.0:# --chdir "
  70. + project_path_list[i] + "/botr/yolov8 yolo_interface:app" + std_out_gpu)
  71. convert_comm = "nohup " + gunicorn_path_list[0] + " -w " + str(len(convert_port_list)) + " -t 300 -b 0.0.0.0:# --chdir " \
  72. + project_path_list[0] + "/format_convert convert:app" + std_out
  73. soffice_comm = "docker run --init -itd --log-opt max-size=10m --log-opt max-file=3 -p #:16000 soffice:v2 bash"
  74. def get_port():
  75. net_conn = psutil.net_connections()
  76. current_port_list = []
  77. for conn in net_conn:
  78. current_port_list.append(str(conn.laddr.port))
  79. current_port_list = list(set(current_port_list))
  80. current_port_list.sort(key=lambda x: x)
  81. return current_port_list
  82. def restart(process_type, port, index=0):
  83. if process_type == "convert":
  84. _comm = re.sub("#", port, convert_comm)
  85. elif process_type == "ocr":
  86. _comm = re.sub("#", port, ocr_comm_list[index])
  87. elif process_type == "otr":
  88. _comm = re.sub("#", port, otr_comm_list[index])
  89. elif process_type == "soffice":
  90. _comm = re.sub("#", port, soffice_comm)
  91. elif process_type == "idc":
  92. _comm = re.sub("#", port, idc_comm_list[index])
  93. elif process_type == "isr":
  94. _comm = re.sub("#", port, isr_comm_list[index])
  95. elif process_type == "atc":
  96. _comm = re.sub("#", port, atc_comm_list[index])
  97. elif process_type == "yolo":
  98. _comm = re.sub("#", port, yolo_comm_list[index])
  99. else:
  100. _comm = "netstat -nltp"
  101. print("no process_type", process_type)
  102. # os.system("echo $(date +%F%n%T)")
  103. print(datetime.datetime.now(), "restart comm", _comm)
  104. os.system(_comm)
  105. def kill_soffice(limit_sec=30):
  106. try:
  107. pid_list = psutil.pids()
  108. for pid in pid_list:
  109. process = psutil.Process(pid)
  110. process_cmd = ''
  111. for c in process.cmdline():
  112. process_cmd += c + " "
  113. if process_cmd.strip() == "":
  114. continue
  115. if process.status() == "zombie":
  116. print("zombie cmd", process_cmd)
  117. if re.search("soffice", process.exe()):
  118. start_time = process.create_time()
  119. now_time = time.time()
  120. run_time = now_time-start_time
  121. if run_time >= limit_sec:
  122. comm = "kill -9 " + str(pid)
  123. print(datetime.datetime.now(), "kill process ", str(pid), str(process.exe()), str(run_time), ">", limit_sec)
  124. os.system(comm)
  125. except:
  126. pass
  127. def kill_nested_timeout_process():
  128. try:
  129. pid_list = psutil.pids()
  130. suspect_pid_list = []
  131. for pid in pid_list:
  132. process = psutil.Process(pid)
  133. process_cmd = ''
  134. for c in process.cmdline():
  135. process_cmd += c + " "
  136. if process_cmd.strip() == "":
  137. continue
  138. if re.search("convert:app", process_cmd):
  139. ppid = process.ppid()
  140. start_time = process.create_time()
  141. now_time = time.time()
  142. run_time = now_time-start_time
  143. if str(ppid) == "1":
  144. suspect_pid_list.append([str(pid), float(run_time)])
  145. # 时间最久的父进程为1的不能杀,是接口主进程
  146. if len(suspect_pid_list) <= 1:
  147. return
  148. else:
  149. suspect_pid_list.sort(key=lambda x: x[1], reverse=True)
  150. for pid, run_time in suspect_pid_list[1:]:
  151. # print("pid", pid, run_time)
  152. comm = "kill -9 " + str(pid)
  153. print(datetime.datetime.now(), "kill process ", str(pid), "father is 1", process_cmd)
  154. os.system(comm)
  155. except:
  156. pass
  157. def monitor():
  158. current_port_list = get_port()
  159. if convert_port_list:
  160. for p in convert_port_list[:1]:
  161. if p not in current_port_list:
  162. restart("convert", p)
  163. if ocr_port_list:
  164. for j in range(len(ocr_port_list)):
  165. for p in ocr_port_list[j][:1]:
  166. if p not in current_port_list:
  167. restart("ocr", p, index=j)
  168. if otr_port_list:
  169. for j in range(len(otr_port_list)):
  170. for p in otr_port_list[j][:1]:
  171. if p not in current_port_list:
  172. restart("otr", p, index=j)
  173. if idc_port_list:
  174. for j in range(len(idc_port_list)):
  175. for p in idc_port_list[j][:1]:
  176. if p not in current_port_list:
  177. restart("idc", p, index=j)
  178. if isr_port_list:
  179. for j in range(len(isr_port_list)):
  180. for p in isr_port_list[j][:1]:
  181. if p not in current_port_list:
  182. restart("isr", p, index=j)
  183. if atc_port_list:
  184. for j in range(len(atc_port_list)):
  185. for p in atc_port_list[j][:1]:
  186. if p not in current_port_list:
  187. restart("atc", p, index=j)
  188. if yolo_port_list:
  189. for j in range(len(yolo_port_list)):
  190. for p in yolo_port_list[j][:1]:
  191. if p not in current_port_list:
  192. restart("yolo", p, index=j)
  193. if soffice_port_list:
  194. for p in soffice_port_list:
  195. if p not in current_port_list:
  196. restart("soffice", p)
  197. kill_soffice()
  198. kill_nested_timeout_process()
  199. # if schedule_port_list:
  200. # for p in schedule_port_list:
  201. # if p not in current_port_list:
  202. # restart("schedule", p)
  203. if __name__ == "__main__":
  204. for i in range(6):
  205. # os.system("echo $(date +%F%n%T)")
  206. monitor()
  207. time.sleep(10)