FORMAT_CONVERSION_MAXCOMPUTE/monitor/monitor_memory.py

import os
import time
import psutil
import signal
from datetime import datetime

MEMORY_LIMIT = 10 * 1024 * 1024 * 1024  # 10GB
CHECK_INTERVAL = 1  # seconds


def check_memory():
    for proc in psutil.process_iter(['pid', 'memory_info', 'cmdline']):
        if '/data/fangjiasheng/format_conversion_maxcompute/format_convert' not in str(proc.info['cmdline']):
            continue
        if proc.info['memory_info'].rss > MEMORY_LIMIT:
            # os.kill(proc.info['pid'], signal.SIGTERM)
            print(proc.info['pid'], 'memory', round(proc.info['memory_info'].rss / 1024 / 1024 / 1024, 2), 'GB >', '15GB', proc.info['cmdline'][-2])
            os.system('kill -9 ' + str(proc.info['pid']))
            print('killed', str(proc.info['pid']), datetime.now())
        # else:
            # print(proc.info['pid'], 'memory', round(proc.info['memory_info'].rss / 1024 / 1024 / 1024, 2), 'GB', proc.info['cmdline'][-2])


if __name__ == '__main__':
    while True:
        check_memory()
        time.sleep(CHECK_INTERVAL)
        # print('loop')