

import re

def test_speed(logfile):
    a = open(logfile,"r",encoding="utf8").read()
    set_a = set()
    _c = 0
    for a in re.split("\n",a):
        a = a.strip()
        if a=="":
            continue
        b = a.split(":")[-1]
        _c += 1
        print(b)
        set_a.add(b)
    print(len(set_a),_c)

def check_start_end(logfile):
    list_start = []
    set_end = set()
    log_str = open(logfile,"r",encoding="utf8").read()
    for a in re.split("\n",log_str):
        a = a.strip()
        if a=="":
            continue
        docid = a.split(":")[-1]
        before = ":".join(a.split(":")[:-1])
        if before.find("start on")>=0:
            list_start.append(docid)
        elif before.find("end on")>=0:
            set_end.add(docid)
        elif before.find("error on")>=0:
            set_end.add(docid)

    check_list = list_start[:-1000]
    print("check_list length:",len(check_list))
    for docid in check_list:
        if docid not in set_end:
            print("docid no end",docid)

if __name__ == '__main__':
    logfile = "log.txt"
    test_speed(logfile)
    # check_start_end(logfile)
