

from BaseDataMaintenance.maintenance.product.htmlparser import *
from BaseDataMaintenance.common.Utils import getLegal_str


requirement_pattern = "(采购需求|需求分析|项目说明|合同内容|招标项目技术要求|采购内容及其它要求|服务要求|招标内容[及与和]要求|服务需求|采购内容|项目概况|项目目标|项目服务内容|项目服务范围|需求内容如下)([:：]|$)"
policy_pattern = "《.+?》"
not_policy_pattern = "(表|函|书|证)》$|采购合同|响应方须知|响应文件格式|营业执照|开标一览|采购需求"
aptitude_pattern = "(资格要求|资质要求)([:：]|$)"
def extract_parameters(_html):


    pd = ParseDocument(_html,True)

    list_data = pd.tree
    list_result = []



    requirement_text = ''
    list_policy = []
    aptitude_text = ''

    _find_count = 0
    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            if _data["sentence_title"] is not None:
                if re.search(requirement_pattern,_text) is not None:

                    childs = get_childs([_data])
                    for c in childs:
                        requirement_text += c["text"]+"\n"
                    _data_i += len(childs)
    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            if _data["sentence_title"] is not None:
                print("aptitude_pattern",_text)
                if re.search(aptitude_pattern,_text) is not None:
                    childs = get_childs([_data])

                    for c in childs:
                        aptitude_text += c["text"]+"\n"
                    _data_i += len(childs)

        if _type=="table":
            list_table = _data["list_table"]
            parent_title = _data["parent_title"]
            if list_table is not None:
                for line in list_table[:2]:
                    for cell_i in range(len(line)):
                        cell = line[cell_i]
                        cell_text = cell[0]
                        if len(cell_text)>120 and re.search(aptitude_pattern,cell_text) is not None:
                            aptitude_text += cell_text+"\n"

    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            if re.search(policy_pattern,_text) is not None:
                for t in re.findall(policy_pattern,_text):
                    if len(t)>0:
                        list_policy.append(t)


    list_policy = list(set(list_policy))
    new_list_policy = []
    for _policy in list_policy:
        if re.search(not_policy_pattern,_policy) is None:
            new_list_policy.append(_policy)
    return requirement_text,new_list_policy,aptitude_text


def valid_xml_char_ordinal(c):
    codepoint = ord(c)
    # conditions ordered by presumed frequency
    return (
            0x20 <= codepoint <= 0xD7FF or
            codepoint in (0x9, 0xA, 0xD) or
            0xE000 <= codepoint <= 0xFFFD or
            0x10000 <= codepoint <= 0x10FFFF
    )


def wash_data(text):
    cleaned_string = ''.join(c for c in text if valid_xml_char_ordinal(c))
    return cleaned_string

def extract_datas():
    import glob
    import pandas as pd

    list_data = []
    for file in glob.glob(r"C:\Users\Administrator\Desktop\html_output\*.html"):
        _content = open(file,"r",encoding="utf-8").read()
        filename = file.split("\\")[-1]
        # if filename!='2023年泗门镇公共视频图像通信链路技术服务采购项目.html':
        #     continue
        requirement_text,list_policy,aptitude_text = extract_parameters(_content)
        print("requirement_text",requirement_text)
        print("list_policy",list_policy)
        print("aptitude_text",aptitude_text)
        list_data.append([wash_data(filename),wash_data(requirement_text),wash_data(",".join(list_policy)),wash_data(aptitude_text)])
    df = pd.DataFrame(list_data,columns=["文件名","采购需求","采购政策","资质要求"])
    df.to_excel("提取结果.xlsx",columns=["文件名","采购需求","采购政策","资质要求"])

def extract_datas_1():
    import glob
    import pandas as pd

    list_data = []
    for file in glob.glob(r"C:\Users\Administrator\Desktop\html_output1\*.html"):
        _content = open(file,"r",encoding="utf-8").read()
        filename = file.split("\\")[-1].split(".")[0]
        # if filename!='2023年泗门镇公共视频图像通信链路技术服务采购项目.html':
        #     continue
        requirement_text,list_policy,aptitude_text = extract_parameters(_content)
        print("requirement_text",requirement_text)
        print("list_policy",list_policy)
        print("aptitude_text",aptitude_text)
        # list_data.append([wash_data(filename),wash_data(requirement_text),wash_data(",".join(list_policy)),wash_data(aptitude_text)])
        list_data.append([wash_data(filename),wash_data(",".join(list_policy))])
    df = pd.DataFrame(list_data,columns=["文件名","采购政策"])
    df.to_excel("提取结果1.xlsx",columns=["文件名","采购政策"])

if __name__ == '__main__':
    # extract_datas()
    extract_datas_1()
    # print(re.search(aptitude_pattern,'二、申请人的资格要求:'))
