BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/outline_extractor.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
@author: bidikeji
@time: 2024/7/19 10:05
"""
import re
from BiddingKG.dl.interface.htmlparser import ParseDocument,get_childs

class Sentence2():
    def __init__(self,text,sentence_index,wordOffset_begin,wordOffset_end):
        self.name = 'sentence2'
        self.text = text
        self.sentence_index = sentence_index
        self.wordOffset_begin = wordOffset_begin
        self.wordOffset_end = wordOffset_end

    def get_text(self):
        return self.text

def extract_sentence_list(sentence_list):
    new_sentence2_list = []
    new_sentence2_list_attach = []
    for sentence in sentence_list:
        sentence_index = sentence.sentence_index
        sentence_text = sentence.sentence_text
        begin_index = 0
        end_index = 0
        for it in re.finditer('([^一二三四五六七八九十，。][一二三四五六七八九十]{1,3}|[^\d\.、，。a-zA-Z]\d{1,2}(\.\d{1,2}){,2})、', sentence_text): # 例：289699210 1、招标内容：滑触线及配件2、招标品牌：3、参标供应商经营形式要求：厂家4、参标供应商资质要求：5、
            temp = it.group(0)
            sentence_text = sentence_text.replace(temp, temp[0] + '，' + temp[1:])
        for item in re.finditer('[，。；;!！？]+', sentence_text): # 20240725去掉英文问号，避免网址被分隔
            end_index = item.end()
            # if end_index!=len(sentence_text):
            #     # if end_index-begin_index<6 and item.group(0) in ['，', ';', '；'] and re.match('[一二三四五六七八九十\d.]+、', sentence_text[begin_index:end_index])==None: # 20240725 注销，避免标题提取错误
            #     #     continue
            if end_index != len(sentence_text) and re.match('[一二三四五六七八九十\d.]{1,2}[、，.]+$', sentence_text[begin_index:end_index]): # 避免表格序号和内容在不同表格情况 例：293178161
                continue
            new_sentence_text = sentence_text[begin_index:end_index]
            sentence2 = Sentence2(new_sentence_text,sentence_index,begin_index,end_index)
            if sentence.in_attachment:
                new_sentence2_list_attach.append(sentence2)
            else:
                new_sentence2_list.append(sentence2)
            begin_index = end_index
        if end_index!=len(sentence_text):
            end_index = len(sentence_text)
            new_sentence_text = sentence_text[begin_index:end_index]
            sentence2 = Sentence2(new_sentence_text, sentence_index, begin_index, end_index)
            if sentence.in_attachment:
                new_sentence2_list_attach.append(sentence2)
            else:
                new_sentence2_list.append(sentence2)
    return new_sentence2_list, new_sentence2_list_attach

requirement_pattern = "(采购需求|需求分析|项目说明|(采购|合同|招标|询比?价|项目|服务|工程|标的|需求|建设)(的?(主要|简要|基本|具体|名称及))?" \
                          "(内容|概况|概述|范围|信息|规模|简介|介绍|说明|摘要|情况)([及与和]((其它|\w{,2})[要需]求|发包范围|数量))?" \
                      "|招标项目技术要求|服务要求|服务需求|项目目标|需求内容如下|建设规模)为?([:：，]|$)"
aptitude_pattern = "资质（资格）要求|资格（资质）要求|单位要求|资质及业绩要求|((资格|资质|准入)[的及]?(要求|条件|标准|限定|门槛)|竞买资格及要求|供应商报价须知)|按以下要求参与竞买|((报名|应征|竞买|投标|竞投|受让|报价|竞价|竞包|竞租|承租|申请|参与|参选|遴选)的?(人|方|单位|企业|客户|机构)?|供应商|受让方)((必?须|需|应[该当]?)(具备|满足|符合|提供)+以?下?)?的?(一般|基本|主要)?(条件|要求|资格(能力)?|资质)+|乙方应当符合下列要求|参与比选条件|合格的投标人|询价要求"

addr_bidopen_pattern = "([开评]标|开启|评选|比选|磋商|遴选|寻源|采购|招标|竞价|议价|委托|询比?价|比价|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选|递交\w{,4}文件)[)）]?(时间[与及和、])?(地址|地点)([与及和、]时间)?([:：，]|$)|开启([:：，]|$)"
addr_bidsend_pattern = "((\w{,4}文件)?(提交|递交)(\w{,4}文件)?|投标)(截止时间[与及和、])?地[点址]([与及和、]截止时间)?([:：，]|$)"
pinmu_name_pattern = "采购品目(名称)?([:：，]|$)"
policy_pattern = "《.+?(通知|办法|条例|规定|规程|规范|须知|规则|标准|细则|意见|协议|条件|要求|手册|法典|方案|指南|指引|法)》"
not_policy_pattern = "(表|函|书|证|\d页|公告|合同|文件|清单)》$|采购合同|响应方须知|响应文件格式|营业执照|开标一览|采购需求"

def extract_parameters(parse_document):
    '''
    通过大纲、预处理后文本正则获取需要字段
    :param parse_document: ParseDocument() 方法返回结果
    :return:
    '''
    list_data = parse_document.tree
    requirement_text = '' # 采购内容
    aptitude_text = '' # 资质要求
    addr_bidopen_text = '' # 开标地址
    addr_bidsend_text = '' # 投标地址
    requirement_scope = [] # 采购内容始末位置
    pinmu_name = '' # 品目名称
    list_policy = [] # 政策法规
    out_lines = []

    _find_count = 0
    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            if _data["sentence_title"] is not None:
                if re.search('[（(][一二三四五六七八九十}]+[)）]|[一二三四五六七八九十]+\s*、|^\d{1,2}[.、][\u4e00-\u9fa5]', _text[:10]):
                    idx = _text.replace(':', '：').find('：')
                    outline_text = _text[:idx] if idx >= 4 else _text
                    out_lines.append((outline_text, _data['sentence_index'], _data['wordOffset_begin']))

                if re.search(requirement_pattern,_text[:30]) is not None and re.search('符合采购需求，', _text[:30])==None:
                    b = (_data['sentence_index'], _data['wordOffset_begin'])
                    childs = get_childs([_data])
                    for c in childs:
                        # requirement_text += c["text"]+"\n"
                        requirement_text += c["text"]
                    e = (c['sentence_index'], c["wordOffset_end"]) if len(childs)>0 else (_data['sentence_index'], _data['wordOffset_end'])
                    requirement_scope.append(b)
                    requirement_scope.append(e)
                    _data_i += len(childs)
                    _data_i -= 1
    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            # print("aptitude_pattern", _text)
            if _data["sentence_title"] is not None:
                # print("aptitude_pattern",_text)

                # outline = re.sub('（?[一二三四五六七八九十\d.]+）?\s*、?', '',
                #                  re.split('[：:，]', _text)[0].replace('(', '（').replace(')', '）'))

                if re.search(aptitude_pattern,_text[:15]) is not None:
                    childs = get_childs([_data])
                    for c in childs:
                        aptitude_text += c["text"]
                        # if c["sentence_title"]:
                        #     aptitude_text += c["text"]+"\n"
                        # else:
                        #     aptitude_text += c["text"]
                    _data_i += len(childs)
                    _data_i -= 1

                # elif re.match('[（(\s★▲\*]?[一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+', _text) and len(_text)<30 and re.search('资质|资格', _text):
                #     out_lines.append(outline)
        if _type=="table":
            list_table = _data["list_table"]
            parent_title = _data["parent_title"]
            if list_table is not None:
                for line in list_table[:2]:
                    for cell_i in range(len(line)):
                        cell = line[cell_i]
                        cell_text = cell[0]
                        if len(cell_text)>120 and re.search(aptitude_pattern,cell_text) is not None:
                            aptitude_text += cell_text+"\n"
    _data_i = -1
    while _data_i < len(list_data) - 1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type == "sentence":
            if _data["sentence_title"] is not None:
                if re.search(addr_bidopen_pattern, _text[:20]) is not None:
                    childs = get_childs([_data], max_depth=1)
                    for c in childs:
                        addr_bidopen_text += c["text"]
                    _data_i += len(childs)
                    _data_i -= 1
                elif re.search(addr_bidsend_pattern, _text[:20]):
                    childs = get_childs([_data], max_depth=1)
                    for c in childs:
                        addr_bidsend_text += c["text"]
                    _data_i += len(childs)
                    _data_i -= 1
                elif re.search(pinmu_name_pattern, _text):
                    childs = get_childs([_data], max_depth=1)
                    for c in childs:
                        pinmu_name += c["text"]
                    _data_i += len(childs)
                    _data_i -= 1
    _data_i = -1
    while _data_i<len(list_data)-1:
        _data_i += 1
        _data = list_data[_data_i]
        _type = _data["type"]
        _text = _data["text"].strip()
        # print(_data.keys())
        if _type=="sentence":
            for it in re.finditer(policy_pattern, _text):
                if it not in list_policy:
                    list_policy.append(it.group(0))

    ser = re.search('地[址点][：为](?P<addr>([\w（）()【】]{2,25}([省市县区州旗]|采购网|平台|公司)[\w（）()【】-]{,60}))[，。]', addr_bidopen_text)
    addr_bidopen_text = ser.group('addr') if ser else ''

    ser = re.search('地[址点][：为](?P<addr>([\w（）()【】]{2,25}([省市县区州旗]|采购网|平台|公司)[\w（）()【】-]{,60}))[，。]', addr_bidsend_text)
    addr_bidsend_text = ser.group('addr') if ser else ''
    if re.search('开启', addr_bidopen_text) and re.search('时间：\d{2,4}年\d{1,2}月\d{1,2}日', addr_bidopen_text) and len(addr_bidopen_text)<40: # 优化类似 364991684只有时间没地址情况
        addr_bidopen_text = ""
    ser = re.search(pinmu_name_pattern, pinmu_name)
    if ser:
        pinmu_name = pinmu_name[ser.end():]
        if re.search('[^\w]$', pinmu_name):
            pinmu_name = pinmu_name[:-1]
    return requirement_text, aptitude_text, addr_bidopen_text, addr_bidsend_text, out_lines, requirement_scope, pinmu_name, list_policy

def extract_addr(content):
    '''
    通过正则提取地址
    :param content:  公告预处理后文本
    :return:
    '''
    addr_bidopen_text = ''
    ser = re.search('([开评]标|开启|评选|比选|磋商|遴选|寻源|采购|招标|竞价|议价|委托|询比?价|比价|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选|递交\w{,4}文件)）?(会议)?地[点址]([(（]网址[)）])?[：为][^，；。]{2,100}[，；。]', content)
    if ser:
        addr_bidopen_text = ser.group(0)
    return addr_bidopen_text

if __name__ == "__main__":
    # with open('D:\html/2.html', 'r', encoding='UTF-8') as f:
    #     html = f.read()
    #
    l = []
    import pandas as pd
    from collections import Counter
    from BiddingKG.dl.interface import Preprocessing
    from BiddingKG.dl.interface.get_label_dic import get_all_label
    from bs4 import BeautifulSoup
    import json

    # df = pd.read_excel('E:/公告招标内容提取结果2.xlsx')
    # df['len']= df['招标内容'].apply(lambda x: len(x))
    # print(len(df), sum(df['len']),sum(df['len'])/len(df), max(df['len']), min(df['len']))
    # print(len([it for it in df['len'] if it>1500]))

    # df = pd.read_csv(r'E:\channel分类数据\2022年每月两天数据/指定日期_html2022-12-10.csv')
    # df1 = pd.read_excel('E:/公告招标内容提取结果.xlsx')
    # df = df[df['docid'].isin(df1['docid'])]
    #
    # df.drop_duplicates(subset=['docchannel', 'web_source_name', 'exist_table'], inplace=True)
    # print(df.columns, len(df))
    #
    #
    # # def get_text(html):
    # #     soup = BeautifulSoup(html, 'lxml')
    # #     text = soup.get_text()
    # #     return text
    # # df['content'] = df['dochtmlcon'].apply(lambda x: get_text(x))
    # # df['标签'] = df.apply(lambda x: get_all_label(x['doctitle'], x['content']), axis=1)
    # # df['标签'] = df['标签'].apply(lambda x: json.dumps(x, ensure_ascii=False, indent=2))
    # # df1 = df[['docid', '标签']]
    #
    # n = 0
    # datas = []
    # for id,title, html in zip(df['docid'],df['doctitle'], df['dochtmlcon']):
    #     # if id not in [289647738, 289647739]:
    #     #     continue
    #     # print(id, type(id))
    #     # parse_document = ParseDocument(html, True)
    #     # requirement_text, aptitude_text = extract_parameters(parse_document)
    #     # if re.search('资\s*[格质]', html)==None:
    #     #     continue
    #
    #     list_articles, list_sentences, list_entitys, list_outlines, _cost_time = Preprocessing.get_preprocessed([[id,html,"","",title,'', '']],useselffool=True)
    #     sentence2_list, sentence2_list_attach = extract_sentence_list(list_sentences[0])
    #
    #     # sentence2_list = []
    #
    #     parse_document = ParseDocument(html, True, list_obj=sentence2_list)
    #     requirement_text, aptitude_text = extract_parameters(parse_document)
    #     # if len(aptitude_text)>0:
    #     #     datas.append((id, aptitude_text[:1500]))
    #     #     print(id, aptitude_text[:10], aptitude_text[-20:])
    #     # else:
    #     #     parse_document = ParseDocument(html, True, list_obj=sentence2_list_attach)
    #     #     requirement_text, aptitude_text = extract_parameters(parse_document)
    #
    #     # if 0<len(aptitude_text)<20:
    #     #     l.append(len(aptitude_text))
    #     #     n += 1
    #     #     print(id, aptitude_text)
    #     #     if n > 5:
    #     #         break
    #
    #     if len(requirement_text)>0:
    #         label_dic = get_all_label(title, list_articles[0].content)
    #         # datas.append((id, requirement_text))
    #         datas.append((id, requirement_text, label_dic))
    #
    # c = Counter(out_lines)
    # print(c.most_common(1000))
    # #
    # # df = pd.DataFrame(datas, columns=['docid', '资质要求'])
    # # df.to_excel('E:/公告资质要求提取结果.xlsx')
    #
    # df = pd.DataFrame(datas, columns=['docid', '招标内容', '标签'])
    # df['标签'] = df['标签'].apply(lambda x: json.dumps(x, ensure_ascii=False, indent=2))
    # df.to_excel('E:/公告招标内容提取结果2.xlsx')

    #     if len(aptitude_text)> 1000:
    #         print(id, aptitude_text[:10], aptitude_text[-20:])
    # print(Counter(l).most_common(50))
    # print(len(df), len(l), min(l), max(l), sum(l)/len(l))
    # n1 = len([it for it in l if it < 500])
    # n2 = len([it for it in l if it < 1000])
    # n3 = len([it for it in l if it < 1500])
    # n4 = len([it for it in l if it < 2000])
    # print(n1, n2, n3, n4, n1/len(l), n2/len(l), n3/len(l), n4/len(l))

    # parse_document = ParseDocument(html,True)
    # requirement_text, new_list_policy, aptitude_text = extract_parameters(parse_document)
    # print(aptitude_text)

    # sentence_text = '5、要求：3.1投标其他条件：1、中国宝武集团项目未列入禁入名单的投标人。2、具有有效的营业执照；'
    # begin_index = 0
    # for item in re.finditer('[，。；;!！？?]+', sentence_text):
    #     end_index = item.end()
    #     if end_index != len(sentence_text):
    #         if end_index - begin_index < 6:
    #             continue
    #     new_sentence_text = sentence_text[begin_index:end_index]
    #     print(new_sentence_text)

    # df = pd.read_excel('E:/公告资质要求提取结果.xlsx')
    # docids = []
    # pos = neg = 0
    # for docid, text in zip(df['docid'], df['资质要求']):
    #     if re.match('[（(\s★▲\*]?[一二三四五六七八九十\dⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫ]+', text) and re.search(aptitude_pattern, text[:15]):
    #         pos += 1
    #         pass
    #     else:
    #         neg += 1
    #         print(docid, text[:50])
    #         docids.append(docid)
    # print('异常：%d, 正常：%d'%(neg, pos))
    # print(docids)