BIDI_EXTRACTION_AI/BiddingKG/dl/preprocess/article.py

# -*- coding: utf-8 -*-
"""文章预处理。

按 ARCHITECTURE.md Phase 4 拆分建议，从 ``interface/Preprocessing.py`` 迁出。
类型：PREPROCESS。

原位置：``interface/Preprocessing.py`` 中以下函数：
    - ``get_preprocessed`` — 预处理入口（NLP处理、实体识别）
    - ``get_preprocessed_article`` — 文章 HTML 预处理

``interface/Preprocessing.py`` 仍 re-export 以上全部名称，老 import 不受影响。
"""
from __future__ import absolute_import

import re
import time

from bs4 import BeautifulSoup

from BiddingKG.dl.common.logging import log
from BiddingKG.dl.common.Utils import del_tabel_achievement
from BiddingKG.dl.interface.Entitys import Article
from BiddingKG.dl.bidway.re_bidway import extract_bidway, bidway_integrate
from BiddingKG.dl.fingerprint.documentFingerprint import getFingerprint
from BiddingKG.dl.preprocess.html_cleaner import special_treatment, article_limit, attachment_filelink, del_achievement, split_header
from BiddingKG.dl.preprocess.table_parser import tableToText
from BiddingKG.dl.preprocess.segmenter import segment, get_preprocessed_sentences
from BiddingKG.dl.preprocess.outline import get_preprocessed_outline
from BiddingKG.dl.entityLink.entityLink import calibrateEnterprise

__all__ = [
    "get_preprocessed",
    "get_preprocessed_article",
]


def get_preprocessed(articles, useselffool=False):
    '''
    @summary:预处理步骤，NLP处理、实体识别
    @param:
        articles:待处理的文章list [[id,source,jointime,doc_id,title]]
    @return:list of articles,list of each article of sentences,list of each article of entitys
    '''
    from BiddingKG.dl.preprocess.ner import get_preprocessed_entitys
    cost_time = dict()
    list_articles = get_preprocessed_article(articles,cost_time)
    list_sentences,list_outlines = get_preprocessed_sentences(list_articles,True,cost_time)
    list_entitys = get_preprocessed_entitys(list_sentences,True,cost_time)

    calibrateEnterprise(list_articles,list_sentences,list_entitys)

    return list_articles,list_sentences,list_entitys,list_outlines,cost_time


def get_preprocessed_article(articles,cost_time = dict(),useselffool=True):
    '''
    :param articles: 待处理的article source html
    :param useselffool: 是否使用selffool
    :return: list_articles
    '''
    list_articles = []
    for article in articles:
        doc_id = article[0]
        sourceContent = article[1]
        sourceContent_raw = article[1] # 原始html数据，fingerprint计算MD5用
        sourceContent = re.sub("<html>|</html>|<body>|</body>","",sourceContent)
        sourceContent = re.sub("##attachment##","",sourceContent)
        sourceContent = sourceContent.replace('<br/>', '<br>')
        sourceContent = re.sub("<br>(\s{0,}<br>)+","<br>",sourceContent)
        # for br_match in re.findall("[^>]+?<br>",sourceContent):
        #     _new = re.sub("<br>","",br_match)
        #     # <br>标签替换为<p>标签
        #     if not re.search("^\s+$",_new):
        #         _new = '<p>'+_new + '</p>'
        #         # print(br_match,_new)
        #         sourceContent = sourceContent.replace(br_match,_new,1)
        # _send_doc_id = article[3]
        _send_doc_id = article[0] # 20260615 修复 extract_json docid 为空
        _title = article[4]
        _title_raw = article[4]
        page_time = article[5]
        web_source_no = article[6]
        '''特别数据源对 html 做特别修改'''
        if web_source_no in ['DX000202-1']:
            sourceContent = special_treatment(sourceContent, web_source_no)

        #表格处理
        key_preprocess = "tableToText"
        start_time = time.time()
        # article_processed = tableToText(BeautifulSoup(sourceContent,"lxml"))
        article_processed = BeautifulSoup(sourceContent,"lxml")

        if re.search('主要标的数量(&nbsp;|\s)+主要标的单价(（万?元）)?(&nbsp;|\s)+合同金额', sourceContent): #处理 空格分割多个表头的情况
            split_header(article_processed)

        '''表格业绩内容删除'''
        del_tabel_achievement(article_processed)

        '''特别数据源对 BeautifulSoup(html) 做特别修改'''
        if web_source_no in ["00753-14","DX008357-11","18021-2"]:
            article_processed = special_treatment(article_processed, web_source_no)
        for _soup in article_processed.descendants:
            # 识别无标签文本，添加<span>标签
            if not _soup.name and not _soup.parent.string and _soup.string.strip()!="":
                # print(_soup.parent.string,_soup.string.strip())
                _soup.wrap(article_processed.new_tag("span"))
        # print(article_processed)
        # 正文和附件内容限制字数30000
        article_processed = article_limit(article_processed, limit_words=30000)
        # 把每个附件识别对应的html放回原来出现的位置
        article_processed = attachment_filelink(article_processed)

        article_processed = get_preprocessed_outline(article_processed)
        # print('article_processed')
        article_processed = tableToText(article_processed)
        # print(article_processed)
        article_processed = segment(article_processed)

        article_processed = re.sub('[,，]*有[,，]*限[,，]*公[,，]*司', '有限公司', article_processed) # 20250723 去掉由于附件处理导致的不合理逗号

        article_processed = article_processed.replace('(', '（').replace(')', '）')  #2022/8/10 统一为中文括号
        article_processed = article_processed.replace('侯选人', '候选人')  #2024/09/03 修复错别字避免预测错误。
        article_processed = article_processed.replace('人选人', '入选人')  #2024/09/03 修复错别字避免预测错误。
        article_processed = article_processed.replace('⺠', '民')  # 2025/02/17 修复错别字 例：连云港市第一人⺠医院。
        article_processed = article_processed.replace('虫国', '中国')  # 2025/12/23 修复错别字 例：虫国移动通信集团
        # article_processed = article_processed.replace(':', '：')  #2023/1/5 统一为中文冒号
        article_processed = re.sub("(?<=[\u4e00-\u9fa5]):|:(?=[\u4e00-\u9fa5])", "：", article_processed)
        article_processed = article_processed.replace('．','.').replace('－', '-') # 2021/12/01 修正OCR识别PDF小数点错误问题
        article_processed = article_processed.replace('报价限价', '招标限价') #2021/12/17 由于报价限价预测为中投标金额所以修改
        article_processed = article_processed.replace('成交工程价款', '成交工程价')  # 2021/12/21 修正为中标价
        article_processed = re.sub('任务(?=编号[:：])', '项目',article_processed)  # 2022/08/10 修正为项目编号
        article_processed = article_processed.replace('招标（建设）单位', '招标单位')  #2022/8/10 修正预测不到表达
        # article_processed = re.sub("采购商(?=[^\u4e00-\u9fa5]|名称)", "招标人", article_processed) # 20250227注销，避免588296281 03338-7	始兴县人民政府 成交信息：采购商名称： 其实是中标人
        article_processed = re.sub('(招标|采购)人(概况|信息)：?[，。]', '采购人信息：', article_processed)  # 2022/8/10统一表达
        article_processed = article_processed.replace('\（%）', '')    # 中标（成交）金额（元）\（%）：498888.00， 处理 江西省政府采购网  金额特殊问题
        article_processed = re.sub('金额：?（(可填写下浮率?、折扣率?或费率|拟签含税总单价总计|[^万元（）\d]{8,20}）)）?：?', '金额：', article_processed)    # 中标（成交）金额：（可填写下浮率、折扣率或费率）：29.3万元  金额特殊问题 例：530377517
        article_processed = re.sub('（可填写下浮率、折扣率或费率）', '', article_processed) # 785976995
        article_processed = re.sub('（不?含(可抵扣增值|\w{,8})税）', '', article_processed)    # 120637247 投标报价（元），（含可抵扣增值税）：277,560.00。
        article_processed = re.sub('供应商的?(名称[及其、]{1,2}地址|联系方式：名称)', '供应商名称', article_processed)  # 18889217, 84422177
        article_processed = re.sub('，最高有效报价者：', '，中标人名称：', article_processed)  # 224678159 # 2023/7/4 四川站源特殊中标修改
        article_processed = re.sub('，最高有效报价：', '，投标报价：', article_processed)  # 224678159 # 2023/7/4 四川站源特殊中标修改
        article_processed = re.sub('备选中标人', '第二候选人', article_processed)  # 341344142 # 2023/7/17 特殊表达修改
        article_processed = re.sub('例：建设银行（甲方全称）', ' ', article_processed)  # 2024/06/12 特殊表达修改 修改 481513912 金采网 附件模板导致错误提取招标人
        article_processed = re.sub('^[,，.。；;、]+', '', article_processed)
        article_processed = re.sub('资，金', '资金', article_processed)
        article_processed = re.sub('金，额', '金额', article_processed)
        article_processed = re.sub('存，款', '存款', article_processed)
        article_processed = re.sub('（适用于采用评定分离评标的项目）|（根据需要可以填写总价、单价、下浮率、费率等）|业绩（响应招标文件的业绩，多项应分别列出）', '', article_processed) # 修复关键词过远导致召回失败 例：579672495
        article_processed = re.sub('采购商名称：欧冶工业品采购组织', '采购商名称：欧冶工业品股份有限公司', article_processed)
        article_processed = re.sub('发布企业：美的集团，', '发布企业：美的集团股份有限公司，', article_processed) # 20251030修复 596252500 美的集团提取不到
        article_processed = re.sub('采购人：中色国贸，', '采购人：中色国际贸易有限公司，', article_processed) # 20251031修复 631759309 中色国贸提取不到
        article_processed = re.sub(r'：([\w（）]{5,20}有限)(，公司|公?[，。])', r'：\1公司', article_processed) # 修复 711472469 公司提取不到
        if web_source_no.startswith('DX002756-'):
            article_processed = re.sub('状态：(进行中|已结束)单位', '，项目单位', article_processed)  # 376225646
        if web_source_no.startswith('DX006116-') and re.search('结果公告如下：.{5,50}，单位名称：', article_processed):  # 2023/11/20 特殊处理 381591924 381592533 这种提取不到情况
            article_processed = re.sub('，单位名称：', '，供应商名称：', article_processed)
        ser = re.search('(采购|招采|招标|比选|招商)人(名称)?/(采购|招采|招标|比选|招商)?代理机构(名称)?：(?P<tenderee>[\w（）]{4,25}(/[\w（）]{4,25})?)/(?P<agency>[\w（）]{4,25})[，。]', article_processed)
        if ser:
            article_processed = article_processed.replace(ser.group(0), '采购人名称：%s，采购代理机构名称：%s，' % (ser.group('tenderee'), ser.group('agency')))

        ser2 = re.search('(采购|招采|招标|比选|招商)人(名称)?[/或](采购|招采|招标|比选|招商)?代理机构(名称)?：(?P<tenderee>[\w（）]{4,25})[，。/]', article_processed)
        if ser2 and re.search('招投?标|采购|代理|咨询', ser2.group('tenderee'))==None:
            article_processed = article_processed.replace(ser2.group(0), '采购人名称：%s，采购代理机构名称：，' % (
            ser2.group('tenderee')))
        if re.search('中标单位名称：[\w（）]{5,25}，中标候选人名次：\d，', article_processed) and re.search('中标候选人名次：\d，中标单位名称：[\w（）]{5,25}，', article_processed)==None:  # 处理类似 304706608 此篇的数据源正文特殊表达
            for it in re.finditer('(?P<tenderer>(中标单位名称：[\w（）]{5,25}，))(?P<rank>(中标候选人名次：\d，))', article_processed):
                article_processed = article_processed.replace(it.group(0), it.group('rank')+it.group('tenderer'))
        ser = re.search('竞得人：\d{8,15}-', article_processed)
        if ser:
            article_processed = article_processed.replace(ser.group(0), '竞得人：') # 修复类似 368120777 关键词角色被编号隔开情况
        article_processed = re.sub("流出方信息：。", "流出方信息：", article_processed) # 修复 367520674 产权批量表格问题
        idx = article_processed.find('供应商报名、缴纳保证金、下载采购文件流程.docx。##attachment##。') # 修复404230599 E交易站源批量附件中标人错误
        if idx > 1000:
            article_processed = article_processed[:idx]
        for it in re.finditer('[一二三四五六七八九十\d]、中标候选人名称，', article_processed): # 修复大纲类标点导致提取不到，例：515521734
            article_processed = re.sub(it.group(0), it.group(0)[:-1]+'：', article_processed)
        ser = re.search('项目[编代][码号]/项目名称：(?P<code>(\[审批\])?[\d\-]{10,30})/?(?P<name>[\u4e00-\u9fa5（）]{4,35}[，。])', article_processed) # 优化项目编号名称一起写的情况 spxm-53340116.html
        if ser:
            article_processed = article_processed.replace(ser.group(0), '项目代码：%s，项目名称：%s' % (
            ser.group('code'), ser.group('name')))
        article_processed = re.sub('四舍五入至', '', article_processed) # 修复 533537050 ，中标价（四舍五入至万元）：6468万元
        if re.search('推荐供应商：', article_processed) and re.search('入围供应商：', article_processed): # 修复 中国工商银行 类网站 入围的才算中标
            article_processed = article_processed.replace('推荐供应商：', '公司名称：')
        if web_source_no.startswith('DX016489') and re.search('排名', article_processed) and re.search('成交供应商单位名称', article_processed): # 20250219 处理特殊站源有排名却叫成交供应商
            article_processed = article_processed.replace('成交供应商单位名称', '成交候选人单位名称')
        if web_source_no.startswith('DX003027') and re.search('招标单位：中招联合信息股份有限公司', article_processed): # 20250402 处理站源招标人错误 明信阳光采购网
            article_processed = article_processed.replace('招标单位：中招联合信息股份有限公司', '')
        if web_source_no == 'DX013859-1' and re.match('YT\d+', article_processed):
            match = re.search('，(\w{5,25}[，。])$', article_processed)
            if match:
                article_processed = article_processed.replace(match.group(1), '中标候选人：%s'%match.group(1))
        if web_source_no == 'DX005060-1' and re.search('序号：1，公司名称：', article_processed): # 修复
            article_processed = re.sub('序号：1，公司名称：', '发布单位：', article_processed, 1)
            if re.search('寻源方案：[^，。]{4,}，供应商名称：[^，。]{4,}，公示截止时间：', article_processed):
                article_processed = '中标公告，中标结果公示，' + article_processed
        if web_source_no.startswith('31151-') and len(re.split('[，。]', article_processed))<5: # 修复 629493676
            match = re.search('，(?P<company>[\u4e00-\u9fa5（）]{5,25})(-\w{2,15})?：(?P<money>[\d,\.]{2,15}万?元)', article_processed)
            if match:
                article_processed = article_processed.replace(match.group(0), '，供应商：%s，投标金额：%s'%(match.group('company'), match.group('money')))
        match = re.search('20\d+元六', article_processed[:1000]) # 修复 637699732 中国船舶重工集团公司第七0五研究所昆明分部+2025052201元六的询价书的询价结果， 金额错误
        if match:
            article_processed = article_processed.replace(match.group(0), match.group(0)[:-2])
        if web_source_no.startswith('27763-') and re.search('，招标方(名称)?，', article_processed):
            article_processed = re.sub('，招标方(名称)?，', '，招标方名称：', article_processed)
        if web_source_no.startswith('00678-') and re.search('采购方需求信息，', article_processed): # 修复 654313319 站源特殊表达
            article_processed = re.sub('采购方需求信息，', '采购方需求信息：', article_processed)
        if web_source_no == 'Y00092' and re.search('、成交候选人，', article_processed):
            article_processed = re.sub('、成交候选人，', '、成交候选人：', article_processed)
        ser = re.search('(?P<k1>中标人：|成交人：)(?P<k2>[\w]{20,50}(采购|项目)：)', article_processed) # 修复 657847532 成交人：库车景胜新能源环保有限公司垃圾仓进口栈道加装热风幕服务询比采购：河南万和电气有限公司，
        if ser:
            article_processed = article_processed.replace(ser.group(0), '%s%s' % (ser.group('k2'), ser.group('k1')))

        '''去除业绩内容'''
        article_processed = del_achievement(article_processed)
        # 修复OCR金额中“，”、“。”识别错误
        article_processed_list = article_processed.split("##attachment##")
        if len(article_processed_list)>1:
            attachment_text = article_processed_list[1]
            for _match in re.finditer("\d。\d{2}",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("。","."),1)
            # for _match in re.finditer("(\d，\d{3})[,，.]",attachment_text):
            for _match in re.finditer("\d，(?=\d{3}[^\d])",attachment_text):
                _match_text = _match.group()
                attachment_text = attachment_text.replace(_match_text,_match_text.replace("，",","),1)
            article_processed_list[1] = attachment_text
            article_processed = "##attachment##".join(article_processed_list)
        '''特别数据源对 预处理后文本 做特别修改'''
        if web_source_no in ['03786-10', '00076-4', 'DX000105-2', '04080-3', '04080-4', '03761-3', '00695-7',"13740-2", '00811-8', '03795-1', '03795-2', 'DX000726-6','DX008791-1','DX011971']:
            article_processed = special_treatment(article_processed, web_source_no)

        # 提取bidway
        list_bidway = extract_bidway(article_processed, _title)
        if list_bidway:
            bidway = list_bidway[0].get("body")
            # bidway名称统一规范
            bidway = bidway_integrate(bidway)
        else:
            bidway = ""

        # 修正被"，"逗号分隔的时间
        repair_time = re.compile("[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[:时点]，?[0-6]\d分?|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]?，?(?:上午|下午)?，?[0-2]?\d，?[时点]|"
                                 "[12]\d，?\d，?\d，?[-—－―/年]，?[0-1]?\d，?[-—－―/月]，?[0-3]?\d，?[日号]|"
                                 "[0-2]?\d，?:，?[0-6]\d，?:，?[0-6]\d"
                                 )
        for _time in set(re.findall(repair_time,article_processed)):
            if re.search("，",_time):
                _time2 = re.sub("，", "", _time)
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time2)
                if item:
                    _time2 = _time2.replace(item.group(),item.group() + " ")
                article_processed = article_processed.replace(_time, _time2)
            else:
                item = re.search("[12]\d{3}[-—－―/][0-1]?\d[-—－―/][0-3]\d(?=\d)", _time)
                if item:
                    _time2 = _time.replace(item.group(),item.group() + " ")
                    article_processed = article_processed.replace(_time, _time2)
        # print('re_rtime',re.findall(repair_time,article_processed))
        # log(article_processed)

        if key_preprocess not in cost_time:
            cost_time[key_preprocess] = 0
        cost_time[key_preprocess] += round(time.time()-start_time,2)

        #article_processed = article[1]
        _article = Article(doc_id,article_processed,sourceContent,_send_doc_id,_title,
                           bidway=bidway)
        _article.fingerprint = getFingerprint(_title_raw+sourceContent_raw)
        _article.page_time = page_time
        list_articles.append(_article)
    return list_articles