BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/proposed_building/pb_extract.py

import copy
import os
import re
import sys
import traceback
from decimal import Decimal
import pandas as pd
from bs4 import BeautifulSoup

sys.path.append(os.path.abspath(os.path.dirname(__file__)) + '/../../../')

from BiddingKG.dl.common.Utils import spanWindow, timeFormat


class PBPredictor:
    def __init__(self):
        self.stage_pattern, self.stage_priority_dict = get_stage_pattern()
        self.industry_pattern = get_industry_pattern()
        self.property_pattern, self.property_priority_dict = get_property_pattern()
        with open(os.path.abspath(os.path.dirname(__file__)) + '/structure_keyword.txt', 'r', encoding='utf-8') as f:
            self.structure_keyword_list = f.readlines()

    def get_col_from_prem(self, prem):
        tenderee, agency, product = None, None, None
        for item in prem:
            prem = item.get('prem')
            for key in prem.keys():
                project = prem.get(key)
                role_list = project.get('roleList')
                for role_dict in role_list:
                    if tenderee is None and role_dict.get('role_name') == 'tenderee':
                        tenderee = role_dict.get('role_text')
                    if agency is None and role_dict.get('role_name') == 'agency':
                        agency = role_dict.get('role_text')
            product = item.get('product')
            begin_time = item.get('time_commencement')
            end_time = item.get('time_completion')
        return tenderee, agency, product, begin_time, end_time

    def predict(self, prem, list_articles, list_sentences, list_entitys, doctitle, code_name_dict, dochtmlcon):
        try:
            for list_article, list_sentence, list_entity in zip(list_articles, list_sentences, list_entitys):
                list_sentence.sort(key=lambda x: x.sentence_index)
                tenderee, agency, product, begin_time, end_time = self.get_col_from_prem(prem)
                content = list_article.content
                dochtmlcon = re.sub('[\r\n]', '', dochtmlcon)
                # 获取无附件的html
                soup = BeautifulSoup(dochtmlcon, 'lxml')
                attachment_div = soup.find('div', class_='richTextFetch')
                if attachment_div:
                    attachment_div.decompose()
                    content_no_att = soup.text
                else:
                    content_no_att = ''

                project_name = code_name_dict.get('name')
                project_code = code_name_dict.get('code')
                if project_code:
                    project_code = project_code[0]
                else:
                    project_code = None

                stage = extract_legal_stage(project_name+doctitle, self.stage_pattern, self.stage_priority_dict, product, tenderee=tenderee, agency=agency)
                industry1 = extract_industry(doctitle+content, self.industry_pattern)
                industry = extract_industry(doctitle+content_no_att, self.industry_pattern)
                # print('industry', industry, industry1)
                if not industry and industry1:
                    industry = industry1
                proportion1, proportion = extract_proportion(content)
                project_digest = extract_project_digest(content)
                project_address = extract_project_address(list_sentence, list_entity)
                location = get_bid_location(doctitle+"\t"+project_name)
                project_name_refind, show_name_refind = get_project_name_refind(project_name, doctitle, tenderee, agency)
                has_elevator = extract_has_elevator(content)
                project_property = extract_project_property(doctitle+"\t"+project_name, self.property_pattern, self.property_priority_dict)
                total_invest, construct_install_fee, engineer_cost = extract_several_money(list_sentence, dochtmlcon)
                max_floor = extract_max_floor(content, dochtmlcon)
                structure = extract_structure(content, dochtmlcon, self.structure_keyword_list)
                has_steel = extract_has_steel_structure(structure)
                wall_type, wall_type2 = extract_wall_type(doctitle+"\t"+project_name, content)

                if stage is not None:
                    has_stage = 1
                else:
                    has_stage = 0

                pb_json = {
                    'tenderee': tenderee,
                    'agency': agency,
                    'project_code': project_code,
                    'project_name': project_name,
                    'doctitle': doctitle,
                    'stage': stage,
                    'industry': industry,
                    'proportion': proportion,
                    'projectDigest': project_digest,
                    'projectAddress': project_address,
                    'location': location,
                    'project_name_refind': project_name_refind,
                    'has_elevator': has_elevator,
                    'project_property': project_property,
                    'total_invest': total_invest,
                    'construct_install_fee': construct_install_fee,
                    'engineer_cost': engineer_cost,
                    'max_floor': max_floor,
                    'structure': structure,
                    'has_steel': has_steel,
                    'wall_type': wall_type,
                    'wall_type2': wall_type2,
                    'begin_time': begin_time,
                    'end_time': end_time,
                    'has_stage': has_stage,
                }

                pb_json = {'pb': pb_json}
                return pb_json

        except:
            traceback.print_exc()
            return {'pb': 'error'}


def extract_legal_stage(content, _pattern, priority_dict, product='', tenderee='', agency=''):
    # 判断这几类直接返回
    if not content:
        return None
    if re.search("拍卖|转让|产权|出让|租赁|招租", content) is not None:
        return None
    # 去掉招标人代理人
    content = re.sub(str(tenderee), '', content)
    content = re.sub(str(agency), '', content)

    # 竣工阶段标志：消防、物业、安保、装修、通风系统、排烟、第三方检测
    # if re.search("消防|物业|安保|装修|通风系统|排烟|第三方检测", content) is not None:
    #     return '竣工阶段'

    # 替换混淆词
    _content = re.sub("设计院|设计总院|造价咨询有限公司", "", content)

    list_stage = []
    for stage_search in re.finditer(_pattern, _content):
        for k,v in stage_search.groupdict().items():
            if v is not None:
                list_stage.append([k, priority_dict.get(k)])
    if len(list_stage) > 0:
        list_stage.sort(key=lambda x: x[1])
        stage = list_stage[0][0]

        # 用product判断竣工阶段是否合法
        if product:
            if not re.search('施工|工程|建设', str(product)):
                stage = None
                for s in list_stage:
                    if s[0] != '竣工阶段':
                        stage = s[0]
                        break

        # 立项排除立项目
        if stage == '立项阶段':
            sub_content = re.sub('立项目', '', _content)
            for stage_search in re.finditer(_pattern, sub_content):
                for k,v in stage_search.groupdict().items():
                    if v is not None:
                        list_stage.append([k, priority_dict.get(k)])
            if len(list_stage)>0:
                list_stage.sort(key=lambda x: x[1])
                stage = list_stage[0][0]

        return stage
    return None


def get_project_name_refind(project_name, doctitle, tenderee='', agency= '', min_len=3):
    # 跳过部分
    re_str11 = '网上超市|服务市场采购|印刷服务|复印纸|车辆维修和保养|商品房预售|办公家具定点|直接订购|定点议价' \
               '|政府采购意向|信息技术服务定点议价|信息技术服务定点采购|法人章刻制中介机构|专用设备|办公设备采购' \
               '|线上摇号选取'
    re_str12 = '物业'
    re_str13 = '公共资源交易平台'
    re_str19 = '环境影响评价(文件|)(审批|审核|受理)|拟作出的建设'

    # 干扰部分
    re_str1 = "＜.*?＞［.*?］|｛.*?｝|〔.*?〕|《.*?》|【.*?】|\(.*?\)|\[.*?\]|（.*?）|\d{1,2}月\d{1,2}[日号]|\d{1,2}:\d{2}"

    re_str4 = '[,.:;，。：；\'\"“”‘’\-/<>#@!$%&*+=·￥|？?｜－＋＃＂﹝﹒！]'

    re_str5 = '[工程项目建设拟对年批第作出的个及在已]|标段|EPC|总承包|招标|文件|开标|记录|公示|验收|勘察|编制|公开选取|准予|论证|各单位|附件|建筑业' \
              '|责任|诚挚|拟作出审批意见|生产|涉及|消防|政府|投资|方案|技术支持|文件|研发|申请报告|出具|现代|产业|依法|报批|行政|审批|许可|开展' \
              '|活动|开放日|系列|某部|零星工程|某(地产|型号|单位)|权限内|办理了|外商|我院|召开|我市|启动|我单位|我(县|区|会)|成功|举办|举行' \
              '|被评为|征(求|询)|包括|不包括|层层分包|合同估算价|万元以内|组织|全(市|区|县)|承接|积极|针对|企业|小规模|安全|助推|装修|改造' \
              '|新建|居住|技术|建设|建筑|安装|园林|绿化|信息化|采购|商品房|预售|许可|房产|测量|报告|业务|零星|维修|水土保持|扩建|夜间|工地' \
              '|整治|高速公路|备案|加油站|设施|环境|保护|合同|履约|在线|询价|面积|受理|经济中心|服务|食品|加工|利用|公开|选取|动物|疫苗|框架' \
              '|协议|房屋|中国|不动产|实验室|限额|以下|单位|入围|审查|合格|意见|新能源|常规|许可|申请|加工|制品|建议书|可研|结算|审核|遴选' \
              ''

    re_str6 = '总承包|设计|环评|监理|施工|竣工|项目|工程|EPC|验收|勘察设计|全过程造价咨询|造价咨询|勘察|可行性研究报告|初步设计|社会稳定风险评估|测绘' \
              '|(地震安全性|环境影响|水土保持)评(价估)'

    re_str7 = '许可信息公开表|办理结果公示|审批信息公开表|验收公示表|信息披露|备案登记表|验收结果公示|' \
              '审批意见公开|受理公示|施工许可|情况说明|合同纠纷调解|施工许可双公示|施工许可证|政策'

    re_str8 = '[〔〕（）\[\]《》()【】｛｝{}［］＜＞]'

    re_str14 = '[〔（\[(【｛{［＜](采购结果|设计|环评|监理|施工|竣工|工程|EPC|验收|勘察设计|全过程造价咨询|造价咨询|勘察)[〕）\]》)】｝}］＞]'

    # 截取部分
    re_str2 = '机场快线|科技园|产业园|工业园|工程|项目|施工|竣工|总承包|改造|监理|可研|验收|勘察设计|全过程造价咨询|造价咨询|勘察|可行性研究报告|EPC|初步设计|社会稳定风险评估'
    re_str3 = '关于(公布核准的|一种用于|核发|作出|同意|开展|调整|请求|规范|要求|进一步|遴选|领取)|关于公[布开示]|关于[为对]|关于|公司|集团|局|委托'
    re_str9 = '改扩建|建设|迁改|土建|测绘|(地震安全性|环境影响|水土保持)评(价估)'

    # 混淆部分
    re_str10 = '局部'
    re_str17 = '(工程|信息|)有限|公司|集团|局|大学|院|学校|学院|中心'
    re_str18 = '(设计|造价|咨询|建设|项目|管理|工程)+有限|(信息|职业|技术|管理)+(大学|学校|学院|中心|院|所)'
    re_str26 = '服务类|(设计|造价|咨询|建设|项目|管理|工程)+(大学|学校|学院|中心|院|所|集团|局)'

    # 需判断删除部分
    re_str15 = '[ 、#※.?<|①=-_—]|=-|##|\*+|[\(（](001|[0-9]|[一二三四五六七八九十]+|)[\)）]|(0001|0000|001|002|01|02)+'
    re_str16 = '[0-9][.、]'

    # 删除特定表达
    re_str20 = '公共资源交易中心.*关于'
    re_str21 = '[\u4e00-\u9fff]{2,}市[\u4e00-\u9fff]{2,}区'
    re_str22 = '[\u4e00-\u9fff]{2,4}区[^至]'
    re_str23 = '.{1,2}招标公告|(PDF|pdf)(版|)'
    re_str25 = '(小区)$'
    re_str27 = '[\u4e00-\u9fff]{2,3}省|[\u4e00-\u9fff]{2,3}市'

    re_str_area = '华北|华南|华东|华中|西南|东北|西北'
    re_str_province = '北京|天津|河北|山西|内蒙古|广东|海南|广西|上海|江苏|浙江|安徽|福建|江西|山东|河南|湖北|湖南|重庆|四川|贵州|云南|西藏|黑龙江|辽宁|吉林|陕西|甘肃|青海|宁夏|新疆|台湾|香港|澳门'
    re_str_city = '东城|和平|石家庄|唐山|秦皇岛|邯郸|邢台|保定|张家口|承德|沧州|廊坊|衡水|太原|大同|阳泉|长治' \
                  '|晋城|朔州|晋中|运城|忻州|临汾|吕梁|呼和浩特|包头|乌海|赤峰|通辽|鄂尔多斯|呼伦贝尔|巴彦淖尔' \
                  '|乌兰察布|兴安盟|锡林郭勒盟|阿拉善盟|广州|韶关|深圳|珠海|汕头|佛山|江门|湛江|茂名|肇庆|惠州' \
                  '|梅州|汕尾|河源|阳江|清远|潮州|揭阳|云浮|海口|三亚|南宁|柳州|桂林|梧州|北海|防城港|钦州|贵港' \
                  '|玉林|百色|贺州|河池|来宾|崇左|黄浦|南京|无锡|徐州|常州|苏州|南通|连云港|淮安|盐城|扬州|镇江' \
                  '|泰州|宿迁|杭州|宁波|温州|嘉兴|湖州|绍兴|金华|衢州|舟山|台州|丽水|合肥|芜湖|蚌埠|淮南|马鞍山' \
                  '|淮北|铜陵|安庆|黄山|滁州|阜阳|宿州|六安|亳州|池州|宣城|福州|厦门|莆田|三明|泉州|漳州|南平' \
                  '|龙岩|宁德|南昌|景德镇|萍乡|九江|新余|鹰潭|赣州|吉安|宜春|抚州|上饶|济南|青岛|淄博|枣庄' \
                  '|东营|烟台|潍坊|济宁|泰安|威海|日照|临沂|德州|聊城|滨州|菏泽|郑州|开封|洛阳|平顶山|安阳|鹤壁' \
                  '|新乡|焦作|濮阳|许昌|漯河|三门峡|南阳|商丘|信阳|周口|驻马店|武汉|黄石|十堰|宜昌|襄阳|鄂州' \
                  '|荆门|孝感|荆州|黄冈|咸宁|随州|恩施土家族|长沙|株洲|湘潭|衡阳|邵阳|岳阳|常德|张家界|益阳' \
                  '|郴州|永州|怀化|娄底|湘西土家族|万州|成都|自贡|攀枝花|泸州|德阳|绵阳|广元|遂宁|内江|乐山' \
                  '|南充|眉山|宜宾|广安|达州|雅安|巴中|资阳|阿坝藏族羌族|甘孜藏族|凉山彝族|贵阳|六盘水|遵义' \
                  '|安顺|铜仁|黔西南布依族|毕节|黔东南苗族|黔南布依族|昆明|曲靖|玉溪|保山|昭通|丽江|普洱|临沧' \
                  '|楚雄彝族|红河哈尼族|文山|西双版纳傣族|大理白族|德宏傣族景颇族|怒江傈僳族|迪庆藏族|拉萨|昌都' \
                  '|山南|日喀则|那曲|阿里地区|林芝|哈尔滨|齐齐哈尔|鸡西|鹤岗|双鸭山|大庆|伊春|佳木斯|七台河' \
                  '|牡丹江|黑河|绥化|大兴安岭|沈阳|大连|鞍山|抚顺|本溪|丹东|锦州|营口|阜新|辽阳|盘锦|铁岭' \
                  '|朝阳|葫芦岛|长春|吉林|四平|辽源|通化|白山|松原|白城|延边朝鲜族|西安|铜川|宝鸡|咸阳|渭南' \
                  '|延安|汉中|榆林|安康|商洛|兰州|嘉峪关|金昌|白银|天水|武威|张掖|平凉|酒泉|庆阳|定西|陇南' \
                  '|临夏回族自治州|甘南藏族|西宁|海东|海北藏族|黄南藏族|海南藏族|果洛藏族|玉树藏族|海西蒙古族' \
                  '|银川|石嘴山|吴忠|固原|中卫|乌鲁木齐|克拉玛依|吐鲁番|哈密|昌吉|博尔塔拉蒙古|巴音郭楞蒙古' \
                  '|阿克苏|克孜勒苏柯尔克孜|喀什|和田地区|伊犁|伊犁哈萨克|塔城地区|阿勒泰|中山|东莞|天门|仙桃|潜江' \
                  '|石河子|五家渠|阿拉尔|图木舒克|三沙|儋州|涪陵|永川|西城|朝阳|丰台|石景山|海淀|门头沟' \
                  '|房山|通州|顺义|昌平|大兴|怀柔|平谷|密云|延庆|河东|河西|河北区|红桥|东丽|西青|津南|北辰' \
                  '|武清|宝坻|滨海|宁河|静海|蓟州|渝中|大渡口|江北|沙坪坝|九龙坡|南岸|北碚|綦江|大足|渝北' \
                  '|巴南|黔江|长寿|江津|合川|南川|璧山|铜梁|潼南|荣昌|开州|徐汇|长宁|静安|普陀|虹口|杨浦' \
                  '|闵行|宝山|嘉定|浦东新|金山|松江|青浦|奉贤|崇明|济源|神农架林区|五指山|文昌|琼海|万宁' \
                  '|东方|定安|屯昌|澄迈|临高|白沙黎族|昌江黎族|乐东黎族|陵水黎族|保亭黎族|琼中黎族|梁平' \
                  '|丰都|城口|垫江|忠县|云阳|奉节|巫山|巫溪|石柱|秀山|武隆|酉阳|彭水|南开|北屯|铁门关' \
                  '|双河|可克达拉|昆玉|胡杨河'

    re_str28 = '({})(地区)?|({})省?|({})[区市]?'.format(re_str_area, re_str_province, re_str_city)
    re_str29 = '(({})(地区)?({})省?)|(({})省?({})[区市]?)'.format(re_str_area, re_str_province, re_str_province, re_str_city)

    # 直接删除部分
    re_str24 = '(的|)(竞争性谈判|竞争性磋商|磋商|中标|单一来源|招标|更正)(采购|)(公告|)'

    add_col = project_name if project_name else '' + doctitle if doctitle else ''
    if re.search(re_str11, add_col) and not re.search(re_str12, add_col):
        return '', ''

    from_col_list = [project_name, doctitle]
    name_refind_flag_dict = {'True': [], 'False': []}
    for col in from_col_list:
        name_refind = ""
        match_flag = False
        if col is not None and len(col) > 0:
            name_refind = col

            # 部分跳过
            if re.search(re_str13, name_refind):
                continue

            # 替换特定表达
            match = re.search(re_str20, name_refind)
            if match:
                name_refind = name_refind[match.span()[1]:]

            # 去掉干扰
            name_refind = re.sub('年度', '年', name_refind)
            name_refind = re.sub(re_str4, '', name_refind)
            name_refind = re.sub(re_str14, '', name_refind)

            # print('name_refind', name_refind)

            # 连续截取工程前的，看哪一部分最适合当refind
            match = re.finditer(re_str2, name_refind)
            prob_name_list = []
            last_index = 0
            project_word_in_org = []
            for m in match:
                # 混淆词，设施工程中的施工
                if m.span()[0] > 0 and name_refind[m.span()[0]-1] in ['设']:
                    continue

                # 判断是不是公司名里的工程
                if re.search(re_str26, name_refind[m.span()[1]:]):
                    project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                    continue
                if re.search(re_str17, name_refind[m.span()[1]:m.span()[1]+3]):
                    project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                    continue
                if re.search(re_str18, name_refind[m.span()[1]:]):
                    project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                    continue

                match_flag = True
                prob_name_list.append(name_refind[last_index:m.span()[1]])
                last_index = m.span()[1]
            # print('match_flag', match_flag, name_refind)

            # 找不到则用第二套截取
            if not prob_name_list:
                match = re.finditer(re_str9, name_refind)
                last_index = 0
                for m in match:
                    # 混淆词，设施工程中的施工
                    if m.span()[0] > 0 and name_refind[m.span()[0]-1] in ['设']:
                        continue

                    # 判断是不是公司名里的工程
                    if re.search(re_str26, name_refind[m.span()[1]:]):
                        project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                        continue
                    if re.search(re_str17, name_refind[m.span()[1]:m.span()[1]+3]):
                        project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                        continue
                    if re.search(re_str18, name_refind[m.span()[1]:]):
                        project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
                        continue
                    match_flag = True
                    prob_name_list.append(name_refind[last_index:m.span()[1]])
                    last_index = m.span()[1]
            if not prob_name_list:
                prob_name_list = [name_refind]

            # print('prob_name_list', prob_name_list)
            # print('project_word_in_org', project_word_in_org)

            # 一开始不去掉括号里的内容，截取后再去掉
            for i, name in enumerate(prob_name_list):
                # 括号内容大于一半字数，则不去掉括号中的字
                match = re.search(re_str1, name)
                # print('name', name)
                # print('match', match)
                if match and len(match.group()) < len(name) / 2:
                    name = re.sub(re_str1, "", name)
                name = re.sub(re_str8, "", name)
                prob_name_list[i] = name

            # 判断refind是否合法
            # print('prob_name_list2', prob_name_list)
            name_refind = ''
            for name in prob_name_list:
                # 截取公司后的
                match = re.finditer(re_str3, name)
                prob_name_list2 = []
                for m in match:
                    # 排除混淆的情况
                    if m.group() in re_str10 and re.search(re_str10, name):
                        continue
                    prob_name_list2.append(name[m.span()[1]:])
                if prob_name_list2:
                    name = prob_name_list2[-1]

                # 剔除工程类判断词
                match1 = re.finditer(re_str6, name)
                for m1 in match1:
                    # 混淆词，设施工程中的施工
                    if m1.span()[0] > 0 and name[m1.span()[0]-1] in ['设']:
                        continue
                    s_index, e_index = m1.span()
                    word = name[s_index:e_index]
                    s_index = s_index - 1 if s_index > 0 else 0
                    e_index = e_index + 1 if e_index < len(name) else len(name)
                    word1 = name[s_index:e_index]
                    if word1 in project_word_in_org:
                        continue
                    name = re.sub(re.escape(word), '=' * len(word), name)
                name = re.sub('={2,}', "", name)

                # 剔除一些无关词占用长度
                if len(re.findall('[\u4e00-\u9fff]', name)) >= min_len \
                        and len(re.findall('[\u4e00-\u9fff]', re.sub(re_str5, '', name))) >= min_len:
                    name_refind = name
                    break

        if match_flag:
            name_refind_flag_dict['True'] += [name_refind]
        else:
            name_refind_flag_dict['False'] += [name_refind]

    # print('name_refind_flag_dict', name_refind_flag_dict)

    true_list = name_refind_flag_dict.get('True')
    false_list = name_refind_flag_dict.get('False')
    name_refind_candidate_list = []
    if true_list:
        true_list.sort(key=lambda x: len(x), reverse=True)
        name_refind = true_list[0]
        name_refind_candidate_list += true_list
    # else:
    #     name_refind = ''
    if false_list:
        false_list.sort(key=lambda x: len(x), reverse=True)
        name_refind_candidate_list += false_list

    # 对候选name_refind循环
    name_refind = ''
    show_name_refind = ''
    for name_refind in name_refind_candidate_list:
        # 直接判断删除数字
        match = re.match(re_str16, name_refind)
        if match and not re.match('[0-9]', name_refind[match.span()[1]:match.span()[1]+1]):
            name_refind = name_refind[match.span()[1]:]

        # 删除开头奇怪数字
        match = re.match(re_str15, name_refind)
        if match and not re.match('[a-zA-Z地块号]', name_refind[match.span()[1]:match.span()[1]+1]):
            name_refind = name_refind[match.span()[1]:]

        # 删除期数
        name_refind = re.sub('[1-9一二三四五六七八九十]期', '', name_refind)

        # 跳过'xx省xx市'
        if re.search(re_str21, name_refind):
            sub_word = re.sub(re_str21, '', name_refind)
            sub_word = re.sub(re_str2 + '|' + re_str9, '', sub_word)
            if len(sub_word) <= 1:
                name_refind = ''
                continue
        match27 = re.search(re_str27, name_refind)
        if match27 and len(match27.group()) == len(name_refind):
            name_refind = ''
            continue
        match28 = re.search(re_str28, name_refind)
        if match28 and len(match28.group()) == len(name_refind):
            name_refind = ''
            continue
        match29 = re.search(re_str29, name_refind)
        if match29 and len(match29.group()) == len(name_refind):
            name_refind = ''
            continue

        # 删除类似'招标公告'表达
        match2 = re.match(re_str23, name_refind)
        if match2:
            name_refind = name_refind[match2.span()[1]:]
        name_refind = re.sub(re_str24, '', name_refind)

        # 跳过文件审批
        if re.search(re_str19, name_refind):
            name_refind = ''
            continue

        # 跳过网上超市
        if re.search(re_str11, name_refind):
            name_refind = ''
            continue

        show_name_refind = copy.deepcopy(name_refind)

        # 删除区
        match2 = re.match(re_str22, name_refind)
        if match2:
            name_refind = name_refind[match2.span()[1]-1:]

        # 删除'小区表达'
        if len(name_refind) >= min_len + 2:
            name_refind = re.sub(re_str25, '', name_refind)

        # 判断name_refind是否是从公司中来的，过滤
        if tenderee in [None, 'None', '-', '']:
            tenderee = ''
        if agency in [None, 'None', '-', '']:
            agency = ''
        try:
            if len(name_refind) >= 4 and (re.search(re.escape(name_refind[-4:]), tenderee) or re.search(re.escape(name_refind[-4:]), agency)):
                name_refind = ''
                show_name_refind = ''
        except:
            pass

        # 判断长度
        if len(name_refind) < min_len:
            name_refind = ''
            show_name_refind = ''
            continue

        break

    return name_refind, show_name_refind


def extract_industry(content, _pattern):
    list_stage = []
    stage_dict = {}
    for stage_search in re.finditer(_pattern, content):
        for k,v in stage_search.groupdict().items():
            if v is not None:
                list_stage.append(k)
                if k in stage_dict.keys():
                    stage_dict[k] += 1
                else:
                    stage_dict[k] = 1
    if len(list_stage)>0:
        stage_cnt_list = [[x, stage_dict.get(x)] for x in stage_dict.keys()]
        stage_cnt_list.sort(key=lambda x: x[1], reverse=True)
        # print('extract_industry ' + str(stage_cnt_list))
        return stage_cnt_list[0][0]
        # return list_stage[0]
    return None


def extract_project_code_name(list_entity):
    project_code = None
    project_name = None
    for p_entity in list_entity:
        if p_entity.entity_type == "name":
            project_name = p_entity.entity_text
        elif p_entity.entity_type == "code":
            project_code = p_entity.entity_text
        if project_name and project_code:
            break
    return project_code, project_name


def extract_tenderee(list_entity):
    tenderee = None
    for p_entity in list_entity:
        if str(p_entity.label) == "0":
            tenderee = p_entity.entity_text
            break
    return tenderee


def extract_project_digest(content):
    _pattern = "(?P<projectDigest>(项目|工程|标的|需求|建设|招标|采购|内容)(概况|规模|简介|信息|范围|内容|说明|摘要).{10,300})"
    _pattern_search = re.search(_pattern,content)
    _projectDigest = ""
    _find = ""
    if _pattern_search is not None:
        _find = _pattern_search.groupdict().get("projectDigest","")
    if len(_find)>0:
        _projectDigest = "。".join(_find.split("。")[0:3])

    # 截掉中标信息
    if len(_projectDigest) >= 10:
        _projectDigest = cut_win_bid_part(_projectDigest)
        if len(_projectDigest) < 10:
            _projectDigest = ""
    return _projectDigest


def extract_project_address(list_sentence, list_entity):
    reg1 = "(项目|建设|工程)所在区域(位于|为|)[：:]?"
    reg2 = "(项目|建设|工程)(地址|地点|)(位于|起于)[：:]?"
    reg3 = "(项目|建设|工程)(地址|地点)[：:]?(位于|起于)"
    reg4 = "(项目|建设|工程)(地址|地点)[为：:]+"


    address_list = []
    candidate_list = []
    for sentence in list_sentence:
        for reg in [reg1, reg2, reg3, reg4]:
            content = sentence.sentence_text
            match = re.finditer(reg, content)

            for m in match:
                end_index = m.span()[1]
                for p_entity in list_entity:
                    if p_entity.sentence_index != sentence.sentence_index:
                        continue
                    if p_entity.entity_type != "location" and p_entity.entity_type != "org":
                        continue

                    text = p_entity.entity_text
                    if text == content[end_index:end_index+len(text)] or text in content[end_index:end_index+len(text)+10]:
                        address_list.append(text)
                    else:
                        candidate_list.append(content[max(0, end_index-10):end_index] + '@@@' + content[end_index:end_index+20] + '@@@' + text)

            if address_list:
                break

    if not address_list:
        # for can in candidate_list:
        #     logging.info('candidate ' + can)
        return None
    else:
        address_list.sort(key=lambda x: len(x), reverse=True)
        # for address in address_list:
        #     logging.info('address ' + address)
        address = address_list[0]

        return address_list[0]


def extract_begin_end_time(list_sentence, list_entity):
    _begin_time = None
    _end_time = None
    for p_entity in list_entity:
        if p_entity.entity_type == "time":
            for _sentence in list_sentence:
                if _sentence.sentence_index == p_entity.sentence_index:
                    _span = spanWindow(tokens=_sentence.tokens,begin_index=p_entity.begin_index,end_index=p_entity.end_index,size=20,center_include=True,word_flag=True,text=p_entity.entity_text)
                    if re.search("开工(时间|日期)",_span[0]) is not None:
                        _time_temp = timeFormat(p_entity.entity_text)
                        if len(_time_temp)>0:
                            _begin_time = _time_temp
                    if re.search("(竣工|完工)(时间|日期)",_span[0]) is not None:
                        _time_temp = timeFormat(p_entity.entity_text)
                        if len(_time_temp)>0:
                            _end_time = _time_temp

    return _begin_time,_end_time


def get_bid_location(content):
    """
    获取标段工程地点
    """
    import BiddingKG.dl.interface.Preprocessing as Preprocessing
    content = re.sub("[,，.。、\-\(\)（）— #+~“”'﹙{}－]", "", content)

    # 预处理
    content_html = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"><html><body><div>' + content + "</div></body></html>"
    list_articles, list_sentences, list_entitys, list_outlines, _cost_time \
        = Preprocessing.get_preprocessed([[0, content_html, "", "", content, "", ""]],
                                         useselffool=True)

    # 获取location
    location = None
    for entitys in list_entitys:
        for entity in entitys:
            if entity.entity_type == "location":
                if re.search("[街路村县区溪湖河城厂]|[小中大]学|学[校院]", entity.entity_text):
                    location = entity.entity_text
                    break

    # location还未空，寻找类似 '薛家湾（张家圪旦）至柳青'的表达
    if not location:
        match = re.search('.{2,4}至.{2,4}', content)
        if match:
            location = match.group()
            # 判断location不在一些特定实体里
            find_flag = False
            for entitys in list_entitys:
                for entity in entitys:
                    if entity.entity_type in ["tenderee", 'agency', 'win_tenderer', 'second_tenderer', 'third_tenderer', 'company', 'org']:
                        if location in entity.entity_text:
                            find_flag = True
                            break
                if find_flag:
                    break
            if find_flag:
                location = None
    return location


def extract_proportion(content, has_preffix=True):
    if not content:
        return "", ""
    # log("content")
    # log(content)
    suffix = "[大概约为是:：【\[\s]*[\d,]+(\.\d+)?[十百千万亿]*([\]】平方kK千万公㎡mM米里顷亩]+2?))"
    reg_dict = {
        0: "(?P<proportion>(总((建筑|建设)面积|长|长度))" + suffix,
        1: "(?P<proportion>((建筑|建设)面积|全长)" + suffix,
        2: "(?P<proportion>((建筑|建设|区域)?面积|全长|项目规模)" + suffix
    }

    if not has_preffix:
        reg_dict[3] = "(?P<proportion>" + suffix

    _proportion = ""
    for i in range(len(list(reg_dict.keys()))):
        if _proportion:
            break
        _pattern = reg_dict.get(i)
        # logging.info('content ' + str(content))
        match = re.search(_pattern, str(content))
        if match:
            _proportion = match.groupdict().get("proportion","")

    if not _proportion:
        return "", ""

    # 统一格式
    multiple_cnt = 1
    digit = ""

    # 确定具体数字
    match = re.search('(?P<d1>[\d,]+)(?P<d2>(\.\d+)?)', _proportion)
    if match:
        # logging.info(str(_proportion) + '  ' + str(match.group()))
        d1 = match.group('d1')
        d2 = match.group('d2')
        try:
            d1 = int(re.sub(',', '', d1))
        except:
            return "", ""
        if d2:
            d2 = Decimal(d2[1:]) / Decimal(str(int(10 ** len(d2[1:]))))
            # print('d1, d2', d1, d2)
            d1 += d2
        digit = d1
    # print('digit', digit)

    # 确定中文倍数
    _proportion2 = re.sub(re.escape(match.group()), '', _proportion)
    match = re.search('[十百千万亿]+', _proportion2)
    _dict = {'十': 10, '百': 100, '千': 1000, '万': 10000, '亿': 100000000}
    if match:
        for c in match.group():
            multiple_cnt *= _dict.get(c)
        _proportion3 = re.sub(re.escape(match.group()), '', _proportion2)
    else:
        _proportion3 = _proportion2
    # print('multiple_cnt2', multiple_cnt)

    # 确定面积/长度
    match = re.search('[平方㎡顷亩]+|[mM]2', _proportion3)
    if match:
        unit = '㎡'
    else:
        unit = 'm'

    # 确定单位倍数
    match = re.search('[平方kK千万公㎡mM米里顷亩]+2?', _proportion3)
    if match:
        if unit == 'm':
            if re.search('[kK千公]', match.group()):
                multiple_cnt *= 1000
            elif re.search('[里]', match.group()):
                multiple_cnt *= Decimal(str(500))
        else:
            if '亩' in match.group():
                multiple_cnt *= Decimal(str(666.67))
            elif '顷' in match.group():
                multiple_cnt *= 10000
            elif re.search('千米|公里|k[mM㎡]', match.group()):
                multiple_cnt *= 1000000
    # print('multiple_cnt1', multiple_cnt)

    # 拼接
    digit = str(digit * multiple_cnt) + unit

    return _proportion, digit


def extract_has_elevator(content):
    reg = '电梯'
    match = re.search(reg, content)
    has_flag = 0
    if match:
        start_index, end_index = match.span()
        has_flag = 1
        if judge_yeji(match.span()[0], content):
            has_flag = 0
        elif re.search('公司', content[end_index:end_index+8]):
            has_flag = 0
    return has_flag


def extract_project_property(content, property_pattern, property_priority_dict):
    property_list = []
    for m in re.finditer(property_pattern, content):
        for k,v in m.groupdict().items():
            if v is not None:
                property_list.append([k, property_priority_dict.get(k)])

    _property = '新建'
    if len(property_list)>0:
        property_list.sort(key=lambda x: x[1])
        _property = property_list[0][0]
    return _property


def extract_several_money(list_sentence, html='', is_obj=True):
    money_type_list = ['总投资', '建安费', '工程造价']

    money_list = []
    all_before_sentence = ''
    for i, sentence in enumerate(list_sentence):
        last_text = ''
        next_text = ''
        if is_obj:
            text = sentence.sentence_text
            all_before_sentence += text
            if i > 0:
                last_text = list_sentence[i-1].sentence_text[-30:]
            if i < len(list_sentence) - 1:
                next_text = list_sentence[i+1].sentence_text[:30]
        else:
            text = sentence
            all_before_sentence += text
            if i > 0:
                last_text = list_sentence[i-1][-30:]
            if i < len(list_sentence) - 1:
                next_text = list_sentence[i+1][:30]

        if judge_yeji(len(all_before_sentence), all_before_sentence, 300+len(text)):
            # print('sentence yeji before ' + text)
            continue
        # if '项目概算总投资为' in text:
        _list, _ = get_several_money(text, 0, False, html=html)
        # logging.info('get_several_money _list ' + str(_list))

        temp_list = []
        for l in _list:
            if l[-1] == '总投资':
                if re.search('业绩', last_text+text+next_text):
                    continue
            temp_list.append(l)
        _list = temp_list

        money_list += _list
        # if money_list:
        #     break

    money_type_dict = {}
    for money, _, _, _, money_type in money_list:
        for _type in money_type_list:
            if _type != money_type:
                continue

                # 科学计数法
            try:
                if 'E+' in money:
                    times = int(money.split('E+')[-1])
                    _money = float(float(money.split('E+')[0]) * (10 ** times))
                else:
                    _money = float(money)
            except:
                continue

            if _type in money_type_dict.keys():
                money_type_dict[_type] += [_money]
            else:
                money_type_dict[_type] = [_money]

    # logging.info('money_type_dict ' + str(money_type_dict))

    result_list = []
    for _type in money_type_list:
        if money_type_dict.get(_type):
            if _type == '建安费':
                temp_list = money_type_dict.get(_type)
                temp_list = list(set(temp_list))
                money = 0
                for m in temp_list:
                    money += m
                result_list.append(money)
            else:
                result_list.append(money_type_dict.get(_type)[0])
        else:
            result_list.append(None)

    for i in range(len(result_list)):
        if result_list[i] is None:
            result_list[i] = 0
        result_list[i] = float(result_list[i])

    return result_list


def extract_max_floor(content, html=None):
    def match_floor(_reg, _content, _reg2=None, _tables_and_divs=None):
        _match = re.finditer(_reg, _content)
        _floor_list = []
        for m in _match:
            if 'reg6' in _reg:
                _floor1 = content[max(0, m.span('reg6')[0]-1):m.span('reg6')[1]+1]
            elif 'reg4' in _reg:
                _floor1 = content[max(0, m.span('reg4')[0]-1):m.span('reg4')[1]+1]
            else:
                _floor1 = content[m.span()[0]:m.span()[1]]
            if judge_yeji(m.span()[0], _content, 300, _tables_and_divs, _floor1):
                continue

            if 'reg6' in _reg:
                _floor = m.group('reg6')
                _floor = re.split('[-~～]', _floor)
                _floor = max(int(_floor[0]), int(_floor[1]))
            elif 'reg4' in _reg:
                _floors = re.findall('\d+', m.group())
                _floors = [int(x) for x in _floors]
                _floors1 = re.findall('[一二两三四五六七八九十]+', m.group())
                _floors1 = [chinese_to_arabic(x) for x in _floors1]
                _floor = max(_floors + _floors1)
            elif '-' in m.group():
                _floor = ''.join(re.findall('\d+-\d+', m.group()))
                if len(_floor) < 1:
                    continue
                _floor = _floor.split('-')
                _floor = max(int(_floor[0]), int(_floor[1]))
            elif '/' in m.group():
                _floor = m.group()
                _floor = re.sub('层', '', _floor)
                _floor = ''.join(re.findall('\d+/\d+', _floor))
                # print('@1', _floor)
                if len(_floor) < 1:
                    continue
                _floor = _floor.split('/')
                _floor = max(int(_floor[0]), int(_floor[1]))
            else:
                _floor = ''.join(re.findall('\d+', m.group()))
                if len(_floor) < 1:
                    _floor = ''.join(re.findall('[一二两三四五六七八九十]+', m.group()))
                    if len(_floor) < 1:
                        continue
                    _floor = chinese_to_arabic(_floor)
                _floor = int(_floor)
            if _reg2:
                _floor_list2 = match_floor(_reg2, _content[m.span()[1]:m.span()[1]+35])
                # print('@2', _floor_list2)
                if _floor_list2:
                    _floor2 = int(_floor_list2[0])
                    _floor = _floor + _floor2
            _floor_list.append(_floor)
        return _floor_list

    reg = '(建筑|)(物|)(层数最大|最大层数|最高层|总层数|最大层|层数)[共为:：]?(\d{1,3}|\d{1,3}层?/\d{1,3}|[一两二三四五六七八九十]{1,3})[层Ff]'
    reg0 = '局部(建筑|)(层数|)[共为]?(\d{1,3}|[一二两三四五六七八九十]{1,3})层'
    reg1 = '地上(建筑|)(层数|)[共为]?(\d{1,3}层?/\d{1,3}|\d{1,3}-\d{1,3}|\d{1,3}|[一二两三四五六七八九十]{1,3})层'
    reg2 = '地下(建筑|)(层数|)[共为]?(\d{1,3}层?/\d{1,3}|\d{1,3}-\d{1,3}|\d{1,3}|[一二两三四五六七八九十]{1,3})层'
    reg3 = '[到至]\d{1,3}层'
    reg4 = '层数分别.{1,20}(?P<reg4>(\d{1,3}|[一二两三四五六七八九十]{1,3}))层'
    reg5 = '共(\d{1,3}|[一二两三四五六七八九十]{1,3})层'
    reg6 = '地上.{1,10}(?P<reg6>\d{1,3}[-~～]\d{1,3})层'

    if html:
        soup = BeautifulSoup(html, 'lxml')
        tables_and_divs = soup.find_all(['table', 'div', 'p'])
    else:
        tables_and_divs = []

    floor_list = []
    # 常规
    floor_list += match_floor(reg, content, _tables_and_divs=tables_and_divs)
    # 局部
    floor_list += match_floor(reg0, content, _tables_and_divs=tables_and_divs)
    # x层到x层
    floor_list += match_floor(reg3, content, _tables_and_divs=tables_and_divs)
    # 地上地下
    floor_list += match_floor(reg1, content, _reg2=reg2, _tables_and_divs=tables_and_divs)
    floor_list += match_floor(reg2, content, _reg2=reg1, _tables_and_divs=tables_and_divs)
    # 层数分别为... xx层
    floor_list += match_floor(reg4, content, _tables_and_divs=tables_and_divs)
    # 共xx层
    floor_list += match_floor(reg5, content, _tables_and_divs=tables_and_divs)
    # 地上... xx~xx层
    floor_list += match_floor(reg6, content, _reg2=reg2, _tables_and_divs=tables_and_divs)

    if floor_list:
        floor_list.sort(key=lambda x: x)
        floor = floor_list[-1]
        if floor <= 0:
            return None
        else:
            return floor_list[-1]
    else:
        return None


def extract_structure(content, html=None, structure_keyword_list=None):
    # reg = '框架结构|钢框架结构|混凝土框架结构|剪力墙结构|框架-剪力墙结构|框架+剪力墙结构|框架和剪力墙结构|框架及剪力墙结构|混凝土剪力墙结构|筒体结构|桅式结构|墙板结构|膜结构|悬索结构|板柱结构|充气结构|网架结构|壳体结构|拱形结构|穹顶结构|混凝土结构|钢筋混凝土框架结构|钢筋混凝土筒仓结构|钢结构|砌体结构|木结构|砖混结构|排架结构|束筒结构|薄壳结构|钢混结构|砖木结构|砌体结构|钢砼结构|框剪结构|钢筋混凝土框架结构|筒中筒结构|框筒结构|桁架结构|拱券结构|钢筋混凝土结构|框架核心筒结构|门式钢架结构|门钢结构|轻钢结构|钢-混凝土框架结构|木框架结构|空间网格结构|框架筒体结构|砖拱结构|钢筋砼结构|核心筒结构|框架-核心筒结构'
    reg1 = '(结构(楼层|)(形式|类型|类别|体系|结构)[为是：:])([^结]{2,8}结构)'

    reg = '|'.join([x[:-1] for x in structure_keyword_list])
    reg = reg_word_sort(reg)
    # logging.info(reg)

    if html:
        soup = BeautifulSoup(html, 'lxml')
        tables_and_divs = soup.find_all(['table', 'div', 'p'])
    else:
        tables_and_divs = []

    # match = re.finditer(reg1, content)
    # structure_list = []
    # for m in match:
    #     structure = m.group(4)
    #     structure1 = content[max(0, m.span(4)[0]-1):m.span(4)[1]+1]
    #     if judge_yeji(m.span()[0], content, 300, tables_and_divs, structure1):
    #         continue
    #     structure_list.append(structure)

    # if structure_list:
    #     structure_list = list(set(structure_list))
    #     structure_list.sort(key=lambda x: x)
    #     return ','.join(structure_list)
    # else:

    structure_list = []
    match = re.finditer(reg, content)
    for m in match:
        structure = m.group()
        structure1 = content[max(0, m.span()[0]-1):m.span()[1]+1]
        if judge_yeji(m.span()[0], content, 300, tables_and_divs, structure1):
            continue
        if structure in ['钢结构']:
            if re.search('公司', content[m.span()[1]:m.span()[1]+8]):
                continue
        structure_list.append(structure)

    if structure_list:
        structure_list = list(set(structure_list))
        structure_list.sort(key=lambda x: x)
        return ','.join(structure_list)
    else:
        return None


def extract_has_steel_structure(content):
    """

    :param content: 传入已提取完的structure
    :return:
    """
    if not content:
        return 0

    reg = '钢结构|门式钢架结构|钢框架结构|钢桁架结构|钢网架结构|钢框结构钢架结构|钢骨架结构|钢骨架式结构|钢管桁架结构|轻钢骨架结构|钢桁架式结构|轻钢门架结构|轻钢门式结构|门式钢屋架结构|钢桁架框架结构|钢框架—支撑结构|钢桁梁结构|网架结构|网壳结构|索膜结构|塔桅结构'
    match = re.search(reg, content)
    has_flag = 0
    if match:
        has_flag = 1
    return has_flag


def extract_wall_type(doctitle, content):
    reg1 = '(幕墙|外墙)'

    wall_class_dict = {
        '玻璃幕墙': '玻璃|玻璃砖',
        '金属幕墙': '铝合金|铝单板|仿木纹铝单板|夹芯保温铝板|铝复合板|蜂窝铝板|仿石材铝板|铝板|彩钢板|不锈钢板|彩涂钢板|珐琅钢板|钛合金板|铝合金装饰网格|镀锌钢网|铜合金|镀层钢板|锌板|钛板',
        '陶土板幕墙': '陶土板',
        '涂料': '涂料|乳胶漆|弹性涂料|质感涂料|真石漆|水包水|水包砂|岩片漆|金属漆|氟碳漆|仿石涂料|封闭底漆|腻子|罩光清漆|柔性耐水腻子|罩面漆|仿瓷涂料|有机硅丙烯酸涂料|氟碳树脂涂料|硅溶胶外墙涂料|无机纤维喷涂',
        '石材': '石材|花岗岩|大理石|砂岩|板岩|岩棉板|岩板',
        '瓷砖': '陶瓷砖|劈开砖|釉面砖|通体砖|抛光砖|马赛克砖|陶砖|玻璃砖',
        '混凝土': '钢筋混凝土|现浇混凝土|预制混凝土|混凝土砌块|蒸压多孔混凝土|纤维混凝土|UHPC板|UHPC|UHPC装饰板|超高性能混凝土|清水混凝土|混凝土|加气混凝土砌块',
        '复合墙板': '复合外墙板|复合墙板|GRC|金属面岩棉板|复合保温幕墙板|金属面复合保温幕墙|金属面岩棉夹心外墙板|夹心保温外墙|金属面岩棉装饰节能一体板|PU板幕墙|保温装饰一体板|保温一体板|岩棉保温装饰一体板|EPS保温板|岩棉保温板|STP保温板|酚醛保温板|聚氨酯保温板|钛锌塑铝复合板|自保温板外墙板|岩棉夹心板外墙板|玻璃棉|EPS板|XPS板|聚氨酯外墙保温板|外墙保温板|保温岩棉复合板|水泥发泡保温板|发泡陶瓷保温板|石墨改性水泥基保温板|挤塑聚苯板|挤塑板|聚苯乙烯板|玻化微珠岩棉板|聚苯板|聚氨酯板|热固复合聚苯乙烯泡沫保温板',
        '水泥': '白水泥|白色硅酸盐水泥',
        '文化砖': '文化砖',
        '木材': 'LVL|单板层积材|高压热固化木纤维板',
        '陶棍幕墙': '陶棍幕墙',
        '风动幕墙': '风铃幕墙|风动幕墙',
        '光电幕墙': '光伏幕墙|光电幕墙',
        '织物幕墙': '织物幕墙',
        '生态幕墙': '生态幕墙|气循环幕墙|呼吸幕墙|绿色幕墙|健康幕墙',

    }

    for key in wall_class_dict.keys():
        reg = wall_class_dict[key]
        reg = reg_word_sort(reg)
        wall_class_dict[key] = '(' + reg + ')'

    wall_list = []
    wall_list2 = []
    match = re.search(reg1, doctitle)
    if match:
        for first_class in wall_class_dict.keys():
            reg2 = wall_class_dict.get(first_class)
            if first_class in ['陶棍幕墙', '风动幕墙', '光电幕墙', '织物幕墙', '生态幕墙']:
                continue

            reg2 = reg2 + '(幕墙|外墙|)'
            match1 = re.search(reg2, content)
            if match1:
                wall_list.append(first_class)
                wall_list2.append(match1.group())

    for first_class in wall_class_dict.keys():
        reg2 = wall_class_dict.get(first_class)

        # 这5类特殊，只要存在就直接提取
        if first_class in ['陶棍幕墙', '风动幕墙', '光电幕墙', '织物幕墙', '生态幕墙']:
            match = re.search(reg2, content)
            if match:
                wall_list.append(first_class)
                wall_list2.append(match.group())
        # 其他类使用两种规则
        else:
            reg3 = reg1 + '.{0,10}' + reg2
            reg4 = reg2 + reg1
            match = re.search(reg3, content)
            if match:
                wall_list.append(first_class)
                match1 = re.search(reg2, match.group())
                if match1:
                    wall_list2.append(match1.group())
            match = re.search(reg4, content)
            if match:
                wall_list.append(first_class)
                wall_list2.append(match.group())

    if wall_list:
        wall_list = list(set(wall_list))
        wall_list.sort(key=lambda x: x)
        wall_list = ','.join(wall_list)
    else:
        wall_list = None

    if wall_list2:
        wall_list2 = list(set(wall_list2))
        wall_list2.sort(key=lambda x: x)
        wall_list2 = ','.join(wall_list2)
    else:
        wall_list2 = None

    return wall_list, wall_list2


def cut_win_bid_part(_str):
    """
    截掉项目概述里的中标相关信息
    """
    origin_str = _str
    reg_list = [
        "(评标|中标|中选)(结果|报告)(公示|)[:：，,]",
        "第[一二三四1-9]名",
        "中标候选人(基本情况|附件)",
        "(排序|第[一二三1-3]|推荐)中标候选人|中标(候选人|结果)(排序|公示|信息)"
    ]
    for reg in reg_list:
        match = re.finditer(reg, _str)
        _index = len(origin_str)
        for m in match:
            _index = m.span()[0]
            break
        if _index == len(origin_str):
            continue
        match = re.finditer("[,，。；;]", _str[:_index])
        _index = 0
        for m in match:
            _index = m.span()[0]
        _str = _str[:_index]

    match = re.finditer("中[标选]|投标(人|企业|单位)|乙方|供应商|成交", _str)
    for m in match:
        index_start = m.span()[0]
        cut_str = re.split("[,，。；;]", _str[index_start:])[0]
        if len(cut_str) < 25:
            cut_str = _str[index_start:index_start+25]
        # cut_str = _str[index_start:index_start+15]
        # print("cut_str", cut_str)

        match2 = re.search("(￥|)[\d,.]+[万元亿]|公司|业绩|最终得分", cut_str)
        # print("match2", match2)
        if match2:
            # 排除混淆
            match3 = re.search("中标法|获取招标文件|交接|支付|账户|结算|限价|承担|服务费|\*|×", cut_str)
            # print("match3", match3)
            if not match3:
                match4 = re.finditer("[,，。；;]", _str[:index_start])
                index_stop = 0
                for m4 in match4:
                    index_stop = m4.span()[0]
                _str = _str[:index_stop]
            break
    # print(_str)
    return _str


def judge_yeji(end_index, content, judge_len=300, tables_and_divs=None, entity=None):
    def is_yeji_table(_tables_and_divs, _entity_text):
        if not _tables_and_divs:
            return 0

        is_yeji = 0
        reg_yeji = '业绩|选取原因|奖项|获奖|供应商信息|近年完成|中标单位信息|评标情况|类似项目|资质|候选人情况'
        for index3, obj in enumerate(_tables_and_divs):
            # if '<tr' not in str(obj):
            if obj.find('tr'):
                continue

            _table = obj
            _table_text = str(_table.get_text())
            # _m = re.search(re.escape(_entity_text), str(_table))
            _m = re.search(re.escape(_entity_text), _table_text)
            if not _m:
                _entity_text = _entity_text[1:]
                _m = re.search(re.escape(_entity_text[1:]), _table_text)
            if not _m:
                _entity_text = _entity_text[:-1]
                _m = re.search(re.escape(_entity_text[:-1]), _table_text)
            # print('_entity_text', _entity_text)
            if _m:
                rows = _table.find_all('tr')

                max_col_span = 0
                for row in rows:
                    if len(row.find_all('td')) > max_col_span:
                        max_col_span = len(row.find_all('td'))
                # print('max_col_span', max_col_span)

                for index, row in enumerate(rows):
                    if re.search(re.escape(_entity_text), str(row.get_text())):
                        cells = row.find_all('td')
                        if len(cells) == 0:
                            continue

                        # print('cells', cells[0])
                        # 第一列是否含业绩
                        # if re.search(reg_yeji, str(cells[0])):
                        if re.search(reg_yeji, str(cells[0].get_text())):
                            # logging.info('is_yeji_table 1')
                            is_yeji = 1
                        # 前面几行是否有占多列的有业绩
                        else:
                            if index > 0:
                                for row2 in rows[:index][::-1]:
                                    # print('len(rows[index2])', len(row2.find_all('td')))
                                    # if len(row2.find_all('td')) <= max_col_span / 2:
                                    #     print(re.search('业绩', str(row2)), str(row2))
                                    if len(row2.find_all('td')) <= max_col_span / 2 and re.search(reg_yeji, str(row2.get_text())):
                                        # logging.info('is_yeji_table 2')
                                        is_yeji = 1

                        break

                # 前面都找不到，那么找表格上方的两行
                div_list = [str(x.get_text()) for x in _tables_and_divs[max(0, index3-2):index3]]
                if not is_yeji and re.search(reg_yeji, ' '.join(div_list)):
                    # logging.info('is_yeji_table 3')
                    is_yeji = 1
                break
        return is_yeji

    # 先判断表格业绩
    if tables_and_divs:
        yeji_table_flag = is_yeji_table(tables_and_divs, entity)
        if yeji_table_flag:
            # logging.info('yeji_table_flag 1')
            return yeji_table_flag

    if len(content) == 0:
        return 0
    if end_index == 0:
        return 0

    reg_yeji = '业绩|选取原因|奖项|获奖|供应商信息|近年完成'
    if re.search(reg_yeji, content[:end_index][-judge_len:]):
        # if '业绩' in content[:end_index][-judge_len:]:
        return 1
    else:
        return 0


def get_several_money(sentence_text, found_yeji, in_attachment=False, html=''):
    def getDigitsDic(_unit):
        '''
        @summary:拿到中文对应的数字
        '''
        DigitsDic = {"零":0, "壹":1, "贰":2, "叁":3, "肆":4, "伍":5, "陆":6, "柒":7, "捌":8, "玖":9,
                     "〇":0, "一":1, "二":2, "三":3, "四":4, "五":5, "六":6, "七":7, "八":8, "九":9}
        return DigitsDic.get(_unit)

    def getMultipleFactor(_unit):
        '''
        @summary:拿到单位对应的值
        '''
        MultipleFactor = {"兆":Decimal(1000000000000),"亿":Decimal(100000000),"万":Decimal(10000),"仟":Decimal(1000),"千":Decimal(1000),"佰":Decimal(100),"百":Decimal(100),"拾":Decimal(10),"十":Decimal(10),"元":Decimal(1),"圆":Decimal(1),"角":round(Decimal(0.1),1),"分":round(Decimal(0.01),2)}
        return MultipleFactor.get(_unit)

    def getUnifyMoney(money):
        '''
        @summary:将中文金额字符串转换为数字金额
        @param:
            money:中文金额字符串
        @return: decimal,数据金额
        '''

        MAX_MONEY = 1000000000000
        MAX_NUM = 12
        #去掉逗号
        money = re.sub("[，,]","",money)
        money = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",money)
        result = Decimal(0)
        chnDigits = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"]
        # chnFactorUnits = ["兆", "亿", "万", "仟", "佰", "拾","圆","元","角","分"]
        chnFactorUnits = ["圆", "元","兆", "亿", "万", "仟", "佰", "拾", "角", "分", '十', '百', '千']

        LowMoneypattern = re.compile("^[\d,]+(\.\d+)?$")
        BigMoneypattern = re.compile("^零?(?P<BigMoney>[%s])$"%("".join(chnDigits)))
        try:
            if re.search(LowMoneypattern,money) is not None:
                return Decimal(money)
            elif re.search(BigMoneypattern,money) is not None:
                return getDigitsDic(re.search(BigMoneypattern,money).group("BigMoney"))
            for factorUnit in chnFactorUnits:
                if re.search(re.compile(".*%s.*"%(factorUnit)),money) is not None:
                    subMoneys = re.split(re.compile("%s(?!.*%s.*)"%(factorUnit,factorUnit)),money)
                    if re.search(re.compile("^(\d+)(\.\d+)?$"),subMoneys[0]) is not None:
                        if MAX_MONEY/getMultipleFactor(factorUnit)<Decimal(subMoneys[0]):
                            return Decimal(0)
                        result += Decimal(subMoneys[0])*(getMultipleFactor(factorUnit))
                    elif len(subMoneys[0])==1:
                        if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[0]) is not None:
                            result += Decimal(getDigitsDic(subMoneys[0]))*(getMultipleFactor(factorUnit))
                    # subMoneys[0]中无金额单位，不可再拆分
                    elif subMoneys[0]=="":
                        result += 0
                    elif re.search(re.compile("[%s]"%("".join(chnFactorUnits))),subMoneys[0]) is None:
                        # print(subMoneys)
                        # subMoneys[0] = subMoneys[0][0]
                        result += Decimal(getUnifyMoney(subMoneys[0])) * (getMultipleFactor(factorUnit))
                    else:
                        result += Decimal(getUnifyMoney(subMoneys[0]))*(getMultipleFactor(factorUnit))
                    if len(subMoneys)>1:
                        if re.search(re.compile("^(\d+(,)?)+(\.\d+)?[百千万亿]?\s?(元)?$"),subMoneys[1]) is not None:
                            result += Decimal(subMoneys[1])
                        elif len(subMoneys[1])==1:
                            if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[1]) is not None:
                                result += Decimal(getDigitsDic(subMoneys[1]))
                        else:
                            result += Decimal(getUnifyMoney(subMoneys[1]))
                    break
        except Exception as e:
            # traceback.print_exc()
            return Decimal(0)
        return result

    # 提取表格用于判断业绩
    if html:
        soup = BeautifulSoup(html, 'lxml')
        tables = soup.find_all('table')
        tables_and_divs = soup.find_all(['table', 'div'])
    else:
        tables_and_divs = []

    money_list = []
    # 使用正则识别金额
    entity_type = "money"
    list_money_pattern = {"cn": "(()(?P<filter_kw>百分之)?(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
                          "key_word": "((?P<text_key_word>(?:[￥¥]+，?|[单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果)(?:[,，\[（\(]*\s*(人民币|单位：)?/?(?P<unit_key_word_before>[万亿]?(?:[美日欧]元|元(/(M2|[\u4e00-\u9fa5]{1,3}))?)?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[\]）\)]?)\s*[，,:：]*(RMB|USD|EUR|JPY|CNY)?[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P<money_key_word>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_key_word>(E-?\d+))?[百千]{,1})(?:[（\(]?(?P<filter_>[%％‰折])*\s*，?((金额)?单位[:：])?(?P<unit_key_word_behind>[万亿]?(?:[美日欧]元|元)?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[）\)]?))",
                          "front_m": "((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万亿]?(?:[美日欧]元|元))\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,7}?))(?P<money_front_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_front_m>(E-?\d+))?(?:，?)[百千]*)())",
                          "behind_m": "(()()(?P<money_behind_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_behind_m>(E-?\d+))?(?:，?)[百千]*)(人民币)?[\(（]?(?P<unit_behind_m>[万亿]?(?:[美日欧]元|元)(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\)）]?)"}
    # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。

    pattern_money = re.compile("%s|%s|%s|%s" % (
        list_money_pattern["cn"], list_money_pattern["key_word"], list_money_pattern["behind_m"],
        list_money_pattern["front_m"]))

    # 修复 元千万元
    match = re.search('元[千百十]万元', sentence_text)
    if match:
        sentence_text = re.sub(re.escape(match.group()), match.group()[1:], sentence_text)

    # 修复 千多万元
    match = re.search('[千百十]多万元', sentence_text)
    if match:
        sentence_text = re.sub(re.escape(match.group()), match.group()[0] + match.group()[2:], sentence_text)

    if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text):
        found_yeji += 1
    if found_yeji >= 2:  # 过滤掉业绩后面的所有金额
        all_match = []
    else:
        ser = re.search('((收费标准|计算[方公]?式)：|\w{3,5}\s*=)+\s*[中标投标成交金额招标人预算价格万元\s（）()\[\]【】\d\.%％‰\+\-*×/]{20,}[，。]?', sentence_text)  # 过滤掉收费标准里面的金额
        if ser:
            all_match = re.finditer(pattern_money, sentence_text.replace(ser.group(0), ' ' * len(ser.group(0))))
        else:
            all_match = re.finditer(pattern_money, sentence_text)
    for _match in all_match:
        if len(_match.group()) > 0:
            notes = ''  # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
            unit = ""
            entity_text = ""
            start_index = ""
            end_index = ""
            text_beforeMoney = ""
            filter = ""
            filter_unit = False
            notSure = False
            science = ""
            if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text[:_match.span()[0]]):  # 2021/7/21过滤掉业绩后面金额
                # print('金额在业绩后面: ', _match.group(0))
                found_yeji += 1
                break
            yeji_table_flag = 0
            for k, v in _match.groupdict().items():
                if v != "" and v is not None:
                    if k == 'text_key_word':
                        notSure = True
                    if k.split("_")[0] == "money":
                        entity_text = v
                        # if is_yeji_table(tables_and_divs, entity_text):
                        if judge_yeji(len(sentence_text), sentence_text, 300, tables_and_divs, entity_text):
                            yeji_table_flag = 1
                            break
                        # print(_match.group(k), 'entity_text: ', sentence_text[_match.start(k): _match.end(k)])
                        if entity_text.endswith(',00'):  # 金额逗号后面不可能为两个0结尾，应该小数点识别错，直接去掉
                            entity_text = entity_text[:-3]
                    if k.split("_")[0] == "unit":
                        if v == '万元' or unit == "":  # 处理  预算金额(元)：160万元 这种出现前后单位不一致情况
                            unit = v
                    if k.split("_")[0] == "text":
                        # print('text_before: ', _match.group(k))
                        text_beforeMoney = v
                    if k.split("_")[0] == "filter":
                        filter = v
                    if re.search("filter_unit", k) is not None:
                        filter_unit = True
                    if k.split("_")[0] == 'science':
                        science = v

            if yeji_table_flag:
                continue

            if filter != "":
                continue
            start_index, end_index = _match.span()
            start_index += len(text_beforeMoney)

            '''过滤掉手机号码作为金额'''
            if re.search('电话|手机|联系|方式|编号|编码|日期|数字|时间', text_beforeMoney):
                # print('过滤掉手机号码作为金额')
                continue
            elif re.search('^1[3-9]\d{9}$', entity_text) and re.search('：\w{1,3}$', text_beforeMoney): # 过滤掉类似 '13863441880', '金额（万元）：季勇13863441880'
                # print('过滤掉手机号码作为金额')
                continue

            if unit == "":  # 2021/7/21 有明显金额特征的补充单位，避免被过滤
                if (re.search('(￥|¥|RMB|CNY)[:：]?$', text_beforeMoney) or re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,}', entity_text)):
                    if entity_text.endswith('万元'):
                        unit = '万元'
                        entity_text = entity_text[:-2]
                    else:
                        unit = '元'
                    # print('1明显金额特征补充单位 元')
                elif re.search('USD[:：]?$', text_beforeMoney):
                    unit = '美元'
                elif re.search('EUR[:：]?$', text_beforeMoney):
                    unit = '欧元'
                elif re.search('JPY[:：]?$', text_beforeMoney):
                    unit = '日元'
                elif re.search('^[-—]+[\d,.]+万元', sentence_text[end_index:]):
                    # print('两个金额连接后面的有单位，用后面单位')
                    unit = '万元'
                elif re.search('([单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费)[:：为]*-?$', text_beforeMoney.strip()) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}', entity_text) == None:
                    if re.search('^[\d，,.]+$', entity_text) and float(re.sub('[,，]', '', entity_text))<500 and re.search('万元', sentence_text):
                        unit = '万元'
                        # print('金额较小且句子中有万元的，补充单位为万元')
                    elif re.search('^\d{1,3}\.\d{4,6}$', entity_text) and re.search('0000$', entity_text) == None:
                        unit = '万元'
                    else:
                        unit = '元'
                        # print('金额前面紧接关键词的补充单位 元')
                elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)', entity_text):
                    unit = '元'
                    # print('3明显金额特征补充单位 元')
                else:
                    # print('过滤掉没单位金额: ',entity_text)
                    continue
            elif unit == '万元':
                if end_index < len(sentence_text) and sentence_text[end_index] == '元' and re.search('\d$', entity_text):
                    unit = '元'
                elif re.search('^[5-9]\d{6,}\.\d{2}$', entity_text): # 五百亿以上的万元改为元
                    unit = '元'

            if unit.find("万") >= 0 and entity_text.find("万") >= 0:  # 2021/7/19修改为金额文本有万，不计算单位
                # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit)
                unit = "元"

            if re.search('.*万元万元', entity_text):  # 2021/7/19 修正两个万元
                # print(' 修正两个万元',entity_text)
                entity_text = entity_text.replace('万元万元', '万元')
            else:
                if filter_unit:
                    continue

            entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", entity_text)
            # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
            if re.search('总投资|投资总额|总预算|总概算|投资规模|批复概算|投资额|投资估算|投资概算',
                         sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]):  # 2021/8/5过滤掉总投资金额
                # print('总投资金额: ', _match.group(0))
                notes = '总投资'
            elif re.search('建安费|建安工程费|建筑安装工程费|建安工程造价',
                           sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]):  # 2021/11/18 投资金额不作为招标金额
                notes = '建安费'
            elif re.search('总工程造价|总造价',
                           sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]):  # 2021/12/20 工程造价不作为招标金额
                notes = '工程造价'

            # 直接判断前面是否有业绩
            if judge_yeji(max(0, _match.span()[0] - 10), sentence_text):
                continue

            if len(unit) > 0:
                if unit.find('万') >= 0 and len(entity_text.split('.')[0]) >= 8:  # 2021/7/19 修正万元金额过大的情况
                    entity_text = str(
                        getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]) / 10000)
                    unit = '元'  # 修正金额后单位 重置为元
                else:
                    entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]))
            else:
                if entity_text.find('万') >= 0 and entity_text.split('.')[0].isdigit() and len(
                        entity_text.split('.')[0]) >= 8:
                    entity_text = str(getUnifyMoney(entity_text) / 10000)
                    # print('修正金额字段含万 过大的情况')
                else:
                    entity_text = str(getUnifyMoney(entity_text))
            if science and re.search('^E-?\d+$', science):  # 科学计数
                entity_text = str(Decimal(entity_text + science)) if Decimal(entity_text + science) > 100 and Decimal(
                    entity_text + science) < 10000000000 else entity_text  # 结果大于100及小于100万才使用科学计算

            if float(entity_text) > 100000000000:  # float(entity_text)<100 or  2022/3/4 取消最小金额限制
                continue

            if notSure and unit == "" and float(entity_text) > 100 * 10000:
                # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit)
                continue
            # print("金额：{0} ,单位：{1}, 前文：{2}, filter: {3}, filter_unit: {4}".format(entity_text, unit, text_beforeMoney,
            #                                                                      filter, filter_unit))
            if re.search('[%％‰折]|费率|下浮率', text_beforeMoney) and float(entity_text)<1000: # 过滤掉可能是费率的金额
                # print('过滤掉可能是费率的金额')
                continue
            money_list.append((entity_text, start_index, end_index, unit, notes))

    # 排除过小的金额
    temp_list = []
    for money in money_list:
        if int(float(money[0])) < 1000:
            continue
        temp_list.append(money)
    money_list = temp_list

    return money_list, found_yeji


def chinese_to_arabic(ch_str):
    chinese_number_dict = {
        '一': 1,
        '二': 2,
        '两': 2,
        '三': 3,
        '四': 4,
        '五': 5,
        '六': 6,
        '七': 7,
        '八': 8,
        '九': 9,
        '十': 10,
        '拾': 10,
        '百': 100,
        '千': 1000,
    }
    no_list = []
    for c in ch_str:
        if c not in chinese_number_dict.keys():
            return None
        no_list.append(chinese_number_dict.get(c))
    arabic_num = 0
    mul_no = None
    for i, no in enumerate(no_list):
        if no in [10, 100, 1000]:
            if mul_no is None:
                arabic_num += no
            else:
                arabic_num += no * mul_no
            mul_no = None
        else:
            mul_no = no
    if mul_no:
        arabic_num += mul_no

    return arabic_num


def reg_word_sort(reg):
    ss = reg.split('|')
    ss.sort(key=lambda x: len(x), reverse=True)
    reg = '|'.join(ss)
    return reg


def get_stage_pattern():
    stage_dict = {
        "立项阶段": "立项|项目投资",
        "可研阶段": "可行性研究|可研",
        "环评阶段": "环境评价|环境影响|环境评测|环评|(水保|环保|环境保护)(编制|验收|监测)",
        "稳评阶段": "稳定风险|社会稳定|风险评估",
        "咨询阶段": "(水影响|能源|交通影响|地质灾害|地址灾害|地震安全性|地震安全性|气象|雷击风险|安全|海洋|森林环境)(评[价估测])|水土保持|(水|交|灾|震|能|气|安|海|林)评",
        "造价阶段": "(决算书|预算|结算|造价|决算)(编制|咨询)",
        "设计阶段": "(施工图(纸|)|初步|项目|工程)(方案|)设计|测绘|规划设计",
        "勘察阶段": "(勘察|勘查)设计|勘察技术|勘查|勘察",
        "施工图审": "(施工图(纸|)|防雷|消防|人防)审查|施工图审",
        "施工许可": "施工许可证",
        "施工准备": "施工准备|监理|资格预审|资审",
        "施工在建": "施工",
        "竣工阶段": "竣工|验收",
        "EPC总承包": "总承包|EPC"
    }

    stage_priority_dict = {
        "立项阶段": 1,
        "可研阶段": 3,
        "环评阶段": 2,
        "稳评阶段": 3,
        "咨询阶段": 2,
        "造价阶段": 2,
        "设计阶段": 4,
        "勘察阶段": 4,
        "施工图审": 2,
        "施工许可": 2,
        "施工准备": 3,
        "施工在建": 5,
        "竣工阶段": 3,
        "EPC总承包": 4
    }

    list_stage_v = []
    for k,v in stage_dict.items():
        list_stage_v.append("(?P<%s>%s)"%(k,v))
    stage_pattern = "|".join(list_stage_v)
    return stage_pattern, stage_priority_dict


def get_industry_pattern():
    filename = os.path.abspath(os.path.dirname(__file__)) + "/proposedBuildingKeyword.xlsx"
    df = pd.read_excel(filename)
    dict_industry_keywords = {}
    for _industry, _keyword in zip(df["类别"], df["关键词"]):
        if _industry not in dict_industry_keywords:
            dict_industry_keywords[_industry] = set()
        dict_industry_keywords[_industry].add(_keyword)
    list_industry_p = []
    for k, v in dict_industry_keywords.items():
        if len(v) > 0:
            list_industry_p.append("(?P<%s>%s)" % (k, "|".join(list(v))))
    # logging.info('get_industry_pattern ' + str(list_industry_p))
    _pattern = re.compile("|".join(list_industry_p))
    return _pattern


def get_property_pattern():
    property_dict = {
        "复合性质": "扩迁建|扩改建|扩翻建|改扩建|改翻建|迁扩建|迁改建|迁扩建|翻改建|翻扩建",
        "迁建": "迁建|搬迁重建",
        "扩建": "扩建|增建|加建|扩大",
        "翻建": "翻建|拆除重建",
        "改建": "改造|改建|技改|提升|改进|整改",
        "装饰装修": "装修|室内装饰|室外装饰|装饰工程|装饰改造工程|装饰装修|装修装饰|幕墙工程|修缮|外墙翻新|翻新工程|整修|修补|装饰|维修",
        "拆除": "拆除",
        "恢复重建": "恢复重建|灾后重建",
        '其他': '整治|修复|环境治理|更换'
    }

    property_priority_dict = {
        "复合性质": 1,
        "迁建": 2,
        "扩建": 3,
        "翻建": 4,
        "改建": 5,
        "装饰装修": 6,
        "拆除": 7,
        "恢复重建": 8,
        '其他': 9,
        "新建": 10
    }

    list_property_v = []
    for k,v in property_dict.items():
        list_property_v.append("(?P<%s>%s)"%(k,v))
    property_pattern = "|".join(list_property_v)
    return property_pattern, property_priority_dict