|
|
@@ -0,0 +1,1729 @@
|
|
|
+import copy
|
|
|
+import os
|
|
|
+import re
|
|
|
+import sys
|
|
|
+import traceback
|
|
|
+from decimal import Decimal
|
|
|
+import pandas as pd
|
|
|
+from bs4 import BeautifulSoup
|
|
|
+
|
|
|
+sys.path.append(os.path.abspath(os.path.dirname(__file__)) + '/../../../')
|
|
|
+
|
|
|
+from BiddingKG.dl.common.Utils import spanWindow, timeFormat
|
|
|
+
|
|
|
+
|
|
|
+class PBPredictor:
|
|
|
+ def __init__(self):
|
|
|
+ self.stage_pattern, self.stage_priority_dict = get_stage_pattern()
|
|
|
+ self.industry_pattern = get_industry_pattern()
|
|
|
+ self.property_pattern, self.property_priority_dict = get_property_pattern()
|
|
|
+ with open(os.path.abspath(os.path.dirname(__file__)) + '/structure_keyword.txt', 'r', encoding='utf-8') as f:
|
|
|
+ self.structure_keyword_list = f.readlines()
|
|
|
+
|
|
|
+ def get_col_from_prem(self, prem):
|
|
|
+ tenderee, agency, product = None, None, None
|
|
|
+ for item in prem:
|
|
|
+ prem = item.get('prem')
|
|
|
+ for key in prem.keys():
|
|
|
+ project = prem.get(key)
|
|
|
+ role_list = project.get('roleList')
|
|
|
+ for role_dict in role_list:
|
|
|
+ if tenderee is None and role_dict.get('role_name') == 'tenderee':
|
|
|
+ tenderee = role_dict.get('role_text')
|
|
|
+ if agency is None and role_dict.get('role_name') == 'agency':
|
|
|
+ agency = role_dict.get('role_text')
|
|
|
+ product = item.get('product')
|
|
|
+ begin_time = item.get('time_commencement')
|
|
|
+ end_time = item.get('time_completion')
|
|
|
+ return tenderee, agency, product, begin_time, end_time
|
|
|
+
|
|
|
+ def predict(self, prem, list_articles, list_sentences, list_entitys, doctitle, code_name_dict, dochtmlcon):
|
|
|
+ try:
|
|
|
+ for list_article, list_sentence, list_entity in zip(list_articles, list_sentences, list_entitys):
|
|
|
+ list_sentence.sort(key=lambda x: x.sentence_index)
|
|
|
+ tenderee, agency, product, begin_time, end_time = self.get_col_from_prem(prem)
|
|
|
+ content = list_article.content
|
|
|
+ dochtmlcon = re.sub('[\r\n]', '', dochtmlcon)
|
|
|
+ # 获取无附件的html
|
|
|
+ soup = BeautifulSoup(dochtmlcon, 'lxml')
|
|
|
+ attachment_div = soup.find('div', class_='richTextFetch')
|
|
|
+ if attachment_div:
|
|
|
+ attachment_div.decompose()
|
|
|
+ content_no_att = soup.text
|
|
|
+ else:
|
|
|
+ content_no_att = ''
|
|
|
+
|
|
|
+ project_name = code_name_dict.get('name')
|
|
|
+ project_code = code_name_dict.get('code')
|
|
|
+ if project_code:
|
|
|
+ project_code = project_code[0]
|
|
|
+ else:
|
|
|
+ project_code = None
|
|
|
+
|
|
|
+ stage = extract_legal_stage(project_name+doctitle, self.stage_pattern, self.stage_priority_dict, product, tenderee=tenderee, agency=agency)
|
|
|
+ industry1 = extract_industry(doctitle+content, self.industry_pattern)
|
|
|
+ industry = extract_industry(doctitle+content_no_att, self.industry_pattern)
|
|
|
+ # print('industry', industry, industry1)
|
|
|
+ if not industry and industry1:
|
|
|
+ industry = industry1
|
|
|
+ proportion1, proportion = extract_proportion(content)
|
|
|
+ project_digest = extract_project_digest(content)
|
|
|
+ project_address = extract_project_address(list_sentence, list_entity)
|
|
|
+ location = get_bid_location(doctitle+"\t"+project_name)
|
|
|
+ project_name_refind, show_name_refind = get_project_name_refind(project_name, doctitle, tenderee, agency)
|
|
|
+ has_elevator = extract_has_elevator(content)
|
|
|
+ project_property = extract_project_property(doctitle+"\t"+project_name, self.property_pattern, self.property_priority_dict)
|
|
|
+ total_invest, construct_install_fee, engineer_cost = extract_several_money(list_sentence, dochtmlcon)
|
|
|
+ max_floor = extract_max_floor(content, dochtmlcon)
|
|
|
+ structure = extract_structure(content, dochtmlcon, self.structure_keyword_list)
|
|
|
+ has_steel = extract_has_steel_structure(structure)
|
|
|
+ wall_type, wall_type2 = extract_wall_type(doctitle+"\t"+project_name, content)
|
|
|
+
|
|
|
+ if stage is not None:
|
|
|
+ has_stage = 1
|
|
|
+ else:
|
|
|
+ has_stage = 0
|
|
|
+
|
|
|
+ pb_json = {
|
|
|
+ 'tenderee': tenderee,
|
|
|
+ 'agency': agency,
|
|
|
+ 'project_code': project_code,
|
|
|
+ 'project_name': project_name,
|
|
|
+ 'doctitle': doctitle,
|
|
|
+ 'stage': stage,
|
|
|
+ 'industry': industry,
|
|
|
+ 'proportion': proportion,
|
|
|
+ 'projectDigest': project_digest,
|
|
|
+ 'projectAddress': project_address,
|
|
|
+ 'location': location,
|
|
|
+ 'project_name_refind': project_name_refind,
|
|
|
+ 'has_elevator': has_elevator,
|
|
|
+ 'project_property': project_property,
|
|
|
+ 'total_invest': total_invest,
|
|
|
+ 'construct_install_fee': construct_install_fee,
|
|
|
+ 'engineer_cost': engineer_cost,
|
|
|
+ 'max_floor': max_floor,
|
|
|
+ 'structure': structure,
|
|
|
+ 'has_steel': has_steel,
|
|
|
+ 'wall_type': wall_type,
|
|
|
+ 'wall_type2': wall_type2,
|
|
|
+ 'begin_time': begin_time,
|
|
|
+ 'end_time': end_time,
|
|
|
+ 'has_stage': has_stage,
|
|
|
+ }
|
|
|
+
|
|
|
+ pb_json = {'pb': pb_json}
|
|
|
+ return pb_json
|
|
|
+
|
|
|
+ except:
|
|
|
+ traceback.print_exc()
|
|
|
+ return {'pb': 'error'}
|
|
|
+
|
|
|
+
|
|
|
+def extract_legal_stage(content, _pattern, priority_dict, product='', tenderee='', agency=''):
|
|
|
+ # 判断这几类直接返回
|
|
|
+ if not content:
|
|
|
+ return None
|
|
|
+ if re.search("拍卖|转让|产权|出让|租赁|招租", content) is not None:
|
|
|
+ return None
|
|
|
+ # 去掉招标人代理人
|
|
|
+ content = re.sub(str(tenderee), '', content)
|
|
|
+ content = re.sub(str(agency), '', content)
|
|
|
+
|
|
|
+ # 竣工阶段标志：消防、物业、安保、装修、通风系统、排烟、第三方检测
|
|
|
+ # if re.search("消防|物业|安保|装修|通风系统|排烟|第三方检测", content) is not None:
|
|
|
+ # return '竣工阶段'
|
|
|
+
|
|
|
+ # 替换混淆词
|
|
|
+ _content = re.sub("设计院|设计总院|造价咨询有限公司", "", content)
|
|
|
+
|
|
|
+ list_stage = []
|
|
|
+ for stage_search in re.finditer(_pattern, _content):
|
|
|
+ for k,v in stage_search.groupdict().items():
|
|
|
+ if v is not None:
|
|
|
+ list_stage.append([k, priority_dict.get(k)])
|
|
|
+ if len(list_stage) > 0:
|
|
|
+ list_stage.sort(key=lambda x: x[1])
|
|
|
+ stage = list_stage[0][0]
|
|
|
+
|
|
|
+ # 用product判断竣工阶段是否合法
|
|
|
+ if product:
|
|
|
+ if not re.search('施工|工程|建设', str(product)):
|
|
|
+ stage = None
|
|
|
+ for s in list_stage:
|
|
|
+ if s[0] != '竣工阶段':
|
|
|
+ stage = s[0]
|
|
|
+ break
|
|
|
+
|
|
|
+ # 立项排除立项目
|
|
|
+ if stage == '立项阶段':
|
|
|
+ sub_content = re.sub('立项目', '', _content)
|
|
|
+ for stage_search in re.finditer(_pattern, sub_content):
|
|
|
+ for k,v in stage_search.groupdict().items():
|
|
|
+ if v is not None:
|
|
|
+ list_stage.append([k, priority_dict.get(k)])
|
|
|
+ if len(list_stage)>0:
|
|
|
+ list_stage.sort(key=lambda x: x[1])
|
|
|
+ stage = list_stage[0][0]
|
|
|
+
|
|
|
+ return stage
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def get_project_name_refind(project_name, doctitle, tenderee='', agency= '', min_len=3):
|
|
|
+ # 跳过部分
|
|
|
+ re_str11 = '网上超市|服务市场采购|印刷服务|复印纸|车辆维修和保养|商品房预售|办公家具定点|直接订购|定点议价' \
|
|
|
+ '|政府采购意向|信息技术服务定点议价|信息技术服务定点采购|法人章刻制中介机构|专用设备|办公设备采购' \
|
|
|
+ '|线上摇号选取'
|
|
|
+ re_str12 = '物业'
|
|
|
+ re_str13 = '公共资源交易平台'
|
|
|
+ re_str19 = '环境影响评价(文件|)(审批|审核|受理)|拟作出的建设'
|
|
|
+
|
|
|
+ # 干扰部分
|
|
|
+ re_str1 = "＜.*?＞［.*?］|｛.*?｝|〔.*?〕|《.*?》|【.*?】|\(.*?\)|\[.*?\]|（.*?）|\d{1,2}月\d{1,2}[日号]|\d{1,2}:\d{2}"
|
|
|
+
|
|
|
+ re_str4 = '[,.:;，。：；\'\"“”‘’\-/<>#@!$%&*+=·￥|？?｜－＋＃＂﹝﹒！]'
|
|
|
+
|
|
|
+ re_str5 = '[工程项目建设拟对年批第作出的个及在已]|标段|EPC|总承包|招标|文件|开标|记录|公示|验收|勘察|编制|公开选取|准予|论证|各单位|附件|建筑业' \
|
|
|
+ '|责任|诚挚|拟作出审批意见|生产|涉及|消防|政府|投资|方案|技术支持|文件|研发|申请报告|出具|现代|产业|依法|报批|行政|审批|许可|开展' \
|
|
|
+ '|活动|开放日|系列|某部|零星工程|某(地产|型号|单位)|权限内|办理了|外商|我院|召开|我市|启动|我单位|我(县|区|会)|成功|举办|举行' \
|
|
|
+ '|被评为|征(求|询)|包括|不包括|层层分包|合同估算价|万元以内|组织|全(市|区|县)|承接|积极|针对|企业|小规模|安全|助推|装修|改造' \
|
|
|
+ '|新建|居住|技术|建设|建筑|安装|园林|绿化|信息化|采购|商品房|预售|许可|房产|测量|报告|业务|零星|维修|水土保持|扩建|夜间|工地' \
|
|
|
+ '|整治|高速公路|备案|加油站|设施|环境|保护|合同|履约|在线|询价|面积|受理|经济中心|服务|食品|加工|利用|公开|选取|动物|疫苗|框架' \
|
|
|
+ '|协议|房屋|中国|不动产|实验室|限额|以下|单位|入围|审查|合格|意见|新能源|常规|许可|申请|加工|制品|建议书|可研|结算|审核|遴选' \
|
|
|
+ ''
|
|
|
+
|
|
|
+ re_str6 = '总承包|设计|环评|监理|施工|竣工|项目|工程|EPC|验收|勘察设计|全过程造价咨询|造价咨询|勘察|可行性研究报告|初步设计|社会稳定风险评估|测绘' \
|
|
|
+ '|(地震安全性|环境影响|水土保持)评(价估)'
|
|
|
+
|
|
|
+ re_str7 = '许可信息公开表|办理结果公示|审批信息公开表|验收公示表|信息披露|备案登记表|验收结果公示|' \
|
|
|
+ '审批意见公开|受理公示|施工许可|情况说明|合同纠纷调解|施工许可双公示|施工许可证|政策'
|
|
|
+
|
|
|
+ re_str8 = '[〔〕（）\[\]《》()【】｛｝{}［］＜＞]'
|
|
|
+
|
|
|
+ re_str14 = '[〔（\[(【｛{［＜](采购结果|设计|环评|监理|施工|竣工|工程|EPC|验收|勘察设计|全过程造价咨询|造价咨询|勘察)[〕）\]》)】｝}］＞]'
|
|
|
+
|
|
|
+ # 截取部分
|
|
|
+ re_str2 = '机场快线|科技园|产业园|工业园|工程|项目|施工|竣工|总承包|改造|监理|可研|验收|勘察设计|全过程造价咨询|造价咨询|勘察|可行性研究报告|EPC|初步设计|社会稳定风险评估'
|
|
|
+ re_str3 = '关于(公布核准的|一种用于|核发|作出|同意|开展|调整|请求|规范|要求|进一步|遴选|领取)|关于公[布开示]|关于[为对]|关于|公司|集团|局|委托'
|
|
|
+ re_str9 = '改扩建|建设|迁改|土建|测绘|(地震安全性|环境影响|水土保持)评(价估)'
|
|
|
+
|
|
|
+ # 混淆部分
|
|
|
+ re_str10 = '局部'
|
|
|
+ re_str17 = '(工程|信息|)有限|公司|集团|局|大学|院|学校|学院|中心'
|
|
|
+ re_str18 = '(设计|造价|咨询|建设|项目|管理|工程)+有限|(信息|职业|技术|管理)+(大学|学校|学院|中心|院|所)'
|
|
|
+ re_str26 = '服务类|(设计|造价|咨询|建设|项目|管理|工程)+(大学|学校|学院|中心|院|所|集团|局)'
|
|
|
+
|
|
|
+ # 需判断删除部分
|
|
|
+ re_str15 = '[ 、#※.?<|①=-_—]|=-|##|\*+|[\(（](001|[0-9]|[一二三四五六七八九十]+|)[\)）]|(0001|0000|001|002|01|02)+'
|
|
|
+ re_str16 = '[0-9][.、]'
|
|
|
+
|
|
|
+ # 删除特定表达
|
|
|
+ re_str20 = '公共资源交易中心.*关于'
|
|
|
+ re_str21 = '[\u4e00-\u9fff]{2,}市[\u4e00-\u9fff]{2,}区'
|
|
|
+ re_str22 = '[\u4e00-\u9fff]{2,4}区[^至]'
|
|
|
+ re_str23 = '.{1,2}招标公告|(PDF|pdf)(版|)'
|
|
|
+ re_str25 = '(小区)$'
|
|
|
+ re_str27 = '[\u4e00-\u9fff]{2,3}省|[\u4e00-\u9fff]{2,3}市'
|
|
|
+
|
|
|
+ re_str_area = '华北|华南|华东|华中|西南|东北|西北'
|
|
|
+ re_str_province = '北京|天津|河北|山西|内蒙古|广东|海南|广西|上海|江苏|浙江|安徽|福建|江西|山东|河南|湖北|湖南|重庆|四川|贵州|云南|西藏|黑龙江|辽宁|吉林|陕西|甘肃|青海|宁夏|新疆|台湾|香港|澳门'
|
|
|
+ re_str_city = '东城|和平|石家庄|唐山|秦皇岛|邯郸|邢台|保定|张家口|承德|沧州|廊坊|衡水|太原|大同|阳泉|长治' \
|
|
|
+ '|晋城|朔州|晋中|运城|忻州|临汾|吕梁|呼和浩特|包头|乌海|赤峰|通辽|鄂尔多斯|呼伦贝尔|巴彦淖尔' \
|
|
|
+ '|乌兰察布|兴安盟|锡林郭勒盟|阿拉善盟|广州|韶关|深圳|珠海|汕头|佛山|江门|湛江|茂名|肇庆|惠州' \
|
|
|
+ '|梅州|汕尾|河源|阳江|清远|潮州|揭阳|云浮|海口|三亚|南宁|柳州|桂林|梧州|北海|防城港|钦州|贵港' \
|
|
|
+ '|玉林|百色|贺州|河池|来宾|崇左|黄浦|南京|无锡|徐州|常州|苏州|南通|连云港|淮安|盐城|扬州|镇江' \
|
|
|
+ '|泰州|宿迁|杭州|宁波|温州|嘉兴|湖州|绍兴|金华|衢州|舟山|台州|丽水|合肥|芜湖|蚌埠|淮南|马鞍山' \
|
|
|
+ '|淮北|铜陵|安庆|黄山|滁州|阜阳|宿州|六安|亳州|池州|宣城|福州|厦门|莆田|三明|泉州|漳州|南平' \
|
|
|
+ '|龙岩|宁德|南昌|景德镇|萍乡|九江|新余|鹰潭|赣州|吉安|宜春|抚州|上饶|济南|青岛|淄博|枣庄' \
|
|
|
+ '|东营|烟台|潍坊|济宁|泰安|威海|日照|临沂|德州|聊城|滨州|菏泽|郑州|开封|洛阳|平顶山|安阳|鹤壁' \
|
|
|
+ '|新乡|焦作|濮阳|许昌|漯河|三门峡|南阳|商丘|信阳|周口|驻马店|武汉|黄石|十堰|宜昌|襄阳|鄂州' \
|
|
|
+ '|荆门|孝感|荆州|黄冈|咸宁|随州|恩施土家族|长沙|株洲|湘潭|衡阳|邵阳|岳阳|常德|张家界|益阳' \
|
|
|
+ '|郴州|永州|怀化|娄底|湘西土家族|万州|成都|自贡|攀枝花|泸州|德阳|绵阳|广元|遂宁|内江|乐山' \
|
|
|
+ '|南充|眉山|宜宾|广安|达州|雅安|巴中|资阳|阿坝藏族羌族|甘孜藏族|凉山彝族|贵阳|六盘水|遵义' \
|
|
|
+ '|安顺|铜仁|黔西南布依族|毕节|黔东南苗族|黔南布依族|昆明|曲靖|玉溪|保山|昭通|丽江|普洱|临沧' \
|
|
|
+ '|楚雄彝族|红河哈尼族|文山|西双版纳傣族|大理白族|德宏傣族景颇族|怒江傈僳族|迪庆藏族|拉萨|昌都' \
|
|
|
+ '|山南|日喀则|那曲|阿里地区|林芝|哈尔滨|齐齐哈尔|鸡西|鹤岗|双鸭山|大庆|伊春|佳木斯|七台河' \
|
|
|
+ '|牡丹江|黑河|绥化|大兴安岭|沈阳|大连|鞍山|抚顺|本溪|丹东|锦州|营口|阜新|辽阳|盘锦|铁岭' \
|
|
|
+ '|朝阳|葫芦岛|长春|吉林|四平|辽源|通化|白山|松原|白城|延边朝鲜族|西安|铜川|宝鸡|咸阳|渭南' \
|
|
|
+ '|延安|汉中|榆林|安康|商洛|兰州|嘉峪关|金昌|白银|天水|武威|张掖|平凉|酒泉|庆阳|定西|陇南' \
|
|
|
+ '|临夏回族自治州|甘南藏族|西宁|海东|海北藏族|黄南藏族|海南藏族|果洛藏族|玉树藏族|海西蒙古族' \
|
|
|
+ '|银川|石嘴山|吴忠|固原|中卫|乌鲁木齐|克拉玛依|吐鲁番|哈密|昌吉|博尔塔拉蒙古|巴音郭楞蒙古' \
|
|
|
+ '|阿克苏|克孜勒苏柯尔克孜|喀什|和田地区|伊犁|伊犁哈萨克|塔城地区|阿勒泰|中山|东莞|天门|仙桃|潜江' \
|
|
|
+ '|石河子|五家渠|阿拉尔|图木舒克|三沙|儋州|涪陵|永川|西城|朝阳|丰台|石景山|海淀|门头沟' \
|
|
|
+ '|房山|通州|顺义|昌平|大兴|怀柔|平谷|密云|延庆|河东|河西|河北区|红桥|东丽|西青|津南|北辰' \
|
|
|
+ '|武清|宝坻|滨海|宁河|静海|蓟州|渝中|大渡口|江北|沙坪坝|九龙坡|南岸|北碚|綦江|大足|渝北' \
|
|
|
+ '|巴南|黔江|长寿|江津|合川|南川|璧山|铜梁|潼南|荣昌|开州|徐汇|长宁|静安|普陀|虹口|杨浦' \
|
|
|
+ '|闵行|宝山|嘉定|浦东新|金山|松江|青浦|奉贤|崇明|济源|神农架林区|五指山|文昌|琼海|万宁' \
|
|
|
+ '|东方|定安|屯昌|澄迈|临高|白沙黎族|昌江黎族|乐东黎族|陵水黎族|保亭黎族|琼中黎族|梁平' \
|
|
|
+ '|丰都|城口|垫江|忠县|云阳|奉节|巫山|巫溪|石柱|秀山|武隆|酉阳|彭水|南开|北屯|铁门关' \
|
|
|
+ '|双河|可克达拉|昆玉|胡杨河'
|
|
|
+
|
|
|
+ re_str28 = '({})(地区)?|({})省?|({})[区市]?'.format(re_str_area, re_str_province, re_str_city)
|
|
|
+ re_str29 = '(({})(地区)?({})省?)|(({})省?({})[区市]?)'.format(re_str_area, re_str_province, re_str_province, re_str_city)
|
|
|
+
|
|
|
+ # 直接删除部分
|
|
|
+ re_str24 = '(的|)(竞争性谈判|竞争性磋商|磋商|中标|单一来源|招标|更正)(采购|)(公告|)'
|
|
|
+
|
|
|
+ add_col = project_name if project_name else '' + doctitle if doctitle else ''
|
|
|
+ if re.search(re_str11, add_col) and not re.search(re_str12, add_col):
|
|
|
+ return '', ''
|
|
|
+
|
|
|
+ from_col_list = [project_name, doctitle]
|
|
|
+ name_refind_flag_dict = {'True': [], 'False': []}
|
|
|
+ for col in from_col_list:
|
|
|
+ name_refind = ""
|
|
|
+ match_flag = False
|
|
|
+ if col is not None and len(col) > 0:
|
|
|
+ name_refind = col
|
|
|
+
|
|
|
+ # 部分跳过
|
|
|
+ if re.search(re_str13, name_refind):
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 替换特定表达
|
|
|
+ match = re.search(re_str20, name_refind)
|
|
|
+ if match:
|
|
|
+ name_refind = name_refind[match.span()[1]:]
|
|
|
+
|
|
|
+ # 去掉干扰
|
|
|
+ name_refind = re.sub('年度', '年', name_refind)
|
|
|
+ name_refind = re.sub(re_str4, '', name_refind)
|
|
|
+ name_refind = re.sub(re_str14, '', name_refind)
|
|
|
+
|
|
|
+ # print('name_refind', name_refind)
|
|
|
+
|
|
|
+ # 连续截取工程前的，看哪一部分最适合当refind
|
|
|
+ match = re.finditer(re_str2, name_refind)
|
|
|
+ prob_name_list = []
|
|
|
+ last_index = 0
|
|
|
+ project_word_in_org = []
|
|
|
+ for m in match:
|
|
|
+ # 混淆词，设施工程中的施工
|
|
|
+ if m.span()[0] > 0 and name_refind[m.span()[0]-1] in ['设']:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 判断是不是公司名里的工程
|
|
|
+ if re.search(re_str26, name_refind[m.span()[1]:]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+ if re.search(re_str17, name_refind[m.span()[1]:m.span()[1]+3]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+ if re.search(re_str18, name_refind[m.span()[1]:]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+
|
|
|
+ match_flag = True
|
|
|
+ prob_name_list.append(name_refind[last_index:m.span()[1]])
|
|
|
+ last_index = m.span()[1]
|
|
|
+ # print('match_flag', match_flag, name_refind)
|
|
|
+
|
|
|
+ # 找不到则用第二套截取
|
|
|
+ if not prob_name_list:
|
|
|
+ match = re.finditer(re_str9, name_refind)
|
|
|
+ last_index = 0
|
|
|
+ for m in match:
|
|
|
+ # 混淆词，设施工程中的施工
|
|
|
+ if m.span()[0] > 0 and name_refind[m.span()[0]-1] in ['设']:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 判断是不是公司名里的工程
|
|
|
+ if re.search(re_str26, name_refind[m.span()[1]:]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+ if re.search(re_str17, name_refind[m.span()[1]:m.span()[1]+3]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+ if re.search(re_str18, name_refind[m.span()[1]:]):
|
|
|
+ project_word_in_org.append(name_refind[max(0, m.span()[0]-1):min(m.span()[1]+1, len(name_refind))])
|
|
|
+ continue
|
|
|
+ match_flag = True
|
|
|
+ prob_name_list.append(name_refind[last_index:m.span()[1]])
|
|
|
+ last_index = m.span()[1]
|
|
|
+ if not prob_name_list:
|
|
|
+ prob_name_list = [name_refind]
|
|
|
+
|
|
|
+ # print('prob_name_list', prob_name_list)
|
|
|
+ # print('project_word_in_org', project_word_in_org)
|
|
|
+
|
|
|
+ # 一开始不去掉括号里的内容，截取后再去掉
|
|
|
+ for i, name in enumerate(prob_name_list):
|
|
|
+ # 括号内容大于一半字数，则不去掉括号中的字
|
|
|
+ match = re.search(re_str1, name)
|
|
|
+ # print('name', name)
|
|
|
+ # print('match', match)
|
|
|
+ if match and len(match.group()) < len(name) / 2:
|
|
|
+ name = re.sub(re_str1, "", name)
|
|
|
+ name = re.sub(re_str8, "", name)
|
|
|
+ prob_name_list[i] = name
|
|
|
+
|
|
|
+ # 判断refind是否合法
|
|
|
+ # print('prob_name_list2', prob_name_list)
|
|
|
+ name_refind = ''
|
|
|
+ for name in prob_name_list:
|
|
|
+ # 截取公司后的
|
|
|
+ match = re.finditer(re_str3, name)
|
|
|
+ prob_name_list2 = []
|
|
|
+ for m in match:
|
|
|
+ # 排除混淆的情况
|
|
|
+ if m.group() in re_str10 and re.search(re_str10, name):
|
|
|
+ continue
|
|
|
+ prob_name_list2.append(name[m.span()[1]:])
|
|
|
+ if prob_name_list2:
|
|
|
+ name = prob_name_list2[-1]
|
|
|
+
|
|
|
+ # 剔除工程类判断词
|
|
|
+ match1 = re.finditer(re_str6, name)
|
|
|
+ for m1 in match1:
|
|
|
+ # 混淆词，设施工程中的施工
|
|
|
+ if m1.span()[0] > 0 and name[m1.span()[0]-1] in ['设']:
|
|
|
+ continue
|
|
|
+ s_index, e_index = m1.span()
|
|
|
+ word = name[s_index:e_index]
|
|
|
+ s_index = s_index - 1 if s_index > 0 else 0
|
|
|
+ e_index = e_index + 1 if e_index < len(name) else len(name)
|
|
|
+ word1 = name[s_index:e_index]
|
|
|
+ if word1 in project_word_in_org:
|
|
|
+ continue
|
|
|
+ name = re.sub(re.escape(word), '=' * len(word), name)
|
|
|
+ name = re.sub('={2,}', "", name)
|
|
|
+
|
|
|
+ # 剔除一些无关词占用长度
|
|
|
+ if len(re.findall('[\u4e00-\u9fff]', name)) >= min_len \
|
|
|
+ and len(re.findall('[\u4e00-\u9fff]', re.sub(re_str5, '', name))) >= min_len:
|
|
|
+ name_refind = name
|
|
|
+ break
|
|
|
+
|
|
|
+ if match_flag:
|
|
|
+ name_refind_flag_dict['True'] += [name_refind]
|
|
|
+ else:
|
|
|
+ name_refind_flag_dict['False'] += [name_refind]
|
|
|
+
|
|
|
+ # print('name_refind_flag_dict', name_refind_flag_dict)
|
|
|
+
|
|
|
+ true_list = name_refind_flag_dict.get('True')
|
|
|
+ false_list = name_refind_flag_dict.get('False')
|
|
|
+ name_refind_candidate_list = []
|
|
|
+ if true_list:
|
|
|
+ true_list.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ name_refind = true_list[0]
|
|
|
+ name_refind_candidate_list += true_list
|
|
|
+ # else:
|
|
|
+ # name_refind = ''
|
|
|
+ if false_list:
|
|
|
+ false_list.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ name_refind_candidate_list += false_list
|
|
|
+
|
|
|
+ # 对候选name_refind循环
|
|
|
+ name_refind = ''
|
|
|
+ show_name_refind = ''
|
|
|
+ for name_refind in name_refind_candidate_list:
|
|
|
+ # 直接判断删除数字
|
|
|
+ match = re.match(re_str16, name_refind)
|
|
|
+ if match and not re.match('[0-9]', name_refind[match.span()[1]:match.span()[1]+1]):
|
|
|
+ name_refind = name_refind[match.span()[1]:]
|
|
|
+
|
|
|
+ # 删除开头奇怪数字
|
|
|
+ match = re.match(re_str15, name_refind)
|
|
|
+ if match and not re.match('[a-zA-Z地块号]', name_refind[match.span()[1]:match.span()[1]+1]):
|
|
|
+ name_refind = name_refind[match.span()[1]:]
|
|
|
+
|
|
|
+ # 删除期数
|
|
|
+ name_refind = re.sub('[1-9一二三四五六七八九十]期', '', name_refind)
|
|
|
+
|
|
|
+ # 跳过'xx省xx市'
|
|
|
+ if re.search(re_str21, name_refind):
|
|
|
+ sub_word = re.sub(re_str21, '', name_refind)
|
|
|
+ sub_word = re.sub(re_str2 + '|' + re_str9, '', sub_word)
|
|
|
+ if len(sub_word) <= 1:
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+ match27 = re.search(re_str27, name_refind)
|
|
|
+ if match27 and len(match27.group()) == len(name_refind):
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+ match28 = re.search(re_str28, name_refind)
|
|
|
+ if match28 and len(match28.group()) == len(name_refind):
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+ match29 = re.search(re_str29, name_refind)
|
|
|
+ if match29 and len(match29.group()) == len(name_refind):
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 删除类似'招标公告'表达
|
|
|
+ match2 = re.match(re_str23, name_refind)
|
|
|
+ if match2:
|
|
|
+ name_refind = name_refind[match2.span()[1]:]
|
|
|
+ name_refind = re.sub(re_str24, '', name_refind)
|
|
|
+
|
|
|
+ # 跳过文件审批
|
|
|
+ if re.search(re_str19, name_refind):
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 跳过网上超市
|
|
|
+ if re.search(re_str11, name_refind):
|
|
|
+ name_refind = ''
|
|
|
+ continue
|
|
|
+
|
|
|
+ show_name_refind = copy.deepcopy(name_refind)
|
|
|
+
|
|
|
+ # 删除区
|
|
|
+ match2 = re.match(re_str22, name_refind)
|
|
|
+ if match2:
|
|
|
+ name_refind = name_refind[match2.span()[1]-1:]
|
|
|
+
|
|
|
+ # 删除'小区表达'
|
|
|
+ if len(name_refind) >= min_len + 2:
|
|
|
+ name_refind = re.sub(re_str25, '', name_refind)
|
|
|
+
|
|
|
+ # 判断name_refind是否是从公司中来的，过滤
|
|
|
+ if tenderee in [None, 'None', '-', '']:
|
|
|
+ tenderee = ''
|
|
|
+ if agency in [None, 'None', '-', '']:
|
|
|
+ agency = ''
|
|
|
+ try:
|
|
|
+ if len(name_refind) >= 4 and (re.search(re.escape(name_refind[-4:]), tenderee) or re.search(re.escape(name_refind[-4:]), agency)):
|
|
|
+ name_refind = ''
|
|
|
+ show_name_refind = ''
|
|
|
+ except:
|
|
|
+ pass
|
|
|
+
|
|
|
+ # 判断长度
|
|
|
+ if len(name_refind) < min_len:
|
|
|
+ name_refind = ''
|
|
|
+ show_name_refind = ''
|
|
|
+ continue
|
|
|
+
|
|
|
+ break
|
|
|
+
|
|
|
+ return name_refind, show_name_refind
|
|
|
+
|
|
|
+
|
|
|
+def extract_industry(content, _pattern):
|
|
|
+ list_stage = []
|
|
|
+ stage_dict = {}
|
|
|
+ for stage_search in re.finditer(_pattern, content):
|
|
|
+ for k,v in stage_search.groupdict().items():
|
|
|
+ if v is not None:
|
|
|
+ list_stage.append(k)
|
|
|
+ if k in stage_dict.keys():
|
|
|
+ stage_dict[k] += 1
|
|
|
+ else:
|
|
|
+ stage_dict[k] = 1
|
|
|
+ if len(list_stage)>0:
|
|
|
+ stage_cnt_list = [[x, stage_dict.get(x)] for x in stage_dict.keys()]
|
|
|
+ stage_cnt_list.sort(key=lambda x: x[1], reverse=True)
|
|
|
+ # print('extract_industry ' + str(stage_cnt_list))
|
|
|
+ return stage_cnt_list[0][0]
|
|
|
+ # return list_stage[0]
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def extract_project_code_name(list_entity):
|
|
|
+ project_code = None
|
|
|
+ project_name = None
|
|
|
+ for p_entity in list_entity:
|
|
|
+ if p_entity.entity_type == "name":
|
|
|
+ project_name = p_entity.entity_text
|
|
|
+ elif p_entity.entity_type == "code":
|
|
|
+ project_code = p_entity.entity_text
|
|
|
+ if project_name and project_code:
|
|
|
+ break
|
|
|
+ return project_code, project_name
|
|
|
+
|
|
|
+
|
|
|
+def extract_tenderee(list_entity):
|
|
|
+ tenderee = None
|
|
|
+ for p_entity in list_entity:
|
|
|
+ if str(p_entity.label) == "0":
|
|
|
+ tenderee = p_entity.entity_text
|
|
|
+ break
|
|
|
+ return tenderee
|
|
|
+
|
|
|
+
|
|
|
+def extract_project_digest(content):
|
|
|
+ _pattern = "(?P<projectDigest>(项目|工程|标的|需求|建设|招标|采购|内容)(概况|规模|简介|信息|范围|内容|说明|摘要).{10,300})"
|
|
|
+ _pattern_search = re.search(_pattern,content)
|
|
|
+ _projectDigest = ""
|
|
|
+ _find = ""
|
|
|
+ if _pattern_search is not None:
|
|
|
+ _find = _pattern_search.groupdict().get("projectDigest","")
|
|
|
+ if len(_find)>0:
|
|
|
+ _projectDigest = "。".join(_find.split("。")[0:3])
|
|
|
+
|
|
|
+ # 截掉中标信息
|
|
|
+ if len(_projectDigest) >= 10:
|
|
|
+ _projectDigest = cut_win_bid_part(_projectDigest)
|
|
|
+ if len(_projectDigest) < 10:
|
|
|
+ _projectDigest = ""
|
|
|
+ return _projectDigest
|
|
|
+
|
|
|
+
|
|
|
+def extract_project_address(list_sentence, list_entity):
|
|
|
+ reg1 = "(项目|建设|工程)所在区域(位于|为|)[：:]?"
|
|
|
+ reg2 = "(项目|建设|工程)(地址|地点|)(位于|起于)[：:]?"
|
|
|
+ reg3 = "(项目|建设|工程)(地址|地点)[：:]?(位于|起于)"
|
|
|
+ reg4 = "(项目|建设|工程)(地址|地点)[为：:]+"
|
|
|
+
|
|
|
+
|
|
|
+ address_list = []
|
|
|
+ candidate_list = []
|
|
|
+ for sentence in list_sentence:
|
|
|
+ for reg in [reg1, reg2, reg3, reg4]:
|
|
|
+ content = sentence.sentence_text
|
|
|
+ match = re.finditer(reg, content)
|
|
|
+
|
|
|
+ for m in match:
|
|
|
+ end_index = m.span()[1]
|
|
|
+ for p_entity in list_entity:
|
|
|
+ if p_entity.sentence_index != sentence.sentence_index:
|
|
|
+ continue
|
|
|
+ if p_entity.entity_type != "location" and p_entity.entity_type != "org":
|
|
|
+ continue
|
|
|
+
|
|
|
+ text = p_entity.entity_text
|
|
|
+ if text == content[end_index:end_index+len(text)] or text in content[end_index:end_index+len(text)+10]:
|
|
|
+ address_list.append(text)
|
|
|
+ else:
|
|
|
+ candidate_list.append(content[max(0, end_index-10):end_index] + '@@@' + content[end_index:end_index+20] + '@@@' + text)
|
|
|
+
|
|
|
+ if address_list:
|
|
|
+ break
|
|
|
+
|
|
|
+ if not address_list:
|
|
|
+ # for can in candidate_list:
|
|
|
+ # logging.info('candidate ' + can)
|
|
|
+ return None
|
|
|
+ else:
|
|
|
+ address_list.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ # for address in address_list:
|
|
|
+ # logging.info('address ' + address)
|
|
|
+ address = address_list[0]
|
|
|
+
|
|
|
+ return address_list[0]
|
|
|
+
|
|
|
+
|
|
|
+def extract_begin_end_time(list_sentence, list_entity):
|
|
|
+ _begin_time = None
|
|
|
+ _end_time = None
|
|
|
+ for p_entity in list_entity:
|
|
|
+ if p_entity.entity_type == "time":
|
|
|
+ for _sentence in list_sentence:
|
|
|
+ if _sentence.sentence_index == p_entity.sentence_index:
|
|
|
+ _span = spanWindow(tokens=_sentence.tokens,begin_index=p_entity.begin_index,end_index=p_entity.end_index,size=20,center_include=True,word_flag=True,text=p_entity.entity_text)
|
|
|
+ if re.search("开工(时间|日期)",_span[0]) is not None:
|
|
|
+ _time_temp = timeFormat(p_entity.entity_text)
|
|
|
+ if len(_time_temp)>0:
|
|
|
+ _begin_time = _time_temp
|
|
|
+ if re.search("(竣工|完工)(时间|日期)",_span[0]) is not None:
|
|
|
+ _time_temp = timeFormat(p_entity.entity_text)
|
|
|
+ if len(_time_temp)>0:
|
|
|
+ _end_time = _time_temp
|
|
|
+
|
|
|
+ return _begin_time,_end_time
|
|
|
+
|
|
|
+
|
|
|
+def get_bid_location(content):
|
|
|
+ """
|
|
|
+ 获取标段工程地点
|
|
|
+ """
|
|
|
+ import BiddingKG.dl.interface.Preprocessing as Preprocessing
|
|
|
+ content = re.sub("[,，.。、\-\(\)（）— #+~“”'﹙{}－]", "", content)
|
|
|
+
|
|
|
+ # 预处理
|
|
|
+ content_html = '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"><html><body><div>' + content + "</div></body></html>"
|
|
|
+ list_articles, list_sentences, list_entitys, list_outlines, _cost_time \
|
|
|
+ = Preprocessing.get_preprocessed([[0, content_html, "", "", content, "", ""]],
|
|
|
+ useselffool=True)
|
|
|
+
|
|
|
+ # 获取location
|
|
|
+ location = None
|
|
|
+ for entitys in list_entitys:
|
|
|
+ for entity in entitys:
|
|
|
+ if entity.entity_type == "location":
|
|
|
+ if re.search("[街路村县区溪湖河城厂]|[小中大]学|学[校院]", entity.entity_text):
|
|
|
+ location = entity.entity_text
|
|
|
+ break
|
|
|
+
|
|
|
+ # location还未空，寻找类似 '薛家湾（张家圪旦）至柳青'的表达
|
|
|
+ if not location:
|
|
|
+ match = re.search('.{2,4}至.{2,4}', content)
|
|
|
+ if match:
|
|
|
+ location = match.group()
|
|
|
+ # 判断location不在一些特定实体里
|
|
|
+ find_flag = False
|
|
|
+ for entitys in list_entitys:
|
|
|
+ for entity in entitys:
|
|
|
+ if entity.entity_type in ["tenderee", 'agency', 'win_tenderer', 'second_tenderer', 'third_tenderer', 'company', 'org']:
|
|
|
+ if location in entity.entity_text:
|
|
|
+ find_flag = True
|
|
|
+ break
|
|
|
+ if find_flag:
|
|
|
+ break
|
|
|
+ if find_flag:
|
|
|
+ location = None
|
|
|
+ return location
|
|
|
+
|
|
|
+
|
|
|
+def extract_proportion(content, has_preffix=True):
|
|
|
+ if not content:
|
|
|
+ return "", ""
|
|
|
+ # log("content")
|
|
|
+ # log(content)
|
|
|
+ suffix = "[大概约为是:：【\[\s]*[\d,]+(\.\d+)?[十百千万亿]*([\]】平方kK千万公㎡mM米里顷亩]+2?))"
|
|
|
+ reg_dict = {
|
|
|
+ 0: "(?P<proportion>(总((建筑|建设)面积|长|长度))" + suffix,
|
|
|
+ 1: "(?P<proportion>((建筑|建设)面积|全长)" + suffix,
|
|
|
+ 2: "(?P<proportion>((建筑|建设|区域)?面积|全长|项目规模)" + suffix
|
|
|
+ }
|
|
|
+
|
|
|
+ if not has_preffix:
|
|
|
+ reg_dict[3] = "(?P<proportion>" + suffix
|
|
|
+
|
|
|
+ _proportion = ""
|
|
|
+ for i in range(len(list(reg_dict.keys()))):
|
|
|
+ if _proportion:
|
|
|
+ break
|
|
|
+ _pattern = reg_dict.get(i)
|
|
|
+ # logging.info('content ' + str(content))
|
|
|
+ match = re.search(_pattern, str(content))
|
|
|
+ if match:
|
|
|
+ _proportion = match.groupdict().get("proportion","")
|
|
|
+
|
|
|
+ if not _proportion:
|
|
|
+ return "", ""
|
|
|
+
|
|
|
+ # 统一格式
|
|
|
+ multiple_cnt = 1
|
|
|
+ digit = ""
|
|
|
+
|
|
|
+ # 确定具体数字
|
|
|
+ match = re.search('(?P<d1>[\d,]+)(?P<d2>(\.\d+)?)', _proportion)
|
|
|
+ if match:
|
|
|
+ # logging.info(str(_proportion) + ' ' + str(match.group()))
|
|
|
+ d1 = match.group('d1')
|
|
|
+ d2 = match.group('d2')
|
|
|
+ try:
|
|
|
+ d1 = int(re.sub(',', '', d1))
|
|
|
+ except:
|
|
|
+ return "", ""
|
|
|
+ if d2:
|
|
|
+ d2 = Decimal(d2[1:]) / Decimal(str(int(10 ** len(d2[1:]))))
|
|
|
+ # print('d1, d2', d1, d2)
|
|
|
+ d1 += d2
|
|
|
+ digit = d1
|
|
|
+ # print('digit', digit)
|
|
|
+
|
|
|
+ # 确定中文倍数
|
|
|
+ _proportion2 = re.sub(re.escape(match.group()), '', _proportion)
|
|
|
+ match = re.search('[十百千万亿]+', _proportion2)
|
|
|
+ _dict = {'十': 10, '百': 100, '千': 1000, '万': 10000, '亿': 100000000}
|
|
|
+ if match:
|
|
|
+ for c in match.group():
|
|
|
+ multiple_cnt *= _dict.get(c)
|
|
|
+ _proportion3 = re.sub(re.escape(match.group()), '', _proportion2)
|
|
|
+ else:
|
|
|
+ _proportion3 = _proportion2
|
|
|
+ # print('multiple_cnt2', multiple_cnt)
|
|
|
+
|
|
|
+ # 确定面积/长度
|
|
|
+ match = re.search('[平方㎡顷亩]+|[mM]2', _proportion3)
|
|
|
+ if match:
|
|
|
+ unit = '㎡'
|
|
|
+ else:
|
|
|
+ unit = 'm'
|
|
|
+
|
|
|
+ # 确定单位倍数
|
|
|
+ match = re.search('[平方kK千万公㎡mM米里顷亩]+2?', _proportion3)
|
|
|
+ if match:
|
|
|
+ if unit == 'm':
|
|
|
+ if re.search('[kK千公]', match.group()):
|
|
|
+ multiple_cnt *= 1000
|
|
|
+ elif re.search('[里]', match.group()):
|
|
|
+ multiple_cnt *= Decimal(str(500))
|
|
|
+ else:
|
|
|
+ if '亩' in match.group():
|
|
|
+ multiple_cnt *= Decimal(str(666.67))
|
|
|
+ elif '顷' in match.group():
|
|
|
+ multiple_cnt *= 10000
|
|
|
+ elif re.search('千米|公里|k[mM㎡]', match.group()):
|
|
|
+ multiple_cnt *= 1000000
|
|
|
+ # print('multiple_cnt1', multiple_cnt)
|
|
|
+
|
|
|
+ # 拼接
|
|
|
+ digit = str(digit * multiple_cnt) + unit
|
|
|
+
|
|
|
+ return _proportion, digit
|
|
|
+
|
|
|
+
|
|
|
+def extract_has_elevator(content):
|
|
|
+ reg = '电梯'
|
|
|
+ match = re.search(reg, content)
|
|
|
+ has_flag = 0
|
|
|
+ if match:
|
|
|
+ start_index, end_index = match.span()
|
|
|
+ has_flag = 1
|
|
|
+ if judge_yeji(match.span()[0], content):
|
|
|
+ has_flag = 0
|
|
|
+ elif re.search('公司', content[end_index:end_index+8]):
|
|
|
+ has_flag = 0
|
|
|
+ return has_flag
|
|
|
+
|
|
|
+
|
|
|
+def extract_project_property(content, property_pattern, property_priority_dict):
|
|
|
+ property_list = []
|
|
|
+ for m in re.finditer(property_pattern, content):
|
|
|
+ for k,v in m.groupdict().items():
|
|
|
+ if v is not None:
|
|
|
+ property_list.append([k, property_priority_dict.get(k)])
|
|
|
+
|
|
|
+ _property = '新建'
|
|
|
+ if len(property_list)>0:
|
|
|
+ property_list.sort(key=lambda x: x[1])
|
|
|
+ _property = property_list[0][0]
|
|
|
+ return _property
|
|
|
+
|
|
|
+
|
|
|
+def extract_several_money(list_sentence, html='', is_obj=True):
|
|
|
+ money_type_list = ['总投资', '建安费', '工程造价']
|
|
|
+
|
|
|
+ money_list = []
|
|
|
+ all_before_sentence = ''
|
|
|
+ for i, sentence in enumerate(list_sentence):
|
|
|
+ last_text = ''
|
|
|
+ next_text = ''
|
|
|
+ if is_obj:
|
|
|
+ text = sentence.sentence_text
|
|
|
+ all_before_sentence += text
|
|
|
+ if i > 0:
|
|
|
+ last_text = list_sentence[i-1].sentence_text[-30:]
|
|
|
+ if i < len(list_sentence) - 1:
|
|
|
+ next_text = list_sentence[i+1].sentence_text[:30]
|
|
|
+ else:
|
|
|
+ text = sentence
|
|
|
+ all_before_sentence += text
|
|
|
+ if i > 0:
|
|
|
+ last_text = list_sentence[i-1][-30:]
|
|
|
+ if i < len(list_sentence) - 1:
|
|
|
+ next_text = list_sentence[i+1][:30]
|
|
|
+
|
|
|
+ if judge_yeji(len(all_before_sentence), all_before_sentence, 300+len(text)):
|
|
|
+ # print('sentence yeji before ' + text)
|
|
|
+ continue
|
|
|
+ # if '项目概算总投资为' in text:
|
|
|
+ _list, _ = get_several_money(text, 0, False, html=html)
|
|
|
+ # logging.info('get_several_money _list ' + str(_list))
|
|
|
+
|
|
|
+ temp_list = []
|
|
|
+ for l in _list:
|
|
|
+ if l[-1] == '总投资':
|
|
|
+ if re.search('业绩', last_text+text+next_text):
|
|
|
+ continue
|
|
|
+ temp_list.append(l)
|
|
|
+ _list = temp_list
|
|
|
+
|
|
|
+ money_list += _list
|
|
|
+ # if money_list:
|
|
|
+ # break
|
|
|
+
|
|
|
+ money_type_dict = {}
|
|
|
+ for money, _, _, _, money_type in money_list:
|
|
|
+ for _type in money_type_list:
|
|
|
+ if _type != money_type:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # 科学计数法
|
|
|
+ try:
|
|
|
+ if 'E+' in money:
|
|
|
+ times = int(money.split('E+')[-1])
|
|
|
+ _money = float(float(money.split('E+')[0]) * (10 ** times))
|
|
|
+ else:
|
|
|
+ _money = float(money)
|
|
|
+ except:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if _type in money_type_dict.keys():
|
|
|
+ money_type_dict[_type] += [_money]
|
|
|
+ else:
|
|
|
+ money_type_dict[_type] = [_money]
|
|
|
+
|
|
|
+ # logging.info('money_type_dict ' + str(money_type_dict))
|
|
|
+
|
|
|
+ result_list = []
|
|
|
+ for _type in money_type_list:
|
|
|
+ if money_type_dict.get(_type):
|
|
|
+ if _type == '建安费':
|
|
|
+ temp_list = money_type_dict.get(_type)
|
|
|
+ temp_list = list(set(temp_list))
|
|
|
+ money = 0
|
|
|
+ for m in temp_list:
|
|
|
+ money += m
|
|
|
+ result_list.append(money)
|
|
|
+ else:
|
|
|
+ result_list.append(money_type_dict.get(_type)[0])
|
|
|
+ else:
|
|
|
+ result_list.append(None)
|
|
|
+
|
|
|
+ for i in range(len(result_list)):
|
|
|
+ if result_list[i] is None:
|
|
|
+ result_list[i] = 0
|
|
|
+ result_list[i] = float(result_list[i])
|
|
|
+
|
|
|
+ return result_list
|
|
|
+
|
|
|
+
|
|
|
+def extract_max_floor(content, html=None):
|
|
|
+ from bs4 import BeautifulSoup
|
|
|
+
|
|
|
+ def match_floor(_reg, _content, _reg2=None, _tables_and_divs=None):
|
|
|
+ _match = re.finditer(_reg, _content)
|
|
|
+ _floor_list = []
|
|
|
+ for m in _match:
|
|
|
+ if 'reg6' in _reg:
|
|
|
+ _floor1 = content[max(0, m.span('reg6')[0]-1):m.span('reg6')[1]+1]
|
|
|
+ elif 'reg4' in _reg:
|
|
|
+ _floor1 = content[max(0, m.span('reg4')[0]-1):m.span('reg4')[1]+1]
|
|
|
+ else:
|
|
|
+ _floor1 = content[m.span()[0]:m.span()[1]]
|
|
|
+ if judge_yeji(m.span()[0], _content, 300, _tables_and_divs, _floor1):
|
|
|
+ continue
|
|
|
+
|
|
|
+ if 'reg6' in _reg:
|
|
|
+ _floor = m.group('reg6')
|
|
|
+ _floor = re.split('[-~～]', _floor)
|
|
|
+ _floor = max(int(_floor[0]), int(_floor[1]))
|
|
|
+ elif 'reg4' in _reg:
|
|
|
+ _floors = re.findall('\d+', m.group())
|
|
|
+ _floors = [int(x) for x in _floors]
|
|
|
+ _floors1 = re.findall('[一二两三四五六七八九十]+', m.group())
|
|
|
+ _floors1 = [chinese_to_arabic(x) for x in _floors1]
|
|
|
+ _floor = max(_floors + _floors1)
|
|
|
+ elif '-' in m.group():
|
|
|
+ _floor = ''.join(re.findall('\d+-\d+', m.group()))
|
|
|
+ if len(_floor) < 1:
|
|
|
+ continue
|
|
|
+ _floor = _floor.split('-')
|
|
|
+ _floor = max(int(_floor[0]), int(_floor[1]))
|
|
|
+ elif '/' in m.group():
|
|
|
+ _floor = m.group()
|
|
|
+ _floor = re.sub('层', '', _floor)
|
|
|
+ _floor = ''.join(re.findall('\d+/\d+', _floor))
|
|
|
+ # print('@1', _floor)
|
|
|
+ if len(_floor) < 1:
|
|
|
+ continue
|
|
|
+ _floor = _floor.split('/')
|
|
|
+ _floor = max(int(_floor[0]), int(_floor[1]))
|
|
|
+ else:
|
|
|
+ _floor = ''.join(re.findall('\d+', m.group()))
|
|
|
+ if len(_floor) < 1:
|
|
|
+ _floor = ''.join(re.findall('[一二两三四五六七八九十]+', m.group()))
|
|
|
+ if len(_floor) < 1:
|
|
|
+ continue
|
|
|
+ _floor = chinese_to_arabic(_floor)
|
|
|
+ _floor = int(_floor)
|
|
|
+ if _reg2:
|
|
|
+ _floor_list2 = match_floor(_reg2, _content[m.span()[1]:m.span()[1]+35])
|
|
|
+ # print('@2', _floor_list2)
|
|
|
+ if _floor_list2:
|
|
|
+ _floor2 = int(_floor_list2[0])
|
|
|
+ _floor = _floor + _floor2
|
|
|
+ _floor_list.append(_floor)
|
|
|
+ return _floor_list
|
|
|
+
|
|
|
+ reg = '(建筑|)(物|)(层数最大|最大层数|最高层|总层数|最大层|层数)[共为:：]?(\d{1,3}|\d{1,3}层?/\d{1,3}|[一两二三四五六七八九十]{1,3})[层Ff]'
|
|
|
+ reg0 = '局部(建筑|)(层数|)[共为]?(\d{1,3}|[一二两三四五六七八九十]{1,3})层'
|
|
|
+ reg1 = '地上(建筑|)(层数|)[共为]?(\d{1,3}层?/\d{1,3}|\d{1,3}-\d{1,3}|\d{1,3}|[一二两三四五六七八九十]{1,3})层'
|
|
|
+ reg2 = '地下(建筑|)(层数|)[共为]?(\d{1,3}层?/\d{1,3}|\d{1,3}-\d{1,3}|\d{1,3}|[一二两三四五六七八九十]{1,3})层'
|
|
|
+ reg3 = '[到至]\d{1,3}层'
|
|
|
+ reg4 = '层数分别.{1,20}(?P<reg4>(\d{1,3}|[一二两三四五六七八九十]{1,3}))层'
|
|
|
+ reg5 = '共(\d{1,3}|[一二两三四五六七八九十]{1,3})层'
|
|
|
+ reg6 = '地上.{1,10}(?P<reg6>\d{1,3}[-~～]\d{1,3})层'
|
|
|
+
|
|
|
+ if html:
|
|
|
+ soup = BeautifulSoup(html, 'lxml')
|
|
|
+ tables_and_divs = soup.find_all(['table', 'div', 'p'])
|
|
|
+ else:
|
|
|
+ tables_and_divs = []
|
|
|
+
|
|
|
+ floor_list = []
|
|
|
+ # 常规
|
|
|
+ floor_list += match_floor(reg, content, _tables_and_divs=tables_and_divs)
|
|
|
+ # 局部
|
|
|
+ floor_list += match_floor(reg0, content, _tables_and_divs=tables_and_divs)
|
|
|
+ # x层到x层
|
|
|
+ floor_list += match_floor(reg3, content, _tables_and_divs=tables_and_divs)
|
|
|
+ # 地上地下
|
|
|
+ floor_list += match_floor(reg1, content, _reg2=reg2, _tables_and_divs=tables_and_divs)
|
|
|
+ floor_list += match_floor(reg2, content, _reg2=reg1, _tables_and_divs=tables_and_divs)
|
|
|
+ # 层数分别为... xx层
|
|
|
+ floor_list += match_floor(reg4, content, _tables_and_divs=tables_and_divs)
|
|
|
+ # 共xx层
|
|
|
+ floor_list += match_floor(reg5, content, _tables_and_divs=tables_and_divs)
|
|
|
+ # 地上... xx~xx层
|
|
|
+ floor_list += match_floor(reg6, content, _reg2=reg2, _tables_and_divs=tables_and_divs)
|
|
|
+
|
|
|
+ if floor_list:
|
|
|
+ floor_list.sort(key=lambda x: x)
|
|
|
+ floor = floor_list[-1]
|
|
|
+ if floor <= 0:
|
|
|
+ return None
|
|
|
+ else:
|
|
|
+ return floor_list[-1]
|
|
|
+ else:
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def extract_structure(content, html=None, structure_keyword_list=None):
|
|
|
+ from bs4 import BeautifulSoup
|
|
|
+
|
|
|
+ # reg = '框架结构|钢框架结构|混凝土框架结构|剪力墙结构|框架-剪力墙结构|框架+剪力墙结构|框架和剪力墙结构|框架及剪力墙结构|混凝土剪力墙结构|筒体结构|桅式结构|墙板结构|膜结构|悬索结构|板柱结构|充气结构|网架结构|壳体结构|拱形结构|穹顶结构|混凝土结构|钢筋混凝土框架结构|钢筋混凝土筒仓结构|钢结构|砌体结构|木结构|砖混结构|排架结构|束筒结构|薄壳结构|钢混结构|砖木结构|砌体结构|钢砼结构|框剪结构|钢筋混凝土框架结构|筒中筒结构|框筒结构|桁架结构|拱券结构|钢筋混凝土结构|框架核心筒结构|门式钢架结构|门钢结构|轻钢结构|钢-混凝土框架结构|木框架结构|空间网格结构|框架筒体结构|砖拱结构|钢筋砼结构|核心筒结构|框架-核心筒结构'
|
|
|
+ reg1 = '(结构(楼层|)(形式|类型|类别|体系|结构)[为是：:])([^结]{2,8}结构)'
|
|
|
+
|
|
|
+ reg = '|'.join([x[:-1] for x in structure_keyword_list])
|
|
|
+ reg = reg_word_sort(reg)
|
|
|
+ # logging.info(reg)
|
|
|
+
|
|
|
+ if html:
|
|
|
+ soup = BeautifulSoup(html, 'lxml')
|
|
|
+ tables_and_divs = soup.find_all(['table', 'div', 'p'])
|
|
|
+ else:
|
|
|
+ tables_and_divs = []
|
|
|
+
|
|
|
+ # match = re.finditer(reg1, content)
|
|
|
+ # structure_list = []
|
|
|
+ # for m in match:
|
|
|
+ # structure = m.group(4)
|
|
|
+ # structure1 = content[max(0, m.span(4)[0]-1):m.span(4)[1]+1]
|
|
|
+ # if judge_yeji(m.span()[0], content, 300, tables_and_divs, structure1):
|
|
|
+ # continue
|
|
|
+ # structure_list.append(structure)
|
|
|
+
|
|
|
+ # if structure_list:
|
|
|
+ # structure_list = list(set(structure_list))
|
|
|
+ # structure_list.sort(key=lambda x: x)
|
|
|
+ # return ','.join(structure_list)
|
|
|
+ # else:
|
|
|
+
|
|
|
+ structure_list = []
|
|
|
+ match = re.finditer(reg, content)
|
|
|
+ for m in match:
|
|
|
+ structure = m.group()
|
|
|
+ structure1 = content[max(0, m.span()[0]-1):m.span()[1]+1]
|
|
|
+ if judge_yeji(m.span()[0], content, 300, tables_and_divs, structure1):
|
|
|
+ continue
|
|
|
+ if structure in ['钢结构']:
|
|
|
+ if re.search('公司', content[m.span()[1]:m.span()[1]+8]):
|
|
|
+ continue
|
|
|
+ structure_list.append(structure)
|
|
|
+
|
|
|
+ if structure_list:
|
|
|
+ structure_list = list(set(structure_list))
|
|
|
+ structure_list.sort(key=lambda x: x)
|
|
|
+ return ','.join(structure_list)
|
|
|
+ else:
|
|
|
+ return None
|
|
|
+
|
|
|
+
|
|
|
+def extract_has_steel_structure(content):
|
|
|
+ """
|
|
|
+
|
|
|
+ :param content: 传入已提取完的structure
|
|
|
+ :return:
|
|
|
+ """
|
|
|
+ if not content:
|
|
|
+ return 0
|
|
|
+
|
|
|
+ reg = '钢结构|门式钢架结构|钢框架结构|钢桁架结构|钢网架结构|钢框结构钢架结构|钢骨架结构|钢骨架式结构|钢管桁架结构|轻钢骨架结构|钢桁架式结构|轻钢门架结构|轻钢门式结构|门式钢屋架结构|钢桁架框架结构|钢框架—支撑结构|钢桁梁结构|网架结构|网壳结构|索膜结构|塔桅结构'
|
|
|
+ match = re.search(reg, content)
|
|
|
+ has_flag = 0
|
|
|
+ if match:
|
|
|
+ has_flag = 1
|
|
|
+ return has_flag
|
|
|
+
|
|
|
+
|
|
|
+def extract_wall_type(doctitle, content):
|
|
|
+ reg1 = '(幕墙|外墙)'
|
|
|
+
|
|
|
+ wall_class_dict = {
|
|
|
+ '玻璃幕墙': '玻璃|玻璃砖',
|
|
|
+ '金属幕墙': '铝合金|铝单板|仿木纹铝单板|夹芯保温铝板|铝复合板|蜂窝铝板|仿石材铝板|铝板|彩钢板|不锈钢板|彩涂钢板|珐琅钢板|钛合金板|铝合金装饰网格|镀锌钢网|铜合金|镀层钢板|锌板|钛板',
|
|
|
+ '陶土板幕墙': '陶土板',
|
|
|
+ '涂料': '涂料|乳胶漆|弹性涂料|质感涂料|真石漆|水包水|水包砂|岩片漆|金属漆|氟碳漆|仿石涂料|封闭底漆|腻子|罩光清漆|柔性耐水腻子|罩面漆|仿瓷涂料|有机硅丙烯酸涂料|氟碳树脂涂料|硅溶胶外墙涂料|无机纤维喷涂',
|
|
|
+ '石材': '石材|花岗岩|大理石|砂岩|板岩|岩棉板|岩板',
|
|
|
+ '瓷砖': '陶瓷砖|劈开砖|釉面砖|通体砖|抛光砖|马赛克砖|陶砖|玻璃砖',
|
|
|
+ '混凝土': '钢筋混凝土|现浇混凝土|预制混凝土|混凝土砌块|蒸压多孔混凝土|纤维混凝土|UHPC板|UHPC|UHPC装饰板|超高性能混凝土|清水混凝土|混凝土|加气混凝土砌块',
|
|
|
+ '复合墙板': '复合外墙板|复合墙板|GRC|金属面岩棉板|复合保温幕墙板|金属面复合保温幕墙|金属面岩棉夹心外墙板|夹心保温外墙|金属面岩棉装饰节能一体板|PU板幕墙|保温装饰一体板|保温一体板|岩棉保温装饰一体板|EPS保温板|岩棉保温板|STP保温板|酚醛保温板|聚氨酯保温板|钛锌塑铝复合板|自保温板外墙板|岩棉夹心板外墙板|玻璃棉|EPS板|XPS板|聚氨酯外墙保温板|外墙保温板|保温岩棉复合板|水泥发泡保温板|发泡陶瓷保温板|石墨改性水泥基保温板|挤塑聚苯板|挤塑板|聚苯乙烯板|玻化微珠岩棉板|聚苯板|聚氨酯板|热固复合聚苯乙烯泡沫保温板',
|
|
|
+ '水泥': '白水泥|白色硅酸盐水泥',
|
|
|
+ '文化砖': '文化砖',
|
|
|
+ '木材': 'LVL|单板层积材|高压热固化木纤维板',
|
|
|
+ '陶棍幕墙': '陶棍幕墙',
|
|
|
+ '风动幕墙': '风铃幕墙|风动幕墙',
|
|
|
+ '光电幕墙': '光伏幕墙|光电幕墙',
|
|
|
+ '织物幕墙': '织物幕墙',
|
|
|
+ '生态幕墙': '生态幕墙|气循环幕墙|呼吸幕墙|绿色幕墙|健康幕墙',
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ for key in wall_class_dict.keys():
|
|
|
+ reg = wall_class_dict[key]
|
|
|
+ reg = reg_word_sort(reg)
|
|
|
+ wall_class_dict[key] = '(' + reg + ')'
|
|
|
+
|
|
|
+ wall_list = []
|
|
|
+ wall_list2 = []
|
|
|
+ match = re.search(reg1, doctitle)
|
|
|
+ if match:
|
|
|
+ for first_class in wall_class_dict.keys():
|
|
|
+ reg2 = wall_class_dict.get(first_class)
|
|
|
+ if first_class in ['陶棍幕墙', '风动幕墙', '光电幕墙', '织物幕墙', '生态幕墙']:
|
|
|
+ continue
|
|
|
+
|
|
|
+ reg2 = reg2 + '(幕墙|外墙|)'
|
|
|
+ match1 = re.search(reg2, content)
|
|
|
+ if match1:
|
|
|
+ wall_list.append(first_class)
|
|
|
+ wall_list2.append(match1.group())
|
|
|
+
|
|
|
+ for first_class in wall_class_dict.keys():
|
|
|
+ reg2 = wall_class_dict.get(first_class)
|
|
|
+
|
|
|
+ # 这5类特殊，只要存在就直接提取
|
|
|
+ if first_class in ['陶棍幕墙', '风动幕墙', '光电幕墙', '织物幕墙', '生态幕墙']:
|
|
|
+ match = re.search(reg2, content)
|
|
|
+ if match:
|
|
|
+ wall_list.append(first_class)
|
|
|
+ wall_list2.append(match.group())
|
|
|
+ # 其他类使用两种规则
|
|
|
+ else:
|
|
|
+ reg3 = reg1 + '.{0,10}' + reg2
|
|
|
+ reg4 = reg2 + reg1
|
|
|
+ match = re.search(reg3, content)
|
|
|
+ if match:
|
|
|
+ wall_list.append(first_class)
|
|
|
+ match1 = re.search(reg2, match.group())
|
|
|
+ if match1:
|
|
|
+ wall_list2.append(match1.group())
|
|
|
+ match = re.search(reg4, content)
|
|
|
+ if match:
|
|
|
+ wall_list.append(first_class)
|
|
|
+ wall_list2.append(match.group())
|
|
|
+
|
|
|
+ if wall_list:
|
|
|
+ wall_list = list(set(wall_list))
|
|
|
+ wall_list.sort(key=lambda x: x)
|
|
|
+ wall_list = ','.join(wall_list)
|
|
|
+ else:
|
|
|
+ wall_list = None
|
|
|
+
|
|
|
+ if wall_list2:
|
|
|
+ wall_list2 = list(set(wall_list2))
|
|
|
+ wall_list2.sort(key=lambda x: x)
|
|
|
+ wall_list2 = ','.join(wall_list2)
|
|
|
+ else:
|
|
|
+ wall_list2 = None
|
|
|
+
|
|
|
+ return wall_list, wall_list2
|
|
|
+
|
|
|
+
|
|
|
+def cut_win_bid_part(_str):
|
|
|
+ """
|
|
|
+ 截掉项目概述里的中标相关信息
|
|
|
+ """
|
|
|
+ origin_str = _str
|
|
|
+ reg_list = [
|
|
|
+ "(评标|中标|中选)(结果|报告)(公示|)[:：，,]",
|
|
|
+ "第[一二三四1-9]名",
|
|
|
+ "中标候选人(基本情况|附件)",
|
|
|
+ "(排序|第[一二三1-3]|推荐)中标候选人|中标(候选人|结果)(排序|公示|信息)"
|
|
|
+ ]
|
|
|
+ for reg in reg_list:
|
|
|
+ match = re.finditer(reg, _str)
|
|
|
+ _index = len(origin_str)
|
|
|
+ for m in match:
|
|
|
+ _index = m.span()[0]
|
|
|
+ break
|
|
|
+ if _index == len(origin_str):
|
|
|
+ continue
|
|
|
+ match = re.finditer("[,，。；;]", _str[:_index])
|
|
|
+ _index = 0
|
|
|
+ for m in match:
|
|
|
+ _index = m.span()[0]
|
|
|
+ _str = _str[:_index]
|
|
|
+
|
|
|
+ match = re.finditer("中[标选]|投标(人|企业|单位)|乙方|供应商|成交", _str)
|
|
|
+ for m in match:
|
|
|
+ index_start = m.span()[0]
|
|
|
+ cut_str = re.split("[,，。；;]", _str[index_start:])[0]
|
|
|
+ if len(cut_str) < 25:
|
|
|
+ cut_str = _str[index_start:index_start+25]
|
|
|
+ # cut_str = _str[index_start:index_start+15]
|
|
|
+ # print("cut_str", cut_str)
|
|
|
+
|
|
|
+ match2 = re.search("(￥|)[\d,.]+[万元亿]|公司|业绩|最终得分", cut_str)
|
|
|
+ # print("match2", match2)
|
|
|
+ if match2:
|
|
|
+ # 排除混淆
|
|
|
+ match3 = re.search("中标法|获取招标文件|交接|支付|账户|结算|限价|承担|服务费|\*|×", cut_str)
|
|
|
+ # print("match3", match3)
|
|
|
+ if not match3:
|
|
|
+ match4 = re.finditer("[,，。；;]", _str[:index_start])
|
|
|
+ index_stop = 0
|
|
|
+ for m4 in match4:
|
|
|
+ index_stop = m4.span()[0]
|
|
|
+ _str = _str[:index_stop]
|
|
|
+ break
|
|
|
+ # print(_str)
|
|
|
+ return _str
|
|
|
+
|
|
|
+
|
|
|
+def judge_yeji(end_index, content, judge_len=300, tables_and_divs=None, entity=None):
|
|
|
+ def is_yeji_table(_tables_and_divs, _entity_text):
|
|
|
+ if not _tables_and_divs:
|
|
|
+ return 0
|
|
|
+
|
|
|
+ is_yeji = 0
|
|
|
+ reg_yeji = '业绩|选取原因|奖项|获奖|供应商信息|近年完成|中标单位信息|评标情况|类似项目|资质|候选人情况'
|
|
|
+ for index3, obj in enumerate(_tables_and_divs):
|
|
|
+ if '<tr' not in str(obj):
|
|
|
+ continue
|
|
|
+
|
|
|
+ _table = obj
|
|
|
+ _m = re.search(re.escape(_entity_text), str(_table))
|
|
|
+ if not _m:
|
|
|
+ _entity_text = _entity_text[1:]
|
|
|
+ _m = re.search(re.escape(_entity_text[1:]), str(_table))
|
|
|
+ if not _m:
|
|
|
+ _entity_text = _entity_text[:-1]
|
|
|
+ _m = re.search(re.escape(_entity_text[:-1]), str(_table))
|
|
|
+ # print('_entity_text', _entity_text)
|
|
|
+ if _m:
|
|
|
+ rows = _table.find_all('tr')
|
|
|
+
|
|
|
+ max_col_span = 0
|
|
|
+ for row in rows:
|
|
|
+ if len(row.find_all('td')) > max_col_span:
|
|
|
+ max_col_span = len(row.find_all('td'))
|
|
|
+ # print('max_col_span', max_col_span)
|
|
|
+
|
|
|
+ for index, row in enumerate(rows):
|
|
|
+ if re.search(re.escape(_entity_text), str(row)):
|
|
|
+ cells = row.find_all('td')
|
|
|
+ if len(cells) == 0:
|
|
|
+ continue
|
|
|
+
|
|
|
+ # print('cells', cells[0])
|
|
|
+ # 第一列是否含业绩
|
|
|
+ if re.search(reg_yeji, str(cells[0])):
|
|
|
+ # logging.info('is_yeji_table 1')
|
|
|
+ is_yeji = 1
|
|
|
+ # 前面几行是否有占多列的有业绩
|
|
|
+ else:
|
|
|
+ if index > 0:
|
|
|
+ for row2 in rows[:index][::-1]:
|
|
|
+ # print('len(rows[index2])', len(row2.find_all('td')))
|
|
|
+ # if len(row2.find_all('td')) <= max_col_span / 2:
|
|
|
+ # print(re.search('业绩', str(row2)), str(row2))
|
|
|
+ if len(row2.find_all('td')) <= max_col_span / 2 and re.search(reg_yeji, str(row2)):
|
|
|
+ # logging.info('is_yeji_table 2')
|
|
|
+ is_yeji = 1
|
|
|
+
|
|
|
+ break
|
|
|
+
|
|
|
+ # 前面都找不到，那么找表格上方的两行
|
|
|
+ div_list = [str(x) for x in _tables_and_divs[max(0, index3-2):index3]]
|
|
|
+ if not is_yeji and re.search(reg_yeji, ' '.join(div_list)):
|
|
|
+ # logging.info('is_yeji_table 3')
|
|
|
+ is_yeji = 1
|
|
|
+ break
|
|
|
+ return is_yeji
|
|
|
+
|
|
|
+ # 先判断表格业绩
|
|
|
+ if tables_and_divs:
|
|
|
+ yeji_table_flag = is_yeji_table(tables_and_divs, entity)
|
|
|
+ if yeji_table_flag:
|
|
|
+ # logging.info('yeji_table_flag 1')
|
|
|
+ return yeji_table_flag
|
|
|
+
|
|
|
+ if len(content) == 0:
|
|
|
+ return 0
|
|
|
+ if end_index == 0:
|
|
|
+ return 0
|
|
|
+
|
|
|
+ reg_yeji = '业绩|选取原因|奖项|获奖|供应商信息|近年完成'
|
|
|
+ if re.search(reg_yeji, content[:end_index][-judge_len:]):
|
|
|
+ # if '业绩' in content[:end_index][-judge_len:]:
|
|
|
+ return 1
|
|
|
+ else:
|
|
|
+ return 0
|
|
|
+
|
|
|
+
|
|
|
+def get_several_money(sentence_text, found_yeji, in_attachment=False, html=''):
|
|
|
+ from bs4 import BeautifulSoup
|
|
|
+
|
|
|
+ def getDigitsDic(_unit):
|
|
|
+ '''
|
|
|
+ @summary:拿到中文对应的数字
|
|
|
+ '''
|
|
|
+ DigitsDic = {"零":0, "壹":1, "贰":2, "叁":3, "肆":4, "伍":5, "陆":6, "柒":7, "捌":8, "玖":9,
|
|
|
+ "〇":0, "一":1, "二":2, "三":3, "四":4, "五":5, "六":6, "七":7, "八":8, "九":9}
|
|
|
+ return DigitsDic.get(_unit)
|
|
|
+
|
|
|
+ def getMultipleFactor(_unit):
|
|
|
+ '''
|
|
|
+ @summary:拿到单位对应的值
|
|
|
+ '''
|
|
|
+ MultipleFactor = {"兆":Decimal(1000000000000),"亿":Decimal(100000000),"万":Decimal(10000),"仟":Decimal(1000),"千":Decimal(1000),"佰":Decimal(100),"百":Decimal(100),"拾":Decimal(10),"十":Decimal(10),"元":Decimal(1),"圆":Decimal(1),"角":round(Decimal(0.1),1),"分":round(Decimal(0.01),2)}
|
|
|
+ return MultipleFactor.get(_unit)
|
|
|
+
|
|
|
+ def getUnifyMoney(money):
|
|
|
+ '''
|
|
|
+ @summary:将中文金额字符串转换为数字金额
|
|
|
+ @param:
|
|
|
+ money:中文金额字符串
|
|
|
+ @return: decimal,数据金额
|
|
|
+ '''
|
|
|
+
|
|
|
+ MAX_MONEY = 1000000000000
|
|
|
+ MAX_NUM = 12
|
|
|
+ #去掉逗号
|
|
|
+ money = re.sub("[，,]","",money)
|
|
|
+ money = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",money)
|
|
|
+ result = Decimal(0)
|
|
|
+ chnDigits = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"]
|
|
|
+ # chnFactorUnits = ["兆", "亿", "万", "仟", "佰", "拾","圆","元","角","分"]
|
|
|
+ chnFactorUnits = ["圆", "元","兆", "亿", "万", "仟", "佰", "拾", "角", "分", '十', '百', '千']
|
|
|
+
|
|
|
+ LowMoneypattern = re.compile("^[\d,]+(\.\d+)?$")
|
|
|
+ BigMoneypattern = re.compile("^零?(?P<BigMoney>[%s])$"%("".join(chnDigits)))
|
|
|
+ try:
|
|
|
+ if re.search(LowMoneypattern,money) is not None:
|
|
|
+ return Decimal(money)
|
|
|
+ elif re.search(BigMoneypattern,money) is not None:
|
|
|
+ return getDigitsDic(re.search(BigMoneypattern,money).group("BigMoney"))
|
|
|
+ for factorUnit in chnFactorUnits:
|
|
|
+ if re.search(re.compile(".*%s.*"%(factorUnit)),money) is not None:
|
|
|
+ subMoneys = re.split(re.compile("%s(?!.*%s.*)"%(factorUnit,factorUnit)),money)
|
|
|
+ if re.search(re.compile("^(\d+)(\.\d+)?$"),subMoneys[0]) is not None:
|
|
|
+ if MAX_MONEY/getMultipleFactor(factorUnit)<Decimal(subMoneys[0]):
|
|
|
+ return Decimal(0)
|
|
|
+ result += Decimal(subMoneys[0])*(getMultipleFactor(factorUnit))
|
|
|
+ elif len(subMoneys[0])==1:
|
|
|
+ if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[0]) is not None:
|
|
|
+ result += Decimal(getDigitsDic(subMoneys[0]))*(getMultipleFactor(factorUnit))
|
|
|
+ # subMoneys[0]中无金额单位，不可再拆分
|
|
|
+ elif subMoneys[0]=="":
|
|
|
+ result += 0
|
|
|
+ elif re.search(re.compile("[%s]"%("".join(chnFactorUnits))),subMoneys[0]) is None:
|
|
|
+ # print(subMoneys)
|
|
|
+ # subMoneys[0] = subMoneys[0][0]
|
|
|
+ result += Decimal(getUnifyMoney(subMoneys[0])) * (getMultipleFactor(factorUnit))
|
|
|
+ else:
|
|
|
+ result += Decimal(getUnifyMoney(subMoneys[0]))*(getMultipleFactor(factorUnit))
|
|
|
+ if len(subMoneys)>1:
|
|
|
+ if re.search(re.compile("^(\d+(,)?)+(\.\d+)?[百千万亿]?\s?(元)?$"),subMoneys[1]) is not None:
|
|
|
+ result += Decimal(subMoneys[1])
|
|
|
+ elif len(subMoneys[1])==1:
|
|
|
+ if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[1]) is not None:
|
|
|
+ result += Decimal(getDigitsDic(subMoneys[1]))
|
|
|
+ else:
|
|
|
+ result += Decimal(getUnifyMoney(subMoneys[1]))
|
|
|
+ break
|
|
|
+ except Exception as e:
|
|
|
+ # traceback.print_exc()
|
|
|
+ return Decimal(0)
|
|
|
+ return result
|
|
|
+
|
|
|
+ def is_yeji_table(_tables_and_divs, _entity_text):
|
|
|
+ if not _tables_and_divs:
|
|
|
+ return 0
|
|
|
+
|
|
|
+ is_yeji = 0
|
|
|
+ reg_yeji = '业绩|选取原因|奖项|获奖'
|
|
|
+ for index3, obj in enumerate(_tables_and_divs):
|
|
|
+ if '<tr' not in str(obj):
|
|
|
+ continue
|
|
|
+
|
|
|
+ _table = obj
|
|
|
+ _m = re.search(re.escape(_entity_text), str(_table))
|
|
|
+ if _m:
|
|
|
+ rows = _table.find_all('tr')
|
|
|
+
|
|
|
+ max_col_span = 0
|
|
|
+ for row in rows:
|
|
|
+ if len(row.find_all('td')) > max_col_span:
|
|
|
+ max_col_span = len(row.find_all('td'))
|
|
|
+ # print('max_col_span', max_col_span)
|
|
|
+
|
|
|
+ for index, row in enumerate(rows):
|
|
|
+ if re.search(re.escape(_entity_text), str(row)):
|
|
|
+ cells = row.find_all('td')
|
|
|
+ # 第一列是否含业绩
|
|
|
+ if re.search(re.escape(_entity_text), str(cells[0])):
|
|
|
+ is_yeji = 1
|
|
|
+ # 前面几行是否有占多列的有业绩
|
|
|
+ else:
|
|
|
+ if index > 0:
|
|
|
+ for row2 in rows[:index][::-1]:
|
|
|
+ # print('len(rows[index2])', len(row2.find_all('td')))
|
|
|
+ # if len(row2.find_all('td')) <= max_col_span / 2:
|
|
|
+ # print(re.search('业绩', str(row2)), str(row2))
|
|
|
+ if len(row2.find_all('td')) <= max_col_span / 2 and re.search(reg_yeji, str(row2)):
|
|
|
+ is_yeji = 1
|
|
|
+
|
|
|
+ break
|
|
|
+
|
|
|
+ # 前面都找不到，那么找表格上方的两行
|
|
|
+ if not is_yeji and re.search(reg_yeji, ' '.join(_tables_and_divs[index3-2:index3])):
|
|
|
+ is_yeji = 1
|
|
|
+ break
|
|
|
+ # print('is_yeji', is_yeji)
|
|
|
+ return is_yeji
|
|
|
+
|
|
|
+
|
|
|
+ # 提取表格用于判断业绩
|
|
|
+ if html:
|
|
|
+ soup = BeautifulSoup(html, 'lxml')
|
|
|
+ tables = soup.find_all('table')
|
|
|
+ tables_and_divs = soup.find_all(['table', 'div'])
|
|
|
+ else:
|
|
|
+ tables_and_divs = []
|
|
|
+
|
|
|
+ money_list = []
|
|
|
+ # 使用正则识别金额
|
|
|
+ entity_type = "money"
|
|
|
+ list_money_pattern = {"cn": "(()(?P<filter_kw>百分之)?(?P<money_cn>[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,})())",
|
|
|
+ "key_word": "((?P<text_key_word>(?:[￥¥]+，?|[单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费|标的基本情况|CNY|成交结果)(?:[,，\[（\(]*\s*(人民币|单位：)?/?(?P<unit_key_word_before>[万亿]?(?:[美日欧]元|元(/(M2|[\u4e00-\u9fa5]{1,3}))?)?(?P<filter_unit2>[台个只吨]*))\s*(/?费率)?(人民币)?[\]）\)]?)\s*[，,:：]*(RMB|USD|EUR|JPY|CNY)?[:：]?(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,8}?))(第[123一二三]名[：:])?(\d+(\*\d+%)+=)?(?P<money_key_word>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_key_word>(E-?\d+))?[百千]{,1})(?:[（\(]?(?P<filter_>[%％‰折])*\s*，?((金额)?单位[:：])?(?P<unit_key_word_behind>[万亿]?(?:[美日欧]元|元)?(?P<filter_unit1>[台只吨斤棵株页亩方条天]*))\s*[）\)]?))",
|
|
|
+ "front_m": "((?P<text_front_m>(?:[（\(]?\s*(?P<unit_front_m_before>[万亿]?(?:[美日欧]元|元))\s*[）\)])\s*[,，:：]*(\s*[^壹贰叁肆伍陆柒捌玖拾佰仟萬億分万元编号时间日期计采a-zA-Z]{,7}?))(?P<money_front_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_front_m>(E-?\d+))?(?:，?)[百千]*)())",
|
|
|
+ "behind_m": "(()()(?P<money_behind_m>\d{1,3}([,，]\d{3})+(\.\d+)?|\d+(\.\d+)?(?P<science_behind_m>(E-?\d+))?(?:，?)[百千]*)(人民币)?[\(（]?(?P<unit_behind_m>[万亿]?(?:[美日欧]元|元)(?P<filter_unit3>[台个只吨斤棵株页亩方条米]*))[\)）]?)"}
|
|
|
+ # 2021/7/19 调整金额，单位提取正则，修复部分金额因为单位提取失败被过滤问题。
|
|
|
+
|
|
|
+ pattern_money = re.compile("%s|%s|%s|%s" % (
|
|
|
+ list_money_pattern["cn"], list_money_pattern["key_word"], list_money_pattern["behind_m"],
|
|
|
+ list_money_pattern["front_m"]))
|
|
|
+
|
|
|
+ # 修复 元千万元
|
|
|
+ match = re.search('元[千百十]万元', sentence_text)
|
|
|
+ if match:
|
|
|
+ sentence_text = re.sub(re.escape(match.group()), match.group()[1:], sentence_text)
|
|
|
+
|
|
|
+ # 修复 千多万元
|
|
|
+ match = re.search('[千百十]多万元', sentence_text)
|
|
|
+ if match:
|
|
|
+ sentence_text = re.sub(re.escape(match.group()), match.group()[0] + match.group()[2:], sentence_text)
|
|
|
+
|
|
|
+ if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text):
|
|
|
+ found_yeji += 1
|
|
|
+ if found_yeji >= 2: # 过滤掉业绩后面的所有金额
|
|
|
+ all_match = []
|
|
|
+ else:
|
|
|
+ ser = re.search('((收费标准|计算[方公]?式)：|\w{3,5}\s*=)+\s*[中标投标成交金额招标人预算价格万元\s（）()\[\]【】\d\.%％‰\+\-*×/]{20,}[，。]?', sentence_text) # 过滤掉收费标准里面的金额
|
|
|
+ if ser:
|
|
|
+ all_match = re.finditer(pattern_money, sentence_text.replace(ser.group(0), ' ' * len(ser.group(0))))
|
|
|
+ else:
|
|
|
+ all_match = re.finditer(pattern_money, sentence_text)
|
|
|
+ for _match in all_match:
|
|
|
+ if len(_match.group()) > 0:
|
|
|
+ notes = '' # 2021/7/20 新增备注金额大写或金额单位 if 金额大写 notes=大写 elif 单位 notes=单位
|
|
|
+ unit = ""
|
|
|
+ entity_text = ""
|
|
|
+ start_index = ""
|
|
|
+ end_index = ""
|
|
|
+ text_beforeMoney = ""
|
|
|
+ filter = ""
|
|
|
+ filter_unit = False
|
|
|
+ notSure = False
|
|
|
+ science = ""
|
|
|
+ if re.search('业绩(公示|汇总|及|报告|\w{,2}(内容|情况|信息)|[^\w])', sentence_text[:_match.span()[0]]): # 2021/7/21过滤掉业绩后面金额
|
|
|
+ # print('金额在业绩后面: ', _match.group(0))
|
|
|
+ found_yeji += 1
|
|
|
+ break
|
|
|
+ yeji_table_flag = 0
|
|
|
+ for k, v in _match.groupdict().items():
|
|
|
+ if v != "" and v is not None:
|
|
|
+ if k == 'text_key_word':
|
|
|
+ notSure = True
|
|
|
+ if k.split("_")[0] == "money":
|
|
|
+ entity_text = v
|
|
|
+ # if is_yeji_table(tables_and_divs, entity_text):
|
|
|
+ if judge_yeji(len(sentence_text), sentence_text, 300, tables_and_divs, entity_text):
|
|
|
+ yeji_table_flag = 1
|
|
|
+ break
|
|
|
+ # print(_match.group(k), 'entity_text: ', sentence_text[_match.start(k): _match.end(k)])
|
|
|
+ if entity_text.endswith(',00'): # 金额逗号后面不可能为两个0结尾，应该小数点识别错，直接去掉
|
|
|
+ entity_text = entity_text[:-3]
|
|
|
+ if k.split("_")[0] == "unit":
|
|
|
+ if v == '万元' or unit == "": # 处理 预算金额(元)：160万元 这种出现前后单位不一致情况
|
|
|
+ unit = v
|
|
|
+ if k.split("_")[0] == "text":
|
|
|
+ # print('text_before: ', _match.group(k))
|
|
|
+ text_beforeMoney = v
|
|
|
+ if k.split("_")[0] == "filter":
|
|
|
+ filter = v
|
|
|
+ if re.search("filter_unit", k) is not None:
|
|
|
+ filter_unit = True
|
|
|
+ if k.split("_")[0] == 'science':
|
|
|
+ science = v
|
|
|
+
|
|
|
+ if yeji_table_flag:
|
|
|
+ continue
|
|
|
+
|
|
|
+ if filter != "":
|
|
|
+ continue
|
|
|
+ start_index, end_index = _match.span()
|
|
|
+ start_index += len(text_beforeMoney)
|
|
|
+
|
|
|
+ '''过滤掉手机号码作为金额'''
|
|
|
+ if re.search('电话|手机|联系|方式|编号|编码|日期|数字|时间', text_beforeMoney):
|
|
|
+ # print('过滤掉手机号码作为金额')
|
|
|
+ continue
|
|
|
+ elif re.search('^1[3-9]\d{9}$', entity_text) and re.search('：\w{1,3}$', text_beforeMoney): # 过滤掉类似 '13863441880', '金额（万元）：季勇13863441880'
|
|
|
+ # print('过滤掉手机号码作为金额')
|
|
|
+ continue
|
|
|
+
|
|
|
+ if unit == "": # 2021/7/21 有明显金额特征的补充单位，避免被过滤
|
|
|
+ if (re.search('(￥|¥|RMB|CNY)[:：]?$', text_beforeMoney) or re.search('[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,}', entity_text)):
|
|
|
+ if entity_text.endswith('万元'):
|
|
|
+ unit = '万元'
|
|
|
+ entity_text = entity_text[:-2]
|
|
|
+ else:
|
|
|
+ unit = '元'
|
|
|
+ # print('1明显金额特征补充单位 元')
|
|
|
+ elif re.search('USD[:：]?$', text_beforeMoney):
|
|
|
+ unit = '美元'
|
|
|
+ elif re.search('EUR[:：]?$', text_beforeMoney):
|
|
|
+ unit = '欧元'
|
|
|
+ elif re.search('JPY[:：]?$', text_beforeMoney):
|
|
|
+ unit = '日元'
|
|
|
+ elif re.search('^[-—]+[\d,.]+万元', sentence_text[end_index:]):
|
|
|
+ # print('两个金额连接后面的有单位，用后面单位')
|
|
|
+ unit = '万元'
|
|
|
+ elif re.search('([单报标限总造]价款?|金额|租金|(中标|成交|合同|承租|投资)）?[价额]|价格|预算(金额)?|(监理|设计|勘察)(服务)?费)[:：为]*-?$', text_beforeMoney.strip()) and re.search('^0|1[3|4|5|6|7|8|9]\d{9}', entity_text) == None:
|
|
|
+ if re.search('^[\d，,.]+$', entity_text) and float(re.sub('[,，]', '', entity_text))<500 and re.search('万元', sentence_text):
|
|
|
+ unit = '万元'
|
|
|
+ # print('金额较小且句子中有万元的，补充单位为万元')
|
|
|
+ elif re.search('^\d{1,3}\.\d{4,6}$', entity_text) and re.search('0000$', entity_text) == None:
|
|
|
+ unit = '万元'
|
|
|
+ else:
|
|
|
+ unit = '元'
|
|
|
+ # print('金额前面紧接关键词的补充单位 元')
|
|
|
+ elif re.search('(^\d{,3}(,?\d{3})+(\.\d{2,7}，?)$)|(^\d{,3}(,\d{3})+，?$)', entity_text):
|
|
|
+ unit = '元'
|
|
|
+ # print('3明显金额特征补充单位 元')
|
|
|
+ else:
|
|
|
+ # print('过滤掉没单位金额: ',entity_text)
|
|
|
+ continue
|
|
|
+ elif unit == '万元':
|
|
|
+ if end_index < len(sentence_text) and sentence_text[end_index] == '元' and re.search('\d$', entity_text):
|
|
|
+ unit = '元'
|
|
|
+ elif re.search('^[5-9]\d{6,}\.\d{2}$', entity_text): # 五百亿以上的万元改为元
|
|
|
+ unit = '元'
|
|
|
+
|
|
|
+ if unit.find("万") >= 0 and entity_text.find("万") >= 0: # 2021/7/19修改为金额文本有万，不计算单位
|
|
|
+ # print('修正金额及单位都有万， 金额：',entity_text, '单位:',unit)
|
|
|
+ unit = "元"
|
|
|
+
|
|
|
+ if re.search('.*万元万元', entity_text): # 2021/7/19 修正两个万元
|
|
|
+ # print(' 修正两个万元',entity_text)
|
|
|
+ entity_text = entity_text.replace('万元万元', '万元')
|
|
|
+ else:
|
|
|
+ if filter_unit:
|
|
|
+ continue
|
|
|
+
|
|
|
+ entity_text = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]", "", entity_text)
|
|
|
+ # print('转换前金额：', entity_text, '单位:', unit, '备注:',notes, 'text_beforeMoney:',text_beforeMoney)
|
|
|
+ if re.search('总投资|投资总额|总预算|总概算|投资规模|批复概算|投资额|投资估算|投资概算',
|
|
|
+ sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]): # 2021/8/5过滤掉总投资金额
|
|
|
+ # print('总投资金额: ', _match.group(0))
|
|
|
+ notes = '总投资'
|
|
|
+ elif re.search('建安费|建安工程费|建筑安装工程费|建安工程造价',
|
|
|
+ sentence_text[max(0, _match.span()[0] - 10):_match.span()[1]]): # 2021/11/18 投资金额不作为招标金额
|
|
|
+ notes = '建安费'
|
|
|
+ elif re.search('总工程造价|总造价',
|
|
|
+ sentence_text[max(0, _match.span()[0] - 8):_match.span()[1]]): # 2021/12/20 工程造价不作为招标金额
|
|
|
+ notes = '工程造价'
|
|
|
+
|
|
|
+ # 直接判断前面是否有业绩
|
|
|
+ if judge_yeji(max(0, _match.span()[0] - 10), sentence_text):
|
|
|
+ continue
|
|
|
+
|
|
|
+ if len(unit) > 0:
|
|
|
+ if unit.find('万') >= 0 and len(entity_text.split('.')[0]) >= 8: # 2021/7/19 修正万元金额过大的情况
|
|
|
+ entity_text = str(
|
|
|
+ getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]) / 10000)
|
|
|
+ unit = '元' # 修正金额后单位 重置为元
|
|
|
+ else:
|
|
|
+ entity_text = str(getUnifyMoney(entity_text) * getMultipleFactor(re.sub("[美日欧]", "", unit)[0]))
|
|
|
+ else:
|
|
|
+ if entity_text.find('万') >= 0 and entity_text.split('.')[0].isdigit() and len(
|
|
|
+ entity_text.split('.')[0]) >= 8:
|
|
|
+ entity_text = str(getUnifyMoney(entity_text) / 10000)
|
|
|
+ # print('修正金额字段含万 过大的情况')
|
|
|
+ else:
|
|
|
+ entity_text = str(getUnifyMoney(entity_text))
|
|
|
+ if science and re.search('^E-?\d+$', science): # 科学计数
|
|
|
+ entity_text = str(Decimal(entity_text + science)) if Decimal(entity_text + science) > 100 and Decimal(
|
|
|
+ entity_text + science) < 10000000000 else entity_text # 结果大于100及小于100万才使用科学计算
|
|
|
+
|
|
|
+ if float(entity_text) > 100000000000: # float(entity_text)<100 or 2022/3/4 取消最小金额限制
|
|
|
+ continue
|
|
|
+
|
|
|
+ if notSure and unit == "" and float(entity_text) > 100 * 10000:
|
|
|
+ # print('过滤掉金额 notSure and unit=="" and float(entity_text)>100*10000：', entity_text, unit)
|
|
|
+ continue
|
|
|
+ # print("金额：{0} ,单位：{1}, 前文：{2}, filter: {3}, filter_unit: {4}".format(entity_text, unit, text_beforeMoney,
|
|
|
+ # filter, filter_unit))
|
|
|
+ if re.search('[%％‰折]|费率|下浮率', text_beforeMoney) and float(entity_text)<1000: # 过滤掉可能是费率的金额
|
|
|
+ # print('过滤掉可能是费率的金额')
|
|
|
+ continue
|
|
|
+ money_list.append((entity_text, start_index, end_index, unit, notes))
|
|
|
+
|
|
|
+ # 排除过小的金额
|
|
|
+ temp_list = []
|
|
|
+ for money in money_list:
|
|
|
+ if int(float(money[0])) < 1000:
|
|
|
+ continue
|
|
|
+ temp_list.append(money)
|
|
|
+ money_list = temp_list
|
|
|
+
|
|
|
+ return money_list, found_yeji
|
|
|
+
|
|
|
+
|
|
|
+def chinese_to_arabic(ch_str):
|
|
|
+ chinese_number_dict = {
|
|
|
+ '一': 1,
|
|
|
+ '二': 2,
|
|
|
+ '两': 2,
|
|
|
+ '三': 3,
|
|
|
+ '四': 4,
|
|
|
+ '五': 5,
|
|
|
+ '六': 6,
|
|
|
+ '七': 7,
|
|
|
+ '八': 8,
|
|
|
+ '九': 9,
|
|
|
+ '十': 10,
|
|
|
+ '拾': 10,
|
|
|
+ '百': 100,
|
|
|
+ '千': 1000,
|
|
|
+ }
|
|
|
+ no_list = []
|
|
|
+ for c in ch_str:
|
|
|
+ if c not in chinese_number_dict.keys():
|
|
|
+ return None
|
|
|
+ no_list.append(chinese_number_dict.get(c))
|
|
|
+ arabic_num = 0
|
|
|
+ mul_no = None
|
|
|
+ for i, no in enumerate(no_list):
|
|
|
+ if no in [10, 100, 1000]:
|
|
|
+ if mul_no is None:
|
|
|
+ arabic_num += no
|
|
|
+ else:
|
|
|
+ arabic_num += no * mul_no
|
|
|
+ mul_no = None
|
|
|
+ else:
|
|
|
+ mul_no = no
|
|
|
+ if mul_no:
|
|
|
+ arabic_num += mul_no
|
|
|
+
|
|
|
+ return arabic_num
|
|
|
+
|
|
|
+
|
|
|
+def reg_word_sort(reg):
|
|
|
+ ss = reg.split('|')
|
|
|
+ ss.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ reg = '|'.join(ss)
|
|
|
+ return reg
|
|
|
+
|
|
|
+
|
|
|
+def get_stage_pattern():
|
|
|
+ stage_dict = {
|
|
|
+ "立项阶段": "立项|项目投资",
|
|
|
+ "可研阶段": "可行性研究|可研",
|
|
|
+ "环评阶段": "环境评价|环境影响|环境评测|环评|(水保|环保|环境保护)(编制|验收|监测)",
|
|
|
+ "稳评阶段": "稳定风险|社会稳定|风险评估",
|
|
|
+ "咨询阶段": "(水影响|能源|交通影响|地质灾害|地址灾害|地震安全性|地震安全性|气象|雷击风险|安全|海洋|森林环境)(评[价估测])|水土保持|(水|交|灾|震|能|气|安|海|林)评",
|
|
|
+ "造价阶段": "(决算书|预算|结算|造价|决算)(编制|咨询)",
|
|
|
+ "设计阶段": "(施工图(纸|)|初步|项目|工程)(方案|)设计|测绘|规划设计",
|
|
|
+ "勘察阶段": "(勘察|勘查)设计|勘察技术|勘查|勘察",
|
|
|
+ "施工图审": "(施工图(纸|)|防雷|消防|人防)审查|施工图审",
|
|
|
+ "施工许可": "施工许可证",
|
|
|
+ "施工准备": "施工准备|监理|资格预审|资审",
|
|
|
+ "施工在建": "施工",
|
|
|
+ "竣工阶段": "竣工|验收",
|
|
|
+ "EPC总承包": "总承包|EPC"
|
|
|
+ }
|
|
|
+
|
|
|
+ stage_priority_dict = {
|
|
|
+ "立项阶段": 1,
|
|
|
+ "可研阶段": 3,
|
|
|
+ "环评阶段": 2,
|
|
|
+ "稳评阶段": 3,
|
|
|
+ "咨询阶段": 2,
|
|
|
+ "造价阶段": 2,
|
|
|
+ "设计阶段": 4,
|
|
|
+ "勘察阶段": 4,
|
|
|
+ "施工图审": 2,
|
|
|
+ "施工许可": 2,
|
|
|
+ "施工准备": 3,
|
|
|
+ "施工在建": 5,
|
|
|
+ "竣工阶段": 3,
|
|
|
+ "EPC总承包": 4
|
|
|
+ }
|
|
|
+
|
|
|
+ list_stage_v = []
|
|
|
+ for k,v in stage_dict.items():
|
|
|
+ list_stage_v.append("(?P<%s>%s)"%(k,v))
|
|
|
+ stage_pattern = "|".join(list_stage_v)
|
|
|
+ return stage_pattern, stage_priority_dict
|
|
|
+
|
|
|
+
|
|
|
+def get_industry_pattern():
|
|
|
+ filename = os.path.abspath(os.path.dirname(__file__)) + "/proposedBuildingKeyword.xlsx"
|
|
|
+ df = pd.read_excel(filename)
|
|
|
+ dict_industry_keywords = {}
|
|
|
+ for _industry, _keyword in zip(df["类别"], df["关键词"]):
|
|
|
+ if _industry not in dict_industry_keywords:
|
|
|
+ dict_industry_keywords[_industry] = set()
|
|
|
+ dict_industry_keywords[_industry].add(_keyword)
|
|
|
+ list_industry_p = []
|
|
|
+ for k, v in dict_industry_keywords.items():
|
|
|
+ if len(v) > 0:
|
|
|
+ list_industry_p.append("(?P<%s>%s)" % (k, "|".join(list(v))))
|
|
|
+ # logging.info('get_industry_pattern ' + str(list_industry_p))
|
|
|
+ _pattern = re.compile("|".join(list_industry_p))
|
|
|
+ return _pattern
|
|
|
+
|
|
|
+
|
|
|
+def get_property_pattern():
|
|
|
+ property_dict = {
|
|
|
+ "复合性质": "扩迁建|扩改建|扩翻建|改扩建|改翻建|迁扩建|迁改建|迁扩建|翻改建|翻扩建",
|
|
|
+ "迁建": "迁建|搬迁重建",
|
|
|
+ "扩建": "扩建|增建|加建|扩大",
|
|
|
+ "翻建": "翻建|拆除重建",
|
|
|
+ "改建": "改造|改建|技改|提升|改进|整改",
|
|
|
+ "装饰装修": "装修|室内装饰|室外装饰|装饰工程|装饰改造工程|装饰装修|装修装饰|幕墙工程|修缮|外墙翻新|翻新工程|整修|修补|装饰|维修",
|
|
|
+ "拆除": "拆除",
|
|
|
+ "恢复重建": "恢复重建|灾后重建",
|
|
|
+ '其他': '整治|修复|环境治理|更换'
|
|
|
+ }
|
|
|
+
|
|
|
+ property_priority_dict = {
|
|
|
+ "复合性质": 1,
|
|
|
+ "迁建": 2,
|
|
|
+ "扩建": 3,
|
|
|
+ "翻建": 4,
|
|
|
+ "改建": 5,
|
|
|
+ "装饰装修": 6,
|
|
|
+ "拆除": 7,
|
|
|
+ "恢复重建": 8,
|
|
|
+ '其他': 9,
|
|
|
+ "新建": 10
|
|
|
+ }
|
|
|
+
|
|
|
+ list_property_v = []
|
|
|
+ for k,v in property_dict.items():
|
|
|
+ list_property_v.append("(?P<%s>%s)"%(k,v))
|
|
|
+ property_pattern = "|".join(list_property_v)
|
|
|
+ return property_pattern, property_priority_dict
|
