| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012 |
- # -*- coding: utf-8 -*-
- """Phase 6 migration from interface/getAttributes.py.
- 本模块承载从 ``BiddingKG.dl.interface.getAttributes`` 原样迁出的时间相关属性
- 装配函数,作为 Phase 6 装配层(assembly)重组的一部分。原位置与行号:
- - ``turnBidWay`` — 采购方式归一(原 3315-3333 行)
- - ``my_time_format_pattern`` — 时间正则常量(原 3370 行)
- - ``time_sub_pattern`` — 时间分隔符正则常量(原 3371 行)
- - ``my_timeFormat`` — 时间文本格式化为 ``YYYY-MM-DD`` 列表(原 3373-3457 行)
- - ``timeAdd`` — 日期加减天数/分钟(原 3459-3463 行)
- - ``getTimeAttributes`` — 装配时间类属性(原 3465-4210 行)
- - ``getOtherAttributes`` — 装配其它属性(资金来源/服务期等)(原 4413-4529 行)
- 注:``extract_serviceTime``、``get_days_between``、``turnMoneySource`` 已先一步
- 下沉到 ``BiddingKG.dl.common.attr_utils``,本模块按依赖方向约束改为从
- ``common.attr_utils`` 显式导入,避免反向依赖 ``interface.getAttributes``。
- """
- from __future__ import absolute_import
- import re
- import time
- from datetime import datetime
- from decimal import Decimal
- from BiddingKG.dl.common.attr_utils import extract_serviceTime, get_days_between, turnMoneySource
- from BiddingKG.dl.ratio.re_ratio import getUnifyNum
- from BiddingKG.dl.common.Utils import isValidDate, getDigitsDic
- __all__ = ["turnBidWay", "my_timeFormat", "timeAdd", "getTimeAttributes", "getOtherAttributes",
- "my_time_format_pattern", "time_sub_pattern"]
- def turnBidWay(bidway):
- if bidway in ("邀请招标","采购方式:邀请"):
- return "邀请招标"
- elif bidway in ("询价","询单","询比","采购方式:询价"):
- return "询价"
- elif bidway in ("竞谈","竞争性谈判","公开竞谈"):
- return "竞争性谈判"
- elif bidway in ("竞争性磋商","磋商"):
- return "竞争性磋商"
- elif bidway in ("竞价","竞标","电子竞价","以电子竞价","电子书面竞投"):
- return "竞价"
- elif bidway in ("公开招标","网上电子投标","网上招标","采购方式:公开","招标为其他"):
- return "公开招标"
- elif bidway in ("单一来源"):
- return "单一来源"
- elif bidway in ("比选"):
- return "比选"
- else:
- return "其他"
- my_time_format_pattern = re.compile("((?:(?P<year>20\d{2}|\d{2}|二[零〇0][零〇一二三四五六七八九0]{2})\s*[-/年.]\s*)?(?P<month>\d{1,2}|[一二三四五六七八九十]{1,3})\s*[-/月.]\s*(?P<day>\d{1,2}|[一二三四五六七八九十]{1,3}))(?:.?(?:[0-1][0-9]|2[0-4]|[0-9]):[0-5][0-9](?::[0-5][0-9])?)?")
- time_sub_pattern = re.compile("[\[\]【】()(){}{}]+")
- from BiddingKG.dl.ratio.re_ratio import getUnifyNum
- def my_timeFormat(_time,page_time):
- # if page_time:
- # current_year = time.strftime("%Y",time.localtime(int(datetime.strptime(page_time, '%Y-%m-%d').timestamp())))
- # else:
- # current_year = time.strftime("%Y",time.localtime())
- _time = re.sub(time_sub_pattern,"",_time)
- all_match = re.finditer(my_time_format_pattern,_time)
- time_list = []
- idx = 0
- global_year = ""
- for _match in all_match:
- if len(_match.group())>0:
- idx += 1
- legal = True
- year = ""
- month = ""
- day = ""
- for k,v in _match.groupdict().items():
- if k=="year":
- year = v
- if k=="month":
- month = v
- if k=="day":
- day = v
- if year!="":
- if year==None: # 例:5月18日
- if idx==2 and global_year: # 例:2025年5月14日-5月18日,第二个时间没年份
- year = global_year
- else:
- legal = False
- else:
- if re.search("^\d+$", year):
- if len(year) == 2:
- year = "20" + year
- # if int(year) - int(current_year) > 5 or int(year) - int(current_year) < -1:
- # legal = False
- # else:
- # if int(year) - int(current_year)>10 or int(year) - int(current_year) < -1:
- # legal = False
- else:
- _year = ""
- for word in year:
- if word == '0':
- _year += word
- else:
- _year += str(getDigitsDic(word))
- year = _year
- else:
- legal = False
- if month!="":
- if re.search("^\d+$", month):
- if int(month) > 12:
- legal = False
- else:
- month = int(getUnifyNum(month))
- if month >= 1 and month <= 12:
- month = str(month)
- else:
- legal = False
- else:
- legal = False
- if day!="":
- if re.search("^\d+$", day):
- if int(day) > 31:
- legal = False
- else:
- day = int(getUnifyNum(day))
- if day >= 1 and day <= 31:
- day = str(day)
- else:
- legal = False
- else:
- legal = False
- if legal and not isValidDate(int(year),int(month),int(day)):
- legal = False
- if legal:
- # 数字字符格式化
- year = str(int(year))
- month = str(int(month))
- day = str(int(day))
- time_list.append("%s-%s-%s"%(year,month.rjust(2,"0"),day.rjust(2,"0")))
- # if idx==1 and not global_year:
- if not global_year:
- global_year = year
- return time_list,global_year
- def timeAdd(_time,days,format="%Y-%m-%d",minutes=0):
- a = time.mktime(time.strptime(_time,format))+86400*days+60*minutes
- _time1 = time.strftime(format,time.localtime(a))
- return _time1
- def getTimeAttributes(list_entity,list_sentence,page_time):
- # from BiddingKG.dl.interface.htmlparser import get_childs
- # document_tree = parse_document.tree
- # new_document_tree = []
- # _data_i = -1
- # while _data_i < len(document_tree) - 1:
- # _data_i += 1
- # _data = document_tree[_data_i]
- # _type = _data["type"]
- # if _type == "sentence":
- # if _data["sentence_title"] is not None:
- # new_document_tree.append(_data)
- # document_tree = new_document_tree
- time_entitys = [i for i in list_entity if i.entity_type=='time']
- time_entitys = sorted(time_entitys,key=lambda x:(x.sentence_index, x.begin_index))
- list_sentence = sorted(list_sentence,key=lambda x:x.sentence_index)
- dict_time = {
- "time_release": [], # 1 发布时间
- "time_bidopen": [], # 2 开标时间
- "time_bidclose": [], # 3 截标时间
- 'time_bidstart': [], # 12 投标(开始)时间、响应文件接收(开始)时间
- 'time_publicityStart': [], # 4 公示开始时间(公示时间、公示期)
- 'time_publicityEnd': [], # 5 公示截止时间
- 'time_getFileStart': [], # 6 文件获取开始时间(文件获取时间)
- 'time_getFileEnd': [], # 7 文件获取截止时间
- 'time_registrationStart': [], # 8 报名开始时间(报名时间)
- 'time_registrationEnd': [], # 9 报名截止时间
- 'time_earnestMoneyStart': [], #10 保证金递交开始时间(保证金递交时间)
- 'time_earnestMoneyEnd': [] , # 11 保证金递交截止时间
- 'time_commencement':[] , #13 开工日期
- 'time_completion': [], # 14 竣工日期
- 'time_listingStart': [], # 15 挂牌开始日期(挂牌时间)
- 'time_listingEnd': [], # 16 挂牌结束日期、挂牌截止日期
- 'time_signContract': [], # 17 合同签订时间
- 'time_contractStart': [], # 18 合同开始时间
- 'time_contractEnd': [] # 19 合同结束时间
- }
- dict_time2label = {
- "time_release": 1, # 1 发布时间
- "time_bidopen": 2, # 2 开标时间
- "time_bidclose": 3, # 3 截标时间
- 'time_bidstart': 12, # 12 投标(开始)时间、响应文件接收(开始)时间
- 'time_publicityStart': 4, # 4 公示开始时间(公示时间、公示期)
- 'time_publicityEnd': 5, # 5 公示截止时间
- 'time_getFileStart': 6, # 6 文件获取开始时间(文件获取时间)
- 'time_getFileEnd': 7, # 7 文件获取截止时间
- 'time_registrationStart': 8, # 8 报名开始时间(报名时间)
- 'time_registrationEnd': 9, # 9 报名截止时间
- 'time_earnestMoneyStart': 10, # 10 保证金递交开始时间(保证金递交时间)
- 'time_earnestMoneyEnd': 11, # 11 保证金递交截止时间
- 'time_commencement': 13, # 13 开工日期
- 'time_completion': 14, # 14 竣工日期
- 'time_listingStart': 15, # 15 挂牌开始日期(挂牌时间)
- 'time_listingEnd': 16, # 16 挂牌结束日期、挂牌截止日期
- 'time_signContract': 17, # 17 合同签订时间
- 'time_contractStart': 18, # 18 合同开始时间
- 'time_contractEnd': 19 # 19 合同结束时间
- }
- last_sentence_index = 0
- last_time_type = ""
- last_time_index = {
- 'time_bidstart':"time_bidclose",
- 'time_publicityStart':"time_publicityEnd",
- 'time_getFileStart':"time_getFileEnd",
- 'time_registrationStart':"time_registrationEnd",
- 'time_earnestMoneyStart':"time_earnestMoneyEnd",
- 'time_commencement':"time_completion",
- 'time_listingStart':"time_listingEnd",
- 'time_contractStart':"time_contractEnd"
- }
- # time_entitys = [[_entity,my_timeFormat(_entity.entity_text,page_time)] for _entity in time_entitys]
- new_time_entitys = []
- year_list = []
- for _entity in time_entitys:
- _time_list,_year = my_timeFormat(_entity.entity_text,page_time)
- _in_attachment = _entity.in_attachment
- if _time_list:
- new_time_entitys.append([_entity,_time_list,_year])
- year_list.append([_year,_in_attachment])
- get_all_time = False if False in [i[1] for i in year_list] else True
- if page_time:
- current_year = time.strftime("%Y",time.localtime(int(datetime.strptime(page_time, '%Y-%m-%d').timestamp())))
- year_list.append([current_year,False])
- else:
- current_year = time.strftime("%Y",time.localtime())
- if get_all_time:
- year_list = [i[0] for i in year_list]
- else:
- year_list = [i[0] for i in year_list if not i[1]]
- year_list = [(y,year_list.count(y)) for y in year_list if y[:2]=='20']
- year_list.sort(key=lambda x:x[1],reverse=True)
- most_year = year_list[0][0] if year_list else ""
- if most_year:
- time_entitys = [item for item in new_time_entitys if int(item[2])-int(most_year)<=10 and int(item[2])-int(most_year)>=-1]
- else:
- time_entitys = new_time_entitys
- # print(time_entitys)
- for entity_idx in range(len(time_entitys)):
- entity = time_entitys[entity_idx][0]
- extract_time = time_entitys[entity_idx][1]
- sentence_text = list_sentence[entity.sentence_index].sentence_text
- previous_entity = time_entitys[entity_idx-1][0] if entity_idx!=0 else None
- previous_extract_time = time_entitys[entity_idx-1][1] if entity_idx!=0 else None
- next_entity = time_entitys[entity_idx+1][0] if entity_idx!=len(time_entitys)-1 else None
- next_extract_time = time_entitys[entity_idx+1][1] if entity_idx!=len(time_entitys)-1 else None
- # 实体有效上下文
- entity_context_begin = previous_entity.wordOffset_end if previous_entity and previous_entity.sentence_index==entity.sentence_index else 0
- entity_context_end = next_entity.wordOffset_begin if next_entity and next_entity.sentence_index==entity.sentence_index else len(sentence_text)
- if entity.sentence_index!=last_sentence_index:
- # sentence_index 不同句子重置last_time_type
- last_time_type = ""
- entity_left = sentence_text[max(entity_context_begin, entity.wordOffset_begin - 2):entity.wordOffset_begin]
- entity_left2 = sentence_text[max(entity_context_begin, entity.wordOffset_begin - 10):entity.wordOffset_begin]
- entity_left3 = sentence_text[max(entity_context_begin, entity.wordOffset_begin - 30):entity.wordOffset_begin]
- entity_right = sentence_text[entity.wordOffset_end:min(entity.wordOffset_end + 3,entity_context_end)]
- entity_right2 = sentence_text[entity.wordOffset_end:entity_context_end]
- entity_right2 = re.sub(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+",'',entity_right2)[:60] # 去除网址
- # print(entity.entity_text,entity_right2)
- label_prob = entity.values[entity.label]
- entity_text = entity.entity_text
- in_attachment = entity.in_attachment
- if extract_time:
- definite_time_list = []
- t = re.compile("(北京时间)?(?P<day>下午|上午|早上)?(?P<hour>(?:[0-1][0-9]|2[0-4]|[0-9]))[::时点](?P<half_hour>半)?(?P<minute>(?:[0-5][0-9]|[0-9]))?[::分]?(?P<second>[0-5][0-9])?秒?")
- _entity_text = re.sub(" (?=[^\d])|(?<=[^\d]) ","",entity_text)
- _entity_text_len = len(_entity_text)
- _entity_text = _entity_text + sentence_text[entity.wordOffset_end:entity.wordOffset_end+20]
- t_in_word_num = len(re.findall(t,_entity_text))
- # t_out_of_word = re.search("^[^\d]{,2}"+t.pattern,re.sub(" (?=[^\d])|(?<=[^\d]) ","",sentence_text[entity.wordOffset_end:]))
- begin_index = 0
- definite_time_idx_list = []
- for _num in range(t_in_word_num):
- if begin_index> _entity_text_len + 8:
- break
- t_in_word = re.search(t, _entity_text[begin_index:])
- # print(_entity_text[begin_index:])
- if t_in_word:
- if _num==0 and t_in_word.start() > _entity_text_len + 8:
- break
- begin_index += t_in_word.end()
- # print('t_in_word',entity_text,t_in_word.groupdict())
- day = t_in_word.groupdict().get('day',"")
- hour = t_in_word.groupdict().get('hour',"")
- half_hour = t_in_word.groupdict().get('half_hour',"")
- minute = t_in_word.groupdict().get('minute',"")
- second = t_in_word.groupdict().get('second',"")
- if not minute:
- second = ""
- if hour:
- if day=='下午' and int(hour)<12:
- hour = str(int(hour)+12)
- if int(hour)>=24:
- continue
- else:
- hour = "00"
- if not minute:
- if half_hour:
- minute = "30"
- else:
- minute = "00"
- if int(minute)>=60:
- continue
- if not second:
- second = "00"
- if int(second)>=60:
- continue
- definite_time = "%s:%s:%s"%(hour.rjust(2,"0"),minute.rjust(2,"0"),second.rjust(2,"0"))
- # print(definite_time)
- definite_time_list.append(definite_time)
- definite_time_idx_list.append([begin_index-len(t_in_word.group()),begin_index])
- if len(extract_time)==1 and len(definite_time_list)>=2: # 实体只包含一个时间,"2024-12-09 09:00~16:00" 考虑单个时间对应两个详细时间段的识别
- # 前两个详细时间的间隔
- distance = definite_time_idx_list[1][0] - definite_time_idx_list[0][1]
- if distance<=8 and int(definite_time_list[1][:2])>=int(definite_time_list[0][:2]): # 判断详细时间都‘小时’顺序从小到大
- new_extract_time = []
- for d_time in definite_time_list[:2]:
- if d_time == "24:00:00": # 修正不规范时间表述
- d_time = "23:59:59"
- new_extract_time.append(extract_time[0] + " " + d_time)
- extract_time = new_extract_time
- else:
- if definite_time_list[0] == "24:00:00": # 修正不规范时间表述
- definite_time_list[0] = "23:59:59"
- if definite_time_list[0] != "00:00:00":
- extract_time[0] = extract_time[0] + " " + definite_time_list[0]
- else:
- min_len = min(len(extract_time),len(definite_time_list))
- for i in range(min_len):
- if definite_time_list[i] == "24:00:00": # 修正不规范时间表述
- definite_time_list[i] = "23:59:59"
- if definite_time_list[i] != "00:00:00":
- extract_time[i] = extract_time[i] + " " + definite_time_list[i]
- if extract_time:
- # 时间变更prob优化
- if re.search("原",entity_left2):
- last_index = 0
- for item in re.finditer("原",entity_left2):
- last_index = item.start() + 1
- label_prob = label_prob - 0.2 * last_index / len(entity_left2)
- # print('prob优化',label_prob,extract_time)
- elif (re.search("改正|更正|修正|修改|更改|变更|延期|调整|变成",entity_left3[-20:]) or re.search("现",entity_left3[-10:])) and not re.search("更正日期",entity_left3[-8:]):
- last_time_label = dict_time2label.get(last_time_type,None)
- if last_time_label and entity.label==0:
- entity.label = last_time_label
- label_prob = 1.5
- elif last_time_label and entity.label==last_time_label:# 前后两个相同类型的时间为变更关系
- label_prob = 2
- # 优化多个并列的时间,如:开标时间和截标时间,截标时间和报名结束时间
- if entity.label in [2,3,9]:
- if entity.label==2 and re.search("截标|投标.{,2}截止|([递提]交|接收)(?:文件)?.{,2}截止|报价.{,2}截止|响应.{,2}截止|文件.{,2}([递提]交|接收)",entity_left3):
- dict_time['time_bidclose'].append((extract_time[0], label_prob-0.1, in_attachment))
- if entity.label==3 and re.search("开标|(评审|比选).{,2}(?:开始)?(时间|日期)|选取.{,2}(时间|日期)",entity_left3):
- dict_time['time_bidopen'].append((extract_time[0], label_prob-0.1, in_attachment))
- if entity.label==3 and re.search("报名",entity_left3):
- dict_time['time_registrationEnd'].append((extract_time[0], 0.5, in_attachment))
- if entity.label==3 and re.search("获取",entity_left3[-20:]):
- dict_time['time_getFileEnd'].append((extract_time[0], 0.45, in_attachment))
- if entity.label==9 and re.search("截标|投标.{,2}截止|([递提]交|接收)(?:文件)?.{,2}截止|报价.{,2}截止|响应.{,2}截止|文件.{,2}([递提]交|接收)",entity_left3):
- dict_time['time_bidclose'].append((extract_time[0], label_prob-0.1, in_attachment))
- if entity.label in [11, 3]:
- if entity.label==11 and re.search("文件.{,2}([递提]交|接收)|截标|投标.{,2}截止|([递提]交|接收)(?:文件)?.{,2}截止|报价.{,2}截止|响应.{,2}截止",entity_left3):
- dict_time['time_bidclose'].append((extract_time[0], 0.5, in_attachment))
- if entity.label==3 and re.search("保证金.{,2}(接受|收取)|(接受|收取).{,2}保证金",entity_left3):
- dict_time['time_earnestMoneyEnd'].append((extract_time[0], 0.5, in_attachment))
- if entity.label in [6, 7]:
- if re.search("文件.{,2}([递提]交|接收)|截标|投标.{,2}截止|([递提]交|接收)(?:文件)?.{,2}截止|报价.{,2}截止|响应.{,2}截止",entity_left3):
- dict_time['time_bidclose'].append((extract_time[0], 0.5, in_attachment))
- if entity.label==0:
- if re.search("文件.{,2}([递提]交|接收)|截标|投标.{,2}截止|([递提]交|接收)(?:文件)?.{,2}截止|报价.{,2}截止|响应.{,2}截止",entity_left3):
- if len(extract_time)>=2:
- dict_time['time_bidstart'].append((extract_time[0], 0.45, in_attachment))
- dict_time['time_bidclose'].append((extract_time[1], 0.45, in_attachment))
- else:
- dict_time['time_bidclose'].append((extract_time[0], 0.45, in_attachment))
- if entity.label==6:
- # "文件获取时间"和"报名时间"并列
- if re.search("报名",entity_left3):
- if len(extract_time)==1:
- dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- else:
- dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- dict_time['time_registrationEnd'].append((extract_time[1], 0.51, in_attachment))
- # 获取文件/报名/报价 时间补充(上下文表达过长无法通过模型识别)
- # if entity.label == 0:
- # if re.search("(获取|领取|售卖|出售|购买|下载).{,4}(招标|投标|采购)?(文件|标书)|(文件|标书).{,4}(获取|售卖|出售|发售|购买)", entity_left3):
- # if len(extract_time)==2:
- # dict_time['time_getFileStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_getFileEnd'].append((extract_time[1], 0.51, in_attachment))
- # else:
- # if next_entity and next_entity.sentence_index==entity.sentence_index:
- # mid_text = sentence_text[entity.wordOffset_end:next_entity.wordOffset_begin]
- # if len(mid_text)<=10 and re.search("至|到|[-—]|[~~]",mid_text) and len(next_extract_time)==1:
- # dict_time['time_getFileStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_getFileEnd'].append((next_extract_time[0], 0.51, in_attachment))
- # if not dict_time['time_getFileEnd']:
- # if re.search("前|止|截止", entity_right) or re.search("前",entity_text[-2:]):
- # dict_time['time_getFileEnd'].append((extract_time[0], 0.51, in_attachment))
- # elif re.search("起|开?始", entity_right) or re.search("起",entity_text[-2:]):
- # dict_time['time_getFileStart'].append((extract_time[0], 0.51, in_attachment))
- # if re.search("(进行|在线|线下|线上|网上).{,2}报名|报名.{,2}(开始)?(时间|日期)", entity_left3):
- # if len(extract_time)==2:
- # dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_registrationEnd'].append((extract_time[1], 0.51, in_attachment))
- # else:
- # if next_entity and next_entity.sentence_index==entity.sentence_index:
- # mid_text = sentence_text[entity.wordOffset_end:next_entity.wordOffset_begin]
- # if len(mid_text)<=10 and re.search("至|到|[-—]|[~~]",mid_text) and len(next_extract_time)==1:
- # dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_registrationEnd'].append((next_extract_time[0], 0.51, in_attachment))
- # if not dict_time['time_registrationEnd']:
- # if re.search("前|止|截止", entity_right) or re.search("前",entity_text[-2:]):
- # dict_time['time_registrationEnd'].append((extract_time[0], 0.51, in_attachment))
- # elif re.search("起|开?始", entity_right) or re.search("起",entity_text[-2:]):
- # dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- #
- # if re.search("(获取|售卖|出售|购买).{,4}(招标|投标|采购)?(文件|标书)|(文件|标书).{,4}(获取|售卖|出售|发售|购买)", entity_right2):
- # if len(extract_time)==2:
- # dict_time['time_getFileStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_getFileEnd'].append((extract_time[1], 0.51, in_attachment))
- # else:
- # if previous_entity and previous_entity.sentence_index==entity.sentence_index:
- # mid_text = sentence_text[previous_entity.wordOffset_end:entity.wordOffset_begin]
- # if len(mid_text)<=10 and re.search("至|到|[-—]|[~~]",mid_text) and len(previous_extract_time)==1:
- # dict_time['time_getFileStart'].append((previous_extract_time[0], 0.51, in_attachment))
- # dict_time['time_getFileEnd'].append((extract_time[0], 0.51, in_attachment))
- # if not dict_time['time_getFileEnd']:
- # if re.search("前|止|截止", entity_right) or re.search("前",entity_text[-2:]):
- # dict_time['time_getFileEnd'].append((extract_time[0], 0.51, in_attachment))
- # elif re.search("起|开?始", entity_right) or re.search("起",entity_text[-2:]):
- # dict_time['time_getFileStart'].append((extract_time[0], 0.51, in_attachment))
- # if re.search("(进行|在线|线下).{,2}报名", entity_right2):
- # if len(extract_time) == 2:
- # dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- # dict_time['time_registrationEnd'].append((extract_time[1], 0.51, in_attachment))
- # else:
- # if previous_entity and previous_entity.sentence_index==entity.sentence_index:
- # mid_text = sentence_text[previous_entity.wordOffset_end:entity.wordOffset_begin]
- # if len(mid_text)<=10 and re.search("至|到|[-—]|[~~]",mid_text) and len(previous_extract_time)==1:
- # dict_time['time_registrationStart'].append((previous_extract_time[0], 0.51, in_attachment))
- # dict_time['time_registrationEnd'].append((extract_time[0], 0.51, in_attachment))
- # if not dict_time['time_registrationEnd']:
- # if re.search("前|止|截止", entity_right) or re.search("前",entity_text[-2:]):
- # dict_time['time_registrationEnd'].append((extract_time[0], 0.51, in_attachment))
- # elif re.search("起|开?始", entity_right) or re.search("起",entity_text[-2:]):
- # dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- # if re.search("(进行|开始).{,4}(报价|投标|竞价)", entity_right2):
- # if len(extract_time) == 2:
- # dict_time['time_bidstart'].append((extract_time[0], 0.51, in_attachment))
- # # dict_time['time_bidclose'].append((extract_time[1], 0.51, in_attachment))
- # 补充公告末尾处的发布时间
- if entity.label==0:
- if entity.is_tail:
- entity.label = 1
- entity.values[1] = 0.5
- dict_time['time_release'].append((extract_time[0], 0.5, in_attachment))
- # 2022/12/12 新增挂牌时间正则
- if re.search("挂牌.{,4}(?:时间|日期)",entity_left2):
- if re.search("挂牌.{,4}(?:时间|日期)",entity_left2).end()>len(entity_left2)/2:
- if len(extract_time) == 1:
- if re.search("挂牌.?(开始|起始).?(?:时间|日期)",entity_left2):
- dict_time['time_listingStart'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_listingStart'
- elif re.search("挂牌.?(截[止至]|结束).?(?:时间|日期)",entity_left2):
- dict_time['time_listingEnd'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_listingEnd'
- elif re.search("挂牌.?(?:时间|日期)",entity_left2):
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_listingEnd'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_listingEnd'
- else:
- dict_time['time_listingStart'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_listingStart'
- else:
- dict_time['time_listingStart'].append((extract_time[0], 0.5, in_attachment))
- dict_time['time_listingEnd'].append((extract_time[1], 0.5, in_attachment))
- last_time_type = ''
- last_sentence_index = entity.sentence_index
- continue
- # 2023/9/13 新增合同相关时间
- if re.search("合同|服务|履[约行]", entity_left3[-15:]):
- if len(extract_time) == 1:
- if re.search("(合同.{,2}签[订定署].{,2}|签[订定署].{,2}合同.{,2})(?:时间|日期)|合同签[订定署].{,1}$", entity_left2):
- dict_time['time_signContract'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_signContract'
- last_sentence_index = entity.sentence_index
- continue
- elif re.search("(合同|服务|履约|(合同|服务)履行).{,4}(?:起始|开始)(?:时间|日期)", entity_left3[-15:]):
- dict_time['time_contractStart'].append((extract_time[0], 0.55, in_attachment))
- last_time_type = 'time_contractStart'
- last_sentence_index = entity.sentence_index
- continue
- elif re.search("(合同|服务|履约).{,2}(?:完成|截止|结束)(?:时间|日期|时限)", entity_left2):
- dict_time['time_contractEnd'].append((extract_time[0], 0.55, in_attachment))
- last_time_type = 'time_contractEnd'
- last_sentence_index = entity.sentence_index
- continue
- elif re.search("(?:合同|服务|履约|(合同|服务)履行)(?:期限?|有效期)|(?:服务|履约|(合同|服务)履行)(?:时间|日期|周期)|服务[时年]限|合同周期", entity_left2):
- if re.search("到|至|截[至止]",entity_left) or re.search("前|止|截止",entity_right) or re.search("前",entity_text[-2:]):
- dict_time['time_contractEnd'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_contractEnd'
- else:
- dict_time['time_contractStart'].append((extract_time[0], 0.5, in_attachment))
- last_time_type = 'time_contractStart'
- last_sentence_index = entity.sentence_index
- continue
- else:
- if re.search("(?:合同|服务|履约|(合同|服务)履行)(?:期限?|有效期)|(?:服务|履约|(合同|服务)履行)(?:时间|日期|周期)|服务[时年]限|合同周期", entity_left2):
- # 排除开始和借宿时间一样的错误模板,例:“履约期限:2023年02月15日至2023年02月15日”
- if extract_time[0]!=extract_time[1]:
- dict_time['time_contractStart'].append((extract_time[0], 0.6, in_attachment))
- dict_time['time_contractEnd'].append((extract_time[1], 0.6, in_attachment))
- last_time_type = ''
- last_sentence_index = entity.sentence_index
- continue
- # 服务期限表达补充
- if entity.label==0:
- re_service = '合同期限|工期/交货期/服务期|工期\(交货期\)|合格工期|服务期限|工期' \
- '|工期要求|项目周期|工期\(交货期\)|计划工期\(服务期限\)|服务时限|履行期限|服务周期|供货期限' \
- '|合格工期|计划工期\(服务期\)|服务期|服务,期|交货\(完工\)(时间|日期)|交付\(服务、完工\)(时间|日期)' \
- '|交货(时间|日期)|工期承诺|(服务|合同|施工|实施|工程|设计)的?(年限|期限|周期|期:)' \
- '|服务期限为|计划工期|工期要求|服务期限|服务期' \
- '|投标工期|设计工期|合格服务周期|总工期|服务(时间|日期)(范围)?|流转期限|维护期限|服务时限|交货期' \
- '|完成(时间|日期)|服务期限|中标工期|项目周期|期限要求|供货期|合同履行日期|计划的?周期' \
- '|履约期限|合同约定完成时限|合同完成日期|承诺完成日期' \
- '|合同起始日起|合同履约期|履约截止日期|承包期限|合同完成日期' \
- '|服务期间|服务履行期|委托(管理)?期限|履约期限、地点等简要信息'
- if len(extract_time)==2:
- if re.search(re_service,entity_left2) or re.search("履约期限、地点等简要信息",entity_left3[-20:]):
- dict_time['time_contractStart'].append((extract_time[0], 0.5, in_attachment))
- dict_time['time_contractEnd'].append((extract_time[1], 0.5, in_attachment))
- last_time_type = ''
- # 报价/投标时间补充(规则补充)
- if entity.label == 0:
- if re.search("[报竞]价.{,2}(开始|起始).{,2}(时间|日期)",entity_left2):
- entity.label = 12
- label_prob = 0.8
- elif re.search("[报竞]价.{,2}起止.{,2}(时间|日期)",entity_left2):
- entity.label = 12
- label_prob = 0.6
- elif re.search("响应.{,2}文件([递提]交|接收).{,2}(时间|日期)[::]|([递提]交|接收).{,2}响应.{,2}文件.{,2}(时间|日期)[::]",entity_left2):
- entity.label = 3
- label_prob = 0.501
- elif re.search("响应.{,2}文件([递提]交|接收).{,2}(时间|日期)|([递提]交|接收).{,2}响应.{,2}文件.{,2}(时间|日期)",entity_left2) and not re.search("截[止至]",entity_left2):
- entity.label = 12
- label_prob = 0.51
- elif re.search("[报竞]价.{,2}截[止至].{,2}(时间|日期)",entity_left2):
- entity.label = 3
- label_prob = 0.8
- elif re.search("(竞价|报价).?(时间|日期)",entity_left2):
- entity.label = 12
- label_prob = 0.51
- elif re.search("(竞价|报价).?(时间|日期)",entity_left3) and re.search("参与|报价|有意",entity_left2):
- entity.label = 12
- label_prob = 0.501
- # 文档结构补充
- # if entity.label == 0:
- # re_registration = re.compile("报名|(文件|标书)[\u4e00-\u9fa5、]{,4}(获取|出售|售卖|购买|下载)|"
- # "(获取|出售|售卖|购买|下载)[\u4e00-\u9fa5、]{,4}(文件|标书)")
- # _data_i = -1
- # while _data_i < len(document_tree) - 1:
- # _data_i += 1
- # _data = document_tree[_data_i]
- # _type = _data["type"]
- # _text = _data["text"].strip()
- # childs = get_childs([_data])
- # last_child = childs[-1]
- # if entity.sentence_index>=_data.sentence_index and entity.wordOffset_begin>=_data.wordOffset_begin and
- # ():
- # if re.search(re_registration, re.split("[::;;,]", _text)[0][:20]) is not None:
- #
- # content_text = ""
- # for c in childs:
- # content_text += c["text"] + ""
- # print('concat_text', content_text)
- if re.search("[,;](完成|截止|结束)(时间|日期)", entity_left2[-8:]) and entity.label==0:
- if entity.sentence_index == last_sentence_index:
- time_type = last_time_index.get(last_time_type)
- if time_type:
- dict_time[time_type].append((extract_time[0], 0.5 + label_prob / 10,in_attachment))
- last_time_type = ""
- last_sentence_index = entity.sentence_index
- continue
- if re.search("至|到|[日\d][-—]$|[~~]", entity_left):
- if entity.sentence_index == last_sentence_index:
- time_type = last_time_index.get(last_time_type)
- if time_type:
- dict_time[time_type].append((extract_time[0], 0.5 + label_prob / 10,in_attachment))
- last_time_type = ""
- last_sentence_index = entity.sentence_index
- continue
- if entity.label!=0:
- if entity.label==1 and label_prob>0.5:
- dict_time['time_release'].append((extract_time[0],label_prob,in_attachment))
- last_time_type = 'time_release'
- elif entity.label==2 and label_prob>0.5:
- dict_time['time_bidopen'].append((extract_time[0],label_prob,in_attachment))
- last_time_type = 'time_bidopen'
- elif entity.label==3 and label_prob>0.5:
- if re.search("(资格)?预审文件[\u4e00-\u9fa5]{,4}截止",entity_left3[-15:]):
- # 排除"资格预审文件提交截止时间"
- last_time_type = ''
- continue
- if len(extract_time)==1:
- dict_time['time_bidclose'].append((extract_time[0],label_prob,in_attachment))
- last_time_type = 'time_bidclose'
- elif len(extract_time)==2:
- dict_time['time_bidstart'].append((extract_time[0], 0.6, in_attachment))
- dict_time['time_bidclose'].append((extract_time[1], label_prob, in_attachment))
- last_time_type = 'time_bidclose'
- elif entity.label==12 and label_prob>0.5:
- if len(extract_time)==1:
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_bidclose'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_bidclose'
- else:
- dict_time['time_bidstart'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_bidstart'
- else:
- dict_time['time_bidstart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_bidclose'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==4 and label_prob>0.5:
- if len(extract_time)==1:
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_publicityEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_publicityEnd'
- else:
- dict_time['time_publicityStart'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_publicityStart'
- else:
- dict_time['time_publicityStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_publicityEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==5 and label_prob>0.5:
- if len(extract_time)==1:
- dict_time['time_publicityEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_publicityEnd'
- else:
- dict_time['time_publicityStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_publicityEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==6 and label_prob>0.5:
- if len(extract_time)==1:
- if (re.search("前|截?止",entity_right) and re.search("前|截?止(?!时间|日期)",entity_right2[:len(entity_right)+3])) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_getFileEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_getFileEnd'
- else:
- dict_time['time_getFileStart'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_getFileStart'
- else:
- dict_time['time_getFileStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_getFileEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==7 and label_prob>0.5:
- if len(extract_time)==1:
- dict_time['time_getFileEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_getFileEnd'
- else:
- dict_time['time_getFileStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_getFileEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==8 and label_prob>0.5:
- if len(extract_time)==1:
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_registrationEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_registrationEnd'
- else:
- dict_time['time_registrationStart'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_registrationStart'
- else:
- dict_time['time_registrationStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_registrationEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==9 and label_prob>0.5:
- if len(extract_time)==1:
- dict_time['time_registrationEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_registrationEnd'
- else:
- dict_time['time_registrationStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_registrationEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==10 and label_prob>0.5:
- if len(extract_time)==1:
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_earnestMoneyEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_earnestMoneyEnd'
- else:
- dict_time['time_earnestMoneyStart'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_earnestMoneyStart'
- else:
- dict_time['time_earnestMoneyStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_earnestMoneyEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==11 and label_prob>0.5:
- if len(extract_time)==1:
- dict_time['time_earnestMoneyEnd'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_earnestMoneyEnd'
- else:
- dict_time['time_earnestMoneyStart'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_earnestMoneyEnd'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==13 and label_prob>0.5:
- if len(extract_time)==1:
- if re.search("前|止|截止",entity_right) or re.search("至|止|到",entity_left) or re.search("前",entity_text[-2:]):
- dict_time['time_completion'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_completion'
- else:
- dict_time['time_commencement'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_commencement'
- else:
- dict_time['time_commencement'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_completion'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- elif entity.label==14 and label_prob>0.5:
- if len(extract_time)==1:
- dict_time['time_completion'].append((extract_time[0], label_prob,in_attachment))
- last_time_type = 'time_completion'
- else:
- dict_time['time_commencement'].append((extract_time[0],label_prob,in_attachment))
- dict_time['time_completion'].append((extract_time[1],label_prob,in_attachment))
- last_time_type = ''
- else:
- last_time_type = ""
- else:
- last_time_type = ""
- else:
- last_time_type = ""
- last_sentence_index = entity.sentence_index
- # 通过文档分析树形结构补充部分时间实体
- def add_time_by_parseDocument(dict_time,parse_document):
- from BiddingKG.dl.interface.htmlparser import get_childs
- document_tree = parse_document.tree
- # if not dict_time['time_getFileStart'] or not dict_time['time_getFileEnd']:
- # time_pattern = re.compile("")
- concat_text_list = []
- if not dict_time['time_registrationStart'] or not dict_time['time_registrationEnd']:
- re_registration = re.compile("报名|(文件|标书)[\u4e00-\u9fa5、]{,4}(获取|出售|售卖|购买|下载)|"
- "(获取|出售|售卖|购买|下载)[\u4e00-\u9fa5、]{,4}(文件|标书)")
- _data_i = -1
- while _data_i < len(document_tree) - 1:
- _data_i += 1
- _data = document_tree[_data_i]
- _type = _data["type"]
- _text = _data["text"].strip()
- # print(_data.keys())
- if _type == "sentence":
- print('_text:',_text,_data["sentence_title"])
- if _data["sentence_title"] is not None:
- print("aptitude_pattern", _text)
- print(_data['sentence_index'],_data['wordOffset_begin'],_data['wordOffset_end'])
- if re.search(re_registration, re.split("[::;;。]",_text)[0][:15]) is not None:
- childs = get_childs([_data])
- concat_text = ""
- for c in childs:
- concat_text += c["text"] + ""
- print('concat_text',concat_text)
- concat_text_list.append(concat_text)
- _data_i += len(childs)-1
- # if _type == "table":
- # list_table = _data["list_table"]
- # parent_title = _data["parent_title"]
- # if list_table is not None:
- # for line in list_table[:2]:
- # for cell_i in range(len(line)):
- # cell = line[cell_i]
- # cell_text = cell[0]
- # if len(cell_text) > 120 and re.search(re_registration, cell_text) is not None:
- # concat_text += cell_text + "\n"
- print('_text',concat_text_list)
- for text in concat_text_list:
- time_list = re.finditer(my_time_format_pattern,text)
- time_list = [(i,my_timeFormat(i.group(),page_time)) for i in time_list]
- for time_idx in range(len(time_list)):
- _time = time_list[time_idx][0]
- extract_time = time_list[time_idx][1]
- entity_left = text[:_time.start()]
- entity_left = re.split("[。;;!??]",entity_left)[-1]
- # entity_left2 = sentence_text[
- # max(entity_context_begin, entity.wordOffset_begin - 10):entity.wordOffset_begin]
- # entity_left3 = sentence_text[
- # max(entity_context_begin, entity.wordOffset_begin - 30):entity.wordOffset_begin]
- entity_right = text[_time.end():]
- entity_right = re.split("[。;;!??]",entity_right)[0]
- # entity_right2 = sentence_text[entity.wordOffset_end:entity_context_end]
- entity_right2 = re.sub(r"(http[s]?://)?(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F])){6,}",
- '', entity_right)[:60] # 去除网址
- print('entity_right2',entity_right2)
- if re.search("(进行|在线|线下).{,2}报名", entity_right2):
- print('报名text',entity_right2)
- if len(extract_time) == 2:
- dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- dict_time['time_registrationEnd'].append((extract_time[1], 0.51, in_attachment))
- else:
- if previous_entity and previous_entity.sentence_index==entity.sentence_index:
- mid_text = sentence_text[previous_entity.wordOffset_end:entity.wordOffset_begin]
- if len(mid_text)<=10 and re.search("至|到|[-—]|[~~]",mid_text) and len(previous_extract_time)==1:
- dict_time['time_registrationStart'].append((previous_extract_time[0], 0.51, in_attachment))
- dict_time['time_registrationEnd'].append((extract_time[0], 0.51, in_attachment))
- if not dict_time['time_registrationEnd']:
- if re.search("前|止|截止", entity_right) or re.search("前",entity_text[-2:]):
- dict_time['time_registrationEnd'].append((extract_time[0], 0.51, in_attachment))
- elif re.search("起|开?始", entity_right) or re.search("起",entity_text[-2:]):
- dict_time['time_registrationStart'].append((extract_time[0], 0.51, in_attachment))
- return dict_time
- # dict_time = add_time_by_parseDocument(dict_time,parse_document)
- # print(dict_time)
- result_dict = dict((key,"") for key in dict_time.keys())
- for time_type,value in dict_time.items():
- list_time = dict_time[time_type]
- if list_time:
- for in_attachment in [False,True]:
- _list_time = [_time for _time in list_time if _time[2]==in_attachment]
- if time_type in ['time_getFileEnd','time_registrationEnd','time_bidclose','time_bidopen']:
- # 过滤与page_time差距过大的时间(需加上公告类别判断[52,101,114])
- # print(time_type,_list_time,timeAdd(page_time,-7))
- tmp_list_time = []
- # for add_day in [ 1, 0, -7, -15, -31]:
- for add_day in [ 1, 0, -7, -15, -31, -60, -90]:
- tmp_list_time = [_time for _time in list_time if _time[0][:10] >= timeAdd(page_time,add_day) and _time[0][:10] < timeAdd(page_time,180)]
- if tmp_list_time:
- break
- _list_time = tmp_list_time
- else:
- if time_type not in ['time_contractEnd','time_completion']:
- _list_time = [_time for _time in list_time if _time[0][:10] >= timeAdd(page_time, -180) and _time[0][:10] < timeAdd(page_time,180)]
- else:
- _list_time = [_time for _time in list_time if _time[0][:10] >= timeAdd(page_time, -180)]
- if _list_time:
- _list_time.sort(key=lambda x:(x[1],len(x[0])),reverse=True) # sort_key: label_prob,时间文本长度(优先有具体时分秒的)
- if in_attachment==True and len(result_dict[time_type])>0:
- break
- result_dict[time_type] = _list_time[0][0]
- # print('time result_dict',result_dict)
- # result_dict 纠错
- if not result_dict['time_bidclose']:
- if result_dict['time_bidstart']: # 无截标时间,投标开始和开标时间一样
- if result_dict['time_bidstart'][:10] in result_dict['time_bidopen']:
- result_dict['time_bidstart'] = ""
- result_dict['time_bidclose'] = result_dict['time_bidopen']
- if not result_dict['time_bidclose']:
- if result_dict['time_getFileEnd']: # 无截标时间,获取文件截止时间和开标时间一样
- if result_dict['time_getFileEnd'][:10] in result_dict['time_bidopen']:
- result_dict['time_bidclose'] = result_dict['time_bidopen']
- else:
- if result_dict['time_bidopen']: # 截标时间 和 开标时间 时分秒互补
- if len(result_dict['time_bidclose'])<len(result_dict['time_bidopen']) and result_dict['time_bidclose'] in result_dict['time_bidopen']:
- result_dict['time_bidclose'] = result_dict['time_bidopen']
- elif len(result_dict['time_bidclose'])>len(result_dict['time_bidopen']) and result_dict['time_bidopen'] in result_dict['time_bidclose']:
- result_dict['time_bidopen'] = result_dict['time_bidclose']
- # 截标时间与开标时间差距过大时,互补修复
- if result_dict['time_bidopen'][:10]>=page_time:
- if result_dict['time_bidclose'][:10] < timeAdd(result_dict['time_bidopen'][:10], -7) and len(dict_time['time_bidclose'])>0:
- result_dict['time_bidclose'] = result_dict['time_bidopen'][:10]
- if result_dict['time_bidclose'][:10]>=page_time:
- if result_dict['time_bidopen'][:10] < timeAdd(result_dict['time_bidclose'][:10], -7) and len(dict_time['time_bidopen'])>0:
- result_dict['time_bidopen'] = result_dict['time_bidclose'][:10]
- # 开始结束对应时间纠错
- for begin_time_type,end_time_type in last_time_index.items():
- if result_dict[end_time_type] and result_dict[end_time_type][:10] < result_dict[begin_time_type][:10]:
- # print('error time',end_time_type,result_dict[end_time_type])
- result_dict[end_time_type] = ""
- return result_dict
- def getOtherAttributes(list_entity,page_time,prem,channel_dic):
- dict_other = {"moneysource":"",
- "person_review":[],
- "serviceTime":"",
- "product":[],
- "total_tendereeMoney":0,
- "total_tendereeMoneyUnit":''}
- list_serviceTime = []
- last_moneysource_prob = 0
- for entity in list_entity:
- if entity.entity_type == 'bidway':
- dict_other["bidway"] = turnBidWay(entity.entity_text)
- elif entity.entity_type=='moneysource':
- if dict_other["moneysource"] and entity.in_attachment:
- continue
- if not dict_other["moneysource"]:
- dict_other["moneysource"] = entity.entity_text
- last_moneysource_prob = entity.prob
- elif entity.prob>last_moneysource_prob:
- dict_other["moneysource"] = entity.entity_text
- last_moneysource_prob = entity.prob
- elif entity.entity_type=='serviceTime':
- # print(entity.entity_text)
- # if list_serviceTime and entity.in_attachment:
- # continue
- if re.search("[^之]日|天|年|月|周|星期", entity.entity_text) or re.search("\d{4}[-./]\d{1,2}", entity.entity_text):
- list_serviceTime.append(entity)
- elif entity.entity_type=="person" and entity.label ==4 and entity.entity_text not in dict_other["person_review"]: # 20240624评审专家去重
- dict_other["person_review"].append(entity.entity_text)
- elif entity.entity_type=='product' and entity.entity_text not in dict_other["product"]: #顺序去重保留
- dict_other["product"].append(entity.entity_text)
- elif entity.entity_type=='money' and entity.notes=='总投资' and float(entity.entity_text)>5000000 and float(dict_other["total_tendereeMoney"])<float(entity.entity_text): # 20250702 限制总投资为500万的以上
- dict_other["total_tendereeMoney"] = str(Decimal(entity.entity_text))
- dict_other["total_tendereeMoneyUnit"] = entity.money_unit
- time_contractEnd = prem[0].get("time_contractEnd","")[:10]
- time_contractStart = prem[0].get("time_contractStart","")[:10]
- serviceTime_dict = {"service_start":"", "service_end":"", "service_days": 0}
- if time_contractEnd:
- serviceTime_dict['service_end'] = time_contractEnd
- if time_contractStart:
- if get_days_between(time_contractStart,time_contractEnd)>0:
- serviceTime_dict['service_start'] = time_contractStart
- # print([i.entity_text for i in list_serviceTime])
- if list_serviceTime and not serviceTime_dict['service_end']:
- list_serviceTime_inAtt = [serviceTime for serviceTime in list_serviceTime if serviceTime.in_attachment==1]
- list_serviceTime = [serviceTime for serviceTime in list_serviceTime if serviceTime.in_attachment==0]
- error_serviceTime = []
- for list_time in [list_serviceTime,list_serviceTime_inAtt]:
- if not serviceTime_dict['service_end'] and not serviceTime_dict['service_days']:
- list_time.sort(key=lambda x: (x.prob,-x.sentence_index,-x.begin_index), reverse=True)
- for _serviceTime in list_time:
- # 优先取具体时间(20XX年x月x日-20XX年x月x日)
- if re.search("20\d{2}[年/.\-]\d{1,2}[月/.\-]\d{1,2}日?[^。\d半一二三四五六七八九十壹两叁贰肆伍陆柒捌玖拾;;]{,4}20\d{2}[年/.\-]\d{1,2}[月/.\-]\d{1,2}日?",_serviceTime.entity_text):
- _extract_time,_ = my_timeFormat(_serviceTime.entity_text,page_time)
- if _extract_time and len(_extract_time)==2:
- # 排除开始和结束时间一样的错误模板,例:“履约期限:2023年02月15日至2023年02月15日”
- if _extract_time[0]!=_extract_time[1]:
- # dict_other["serviceTime"] = _serviceTime.entity_text
- # extract_time = extract_serviceTime(_serviceTime.entity_text)
- # if extract_time['service_end']:
- serviceTime_dict['service_start'] = _extract_time[0]
- serviceTime_dict['service_end'] = _extract_time[1]
- break
- else:
- error_serviceTime.append(_serviceTime.entity_text)
- if not serviceTime_dict['service_end']:
- for _serviceTime in list_time:
- # 优先取具体时间(20XX年x月-20XX年x月)
- if re.search("20\d{2}[年/.\-]\d{1,2}月?[^。\d半一二三四五六七八九十壹两叁贰肆伍陆柒捌玖拾;;]{,3}20\d{2}[年/.\-]\d{1,2}月?", _serviceTime.entity_text):
- # dict_other["serviceTime"] = _serviceTime.entity_text
- extract_time = extract_serviceTime(_serviceTime.entity_text,page_time)
- if extract_time['service_end']:
- serviceTime_dict = extract_time
- break
- if not serviceTime_dict['service_end']:
- for _serviceTime in list_time:
- # 优先取具体时间(20XX年x月x日)
- if re.search("20\d{2}[年/.\-]\d{1,2}[月/.\-]\d{1,2}日?",_serviceTime.entity_text):
- if _serviceTime.entity_text not in error_serviceTime:
- # dict_other["serviceTime"] = _serviceTime.entity_text
- extract_time = extract_serviceTime(_serviceTime.entity_text,page_time)
- if extract_time['service_end']:
- serviceTime_dict = extract_time
- break
- if not serviceTime_dict['service_end'] and not serviceTime_dict['service_days']:
- for _serviceTime in list_time:
- if _serviceTime.entity_text not in error_serviceTime:
- # dict_other["serviceTime"] = _serviceTime.entity_text
- extract_time = extract_serviceTime(_serviceTime.entity_text,page_time)
- # service_days > 3
- if extract_time['service_end'] or extract_time['service_days']>3:
- serviceTime_dict = extract_time
- break
- # 若上一步仍无结果,取消service_days > 3 的条件
- if not serviceTime_dict['service_end'] and not serviceTime_dict['service_days']:
- for _serviceTime in list_time:
- if _serviceTime.entity_text not in error_serviceTime:
- # dict_other["serviceTime"] = _serviceTime.entity_text
- extract_time = extract_serviceTime(_serviceTime.entity_text,page_time)
- if extract_time['service_end'] or extract_time['service_days']:
- serviceTime_dict = extract_time
- break
- if serviceTime_dict['service_start'] and serviceTime_dict['service_end']:
- service_days = get_days_between(serviceTime_dict['service_start'],serviceTime_dict['service_end'])
- serviceTime_dict['service_days'] = service_days
- dict_other["serviceTime"] = serviceTime_dict
- if not time_contractEnd and channel_dic['docchannel']['docchannel']=='合同公告': # 用serviceTime补充合同开始结束时间,公告类型为合同公告
- if serviceTime_dict['service_start'] and serviceTime_dict['service_end']:
- prem[0]["time_contractStart"] = serviceTime_dict['service_start']
- prem[0]["time_contractEnd"] = serviceTime_dict['service_end']
- if dict_other['moneysource']:
- dict_other['moneysource'] = turnMoneySource(dict_other['moneysource'])
- # dict_other["product"] = list(set(dict_other["product"])) # 已在添加时 顺序去重保留
- return dict_other
|