Merge remote-tracking branch 'origin/master'

BiddingKG/dl/common/Utils.py

@@ -801,8 +801,8 @@ def uniform_package_name(package_name):
     name = ""
     if kw:
         name += kw.group(0)
-    if re.search('[a-zA-Z0-9-]{5,}$', package_name):   # 五个字符以上编号
-        _digit = re.search('[a-zA-Z0-9-]{5,}$', package_name).group(0).upper()
+    if re.search('^[a-zA-Z0-9-]{5,}$', package_name):   # 五个字符以上编号
+        _digit = re.search('^[a-zA-Z0-9-]{5,}$', package_name).group(0).upper()
         name += _digit
     elif re.search('(?P<eng>[a-zA-Z])包[：）]?第?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))标段?', package_name): # 处理类似 A包2标段
         ser = re.search('(?P<eng>[a-zA-Z])包[：）]?第?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))标段?', package_name)
@@ -842,6 +842,9 @@ def uniform_package_name(package_name):
         _digit = re.search('^([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4})$', package_name).group(0)
         _digit = uniform_num(_digit)
         name += _digit
+    elif re.search('^[a-zA-Z0-9-]+$', package_name):
+        _char = re.search('^[a-zA-Z0-9-]+$', package_name).group(0)
+        name += _char.upper()
     if name == "":
         return package_name_raw
     else:

BiddingKG/dl/interface/Preprocessing.py

@@ -15,7 +15,7 @@ sys.path.append(os.path.abspath("../.."))
 sys.path.append(os.path.abspath(".."))
 from BiddingKG.dl.common.Utils import *
 from BiddingKG.dl.interface.Entitys import *
-from BiddingKG.dl.interface.predictor import getPredictor
+from BiddingKG.dl.interface.predictor import getPredictor, TableTag2List
 from BiddingKG.dl.common.nerUtils import *
 from BiddingKG.dl.money.moneySource.ruleExtra import extract_moneySource
 from BiddingKG.dl.time.re_servicetime import extract_servicetime
@@ -128,6 +128,9 @@ def tableToText(soup):
         for item in inner_table:
             if len(item)>maxWidth:
                 maxWidth = len(item)
+        if maxWidth > 100:
+            # log('表格列数大于100，表格异常不做处理。')
+            return []
         for i in range(len(inner_table)):
             if len(inner_table[i])<maxWidth:
                 for j in range(maxWidth-len(inner_table[i])):
@@ -1046,10 +1049,35 @@ def tableToText(soup):
             if _td_len_list:
                 if len(list(set(_td_len_list))) >= 8 or max(_td_len_list) > 100:
                     return None
-        fixSpan(tbody)
-        inner_table = getTable(tbody)
+
+        # fixSpan(tbody)
+        # inner_table = getTable(tbody)
+        # inner_table = fixTable(inner_table)
+
+        table2list = TableTag2List()
+        inner_table = table2list.table2list(tbody, segment)
         inner_table = fixTable(inner_table)
+
+        if inner_table == []:
+            tbody.string = segment(tbody,final=False)
+            table_max_len = 30000
+            tbody.string = tbody.string[:table_max_len]
+            # log('异常表格直接取全文')
+            tbody.name = "turntable"
+            return None
+
         if len(inner_table)>0 and len(inner_table[0])>0:
+
+            for tr in inner_table:
+                for td in tr:
+                    if isinstance(td, str):
+                        tbody.string = segment(tbody,final=False)
+                        table_max_len = 30000
+                        tbody.string = tbody.string[:table_max_len]
+                        # log('异常表格，不做表格处理，直接取全文')
+                        tbody.name = "turntable"
+                        return None
+
             #inner_table,head_list = setHead_withRule(inner_table,pat_head,pat_value,3)
             #inner_table,head_list = setHead_inline(inner_table)
             # inner_table, head_list = setHead_initem(inner_table,pat_head)
@@ -1069,7 +1097,6 @@ def tableToText(soup):
             # for item in inner_table:
             #     print(item)
 
-
             tbody.string = getTableText(inner_table,head_list)
             table_max_len = 30000
             tbody.string = tbody.string[:table_max_len]
@@ -1311,12 +1338,12 @@ def segment(soup,final=True):
     # 感叹号替换为中文句号
     text = re.sub("(?<=[\u4e00-\u9fa5])[!！]|[!！](?=[\u4e00-\u9fa5])","。",text)
     #替换格式未识别的问号为" " ,update:2021/7/20
-    text = re.sub("[？\?]{2,}"," ",text)
+    text = re.sub("[？\?]{2,}|\n"," ",text)
 
 
     #替换"""为"“",否则导入deepdive出错
     # text = text.replace('"',"“").replace("\r","").replace("\n","，")
-    text = text.replace('"',"“").replace("\r","").replace("\n","")  #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
+    text = text.replace('"',"“").replace("\r","").replace("\n","").replace("\\n","") #2022/1/4修复 非分段\n 替换为逗号造成 公司拆分 span \n南航\n上海\n分公司
     # print('==1',text)
     # text = re.sub("\s{4,}","，",text)
     # 解决公告中的" "空格替换问题

BiddingKG/dl/interface/extract.py

@@ -202,10 +202,11 @@ def predict(doc_id,text,title="",page_time="",web_source_no='',web_source_name="
     '''表格要素提取'''
     table_prem = predictor.getPredictor("tableprem").predict(text)
     if table_prem:
-        if 'Project' in prem[0]['prem']:
-            table_prem.update({'Project': prem[0]['prem']['Project']})
-        prem[0]['prem'] = table_prem
+        getAttributes.update_prem(old_prem=prem[0]['prem'], new_prem=table_prem)
 
+    '''候选人提取'''
+    candidate_top3_prem, candidate_dic = predictor.getPredictor("candidate").predict(text, list_sentences, list_entitys)
+    getAttributes.update_prem(old_prem=prem[0]['prem'], new_prem=candidate_top3_prem)
 
     '''获取联合体信息'''
     getAttributes.get_win_joint(prem, list_entitys, list_sentences, list_articles)
@@ -253,7 +254,8 @@ def predict(doc_id,text,title="",page_time="",web_source_no='',web_source_name="
 
     # data_res = Preprocessing.union_result(Preprocessing.union_result(codeName, prem),list_punish_dic)[0]
     # data_res = Preprocessing.union_result(Preprocessing.union_result(Preprocessing.union_result(codeName, prem),list_punish_dic), list_channel_dic)[0]
-    data_res = dict(codeName[0], **prem[0], **channel_dic, **product_attrs[0], **product_attrs[1], **payment_way_dic, **fail_reason, **industry, **district)
+    version_date = {'version_date': '2022-11-24'}
+    data_res = dict(codeName[0], **prem[0], **channel_dic, **product_attrs[0], **product_attrs[1], **payment_way_dic, **fail_reason, **industry, **district, **candidate_dic, **version_date)
     data_res["doctitle_refine"] = doctitle_refine
     data_res["nlp_enterprise"] = nlp_enterprise
     data_res["nlp_enterprise_attachment"] = nlp_enterprise_attachment

BiddingKG/dl/interface/getAttributes.py

@@ -870,10 +870,15 @@ def getPackagesFromArticle(list_sentence, list_entity):
             tokens = list_sentence[i].tokens
             _names = []
             for iter in re.finditer(package_number_pattern, content):
-                if re.match('\d', iter.group(0)) and iter.end() < len(content) and content[
-                    iter.end()].isdigit():  # 排除2.10标段3 这种情况
+                # print('提取到标段：%s， 前后文：%s'%(iter.group(), content[iter.start()-5:iter.end()+5]))
+                if re.match('\d', iter.group(0)) and re.search('\d.$', content[:iter.start()]):  # 排除2.10标段3  5.4标段划分 这种情况
+                    # print('过滤掉错误包：', iter.group())
                     continue
                 if re.search('[承每书/]包|XX|xx', iter.group(0)) or re.search('[a-zA-Z0-9一二三四五六七八九十ⅠⅡⅢⅣⅤⅥⅦ-]{6,}', iter.group(0)):
+                    # print('过滤掉错误包：', iter.group())
+                    continue
+                elif iter.end()+2 < len(content) and  re.search('标准|标的物|标志|包装|划分', content[iter.start():iter.end()+2]):
+                    # print('过滤掉错误包：',iter.group())
                     continue
                 temp_package_number = uniform_package_name(iter.group(0))
                 True_package.add(temp_package_number)
@@ -3370,6 +3375,52 @@ def get_win_joint(prem, list_entitys, list_sentences, list_articles):
     except Exception as e:
         print('获取联合体抛出异常', e)
 
+def update_prem(old_prem, new_prem):
+    '''
+    根据新旧对比，更新数据
+    :param old_prem:
+    :param new_prem: 表格提取的要素
+    :return:
+    '''
+    if len(new_prem) >= 1 :
+        '''如果表格提取的包大于2，原来的包比表格提取的包多则删除原来多余的包，以表格的为准'''
+        if len(new_prem) > 2 and len(old_prem) > len(new_prem):
+            del_k = []
+            for k in old_prem:
+                if k not in new_prem and k != 'Project':
+                    del_k.append(k)
+            for k in del_k:
+                old_prem.pop(k)
+
+        for k, v in new_prem.items():
+            if k == 'Project':
+                if 'Project' in old_prem:
+                    for d in old_prem['Project']['roleList']:
+                        for d2 in v['roleList']:
+                            if d['role_name'] == d2['role_name']:
+                                d['role_text'] = d2['role_text']
+                                d['role_money']['money'] = d2['role_money']['money']
+                                d['role_money']['money_unit'] = d2['role_money']['money_unit']
+                                v['roleList'].remove(d2)
+                    for d2 in v['roleList']:
+                        old_prem['Project']['roleList'].append(d2)
+                else:
+                    old_prem[k] = v
+            else:
+                if k not in old_prem:
+                    old_prem[k] = v
+                else:
+                    for d in old_prem[k]['roleList']:
+                        for d2 in v['roleList']:
+                            if d['role_name'] == d2['role_name']:
+                                d['role_text'] = d2['role_text']
+                                d['role_money']['money'] = d2['role_money']['money']
+                                d['role_money']['money_unit'] = d2['role_money']['money_unit']
+                                v['roleList'].remove(d2)
+                    for d2 in v['roleList']:
+                        old_prem[k]['roleList'].append(d2)
+
+    # return old_prem
 
 if __name__=="__main__":
     '''

BiddingKG/dl/interface/predictor.py

@@ -55,6 +55,7 @@ dict_predictor = {"codeName":{"predictor":None,"Lock":RLock()},
                   "moneygrade": {"predictor": None, "Lock": RLock()},
                   "district": {"predictor": None, "Lock": RLock()},
                   'tableprem': {"predictor": None, "Lock": RLock()},
+                  'candidate': {"predictor": None, "Lock": RLock()},
                   }
 
 
@@ -100,6 +101,8 @@ def getPredictor(_type):
                     dict_predictor[_type]["predictor"] = DistrictPredictor()
                 if _type == 'tableprem':
                     dict_predictor[_type]["predictor"] = TablePremExtractor()
+                if _type == 'candidate':
+                    dict_predictor[_type]["predictor"] = CandidateExtractor()
             return dict_predictor[_type]["predictor"]
     raise NameError("no this type of predictor")
 
@@ -678,7 +681,9 @@ class PREMPredict():
             label = np.argmax(predict_y[i])
             values = predict_y[i]
             text = text_list[i]
-            if label == 2:
+            if label in [0, 1, 2, 3, 4] and values[label] < 0.5: # 小于阈值的设为其他，让后面的规则召回重新判断
+                label = 5
+            elif label == 2:
                 if re.search('中标单位和.{,25}签订合同', text):
                     label = 0
                     values[label] = 0.501
@@ -735,10 +740,14 @@ class PREMPredict():
             label = np.argmax(predict_y[i])
             values = predict_y[i]
             text = text_list[i]
-            if label == 1 and re.search('[:：，。](总金额|总价|单价)', text):
+            if label in [0, 1] and values[label] < 0.5: # 小于阈值的设为其他金额，让后面的规则召回重新判断
+                label = 2
+            elif label == 1 and re.search('[:：，。](总金额|总价|单价)', text):
                 values[label] = 0.49
             elif label ==0 and entity.notes in ["投资", "工程造价"]:
                 values[label] = 0.49
+            elif label == 0 and re.search('最低限价', text):
+                values[label] = 0.49
             entity.set_Money(label, values)
 
     def correct_money_by_rule(self, title, list_entitys, list_articles):
@@ -1196,7 +1205,7 @@ class RoleRulePredictor():
 
         self.SET_NOT_TENDERER = set(["人民政府","人民法院","中华人民共和国","人民检察院","评标委员会","中国政府","中国海关","中华人民共和国政府"])
         
-        self.pattern_money_tenderee = re.compile("投标最高限价|采购计划金额|项目预算|招标金额|采购金额|项目金额|建安费用|投资估算|采购(单位|人)委托价|限价|拦标价|预算金额|标底|总计|限额")
+        self.pattern_money_tenderee = re.compile("投?标?最高限价|采购计划金额|项目预算|招标金额|采购金额|项目金额|建安费用|投资估算|采购(单位|人)委托价|招标限价|拦标价|预算金额|标底|总计|限额|资金来源为\w{2,4}资金")
         self.pattern_money_tenderer = re.compile("((合同|成交|中标|应付款|交易|投标|验收|订单)[）\)]?(总?金额|结果|[单报]?价))|总价|标的基本情况|承包价")
         self.pattern_money_tenderer_whole = re.compile("(以金额.*中标)|中标供应商.*单价|以.*元中标")
         self.pattern_money_other = re.compile("代理费|服务费")
@@ -1394,6 +1403,10 @@ class RoleRulePredictor():
                                                                                        _span[0]) is None:
                                     p_entity.values[1] = 0.8 + p_entity.values[1] / 10
                                     p_entity.label = 1
+                                elif re.search('(预算金额|最高(投标)?上?限[价额]?格?|招标控制价)）?：?([\d.,]+万?元[，（]其中)?(第?[一二三四五0-9](标[段|包]|[分子]包)：?[\d.,]+万?元，)*第?[一二三四五0-9](标[段|包]|[分子]包)：?$'
+                                        , _sentence.sentence_text[:p_entity.wordOffset_begin]): # 处理几个标段金额相邻情况 例子：191705231
+                                    p_entity.values[0] = 0.8 + p_entity.values[0] / 10
+                                    p_entity.label = 0
 
             # 增加招标金额扩展，招标金额+连续的未识别金额，并且都可以匹配到标段信息，则将为识别的金额设置为招标金额
             list_p = []
@@ -2964,27 +2977,29 @@ class DocChannel():
       self.life_dic = {
           '采购意向': '采购意向|招标意向|选取意向|意向公告|意向公示',
           '招标预告': '(预计|计划)(采购|招标)(时间|日期)|采购(计划编号|需求方案|预告|预案)|(预|需求)公示|需求(方案|信息|论证|公告|公示)',
-          '招标公告': '(采购|招标|竞选|报名)条件|报名(时间|流程|方法|要求|\w{,5}材料)[：\s]|参加竞价采购交易资格|(申请人|投标人|供应商|报价人|参选人)的?资格要求|获取(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答)文件|(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答)文件的?(获取|领取)',
+          '招标公告': '(采购|招标|竞选|报名)条件|报名(时间|流程|方法|要求|\w{,5}材料)[：\s]|[^\w]成交规则|参加竞价采购交易资格|(申请人|投标人|供应商|报价人|参选人)的?资格要求|获取(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答)文件|(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答)文件的?(获取|领取)',
           '资审结果': '资审及业绩公示|资审结果及业绩|资格后审情况报告|资格(后审|预审|审查)结果(公告|公示)|(预审|审查)工作已经?结束|未通过原因', #|资格
           '招标答疑': '现澄清(为|如下)|答疑补遗|澄清内容如下|第[0-9一二三四五]次澄清|答疑澄清|(最高(投标)?限价|控制价|拦标价)公示',  # |异议的回复
           '公告变更': '第[\d一二]次变更|(更正|变更)(公告|公示|信息|内容|事项|原因|理由|日期|时间|如下)|原公告((主要)?(信息|内容)|发布时间)|(变更|更正)[前后]内容|现?在?(变更|更正|修改|更改)(内容)?为|(公告|如下|信息|内容|事项|结果|文件|发布|时间|日期)(更正|变更)',
+          '公告变更neg': '履约变更内容',
           '候选人公示': '候选人公示|评标结果公示|中标候选人名单公示',
-          '中标信息': '供地结果信息|采用单源直接采购的?情况说明|[特现]?将\w{,4}(成交|中标|中选|选定结果|选取结果|入围结果)\w{,4}(进行公示|公[示布]如下)|(中标|中选)(供应商|承包商|候选人|入围单位)如下|拟定供应商的情况|((中标|中选)(候选人|人|成交)|成交)\w{,3}(信息|情况)[：:\s]',
+          '候选人公示neg': '中标候选人公示期',
+          '中标信息': '供地结果信息|采用单源直接采购的?情况说明|[特现]?将\w{,4}(成交|中标|中选|选定结果|选取结果|入围结果|竞价结果)\w{,4}(进行公示|公[示布]如下)|(询价|竞价|遴选)(成交|中标|中选)(公告|公示)|(成交|中标|中选|选定|选取|入围|询价)结果(如下|公告|公示)|(中标|中选)(供应商|承包商|候选人|入围单位)如下|拟定供应商的情况|((中标|中选)(候选人|人|成交)|成交)\w{,3}(信息|情况)[：:\s]',
           '中标信息2': '\s(成交|中标|中选)(信息|日期|时间|总?金额|价格)[：:\s]|(采购|招标|成交|中标|中选|评标)结果|单一来源采购原因|拟采取单一来源方式采购|单一来源采购公示',
           '中标信息3': '(中标|中选|成交|拟定|拟选用|最终选定的?|受让|唯一)(供应商|供货商|服务商|机构|企业|公司|单位|候选人|人)(名称)?[：:\s]|[、\s](第一名|(拟定|推荐|入围)?(供应商|供货商)|(中选|中标|供货)单位|中选人)[：:\s]',
-          '中标信息neg': '按项目控制价下浮\d%即为成交价|成交原则|不得确定为(中标|成交)|招标人按下列原则选择中标人|评选成交供应商：|拟邀请供应商|除单一来源采购项目外|单一来源除外|(各.{,5}|尊敬的)(供应商|供货商)[：\s]|竞拍起止时间：|询价结果[\s\n:：]*不公开|本项目已具备招标条件|现对该项目进行招标公告|发布\w{2}结果后\d天内送达|本次\w{2}结果不对外公示|供应商\s*资格要求|成交情况：\s*[流废]标',
+          '中标信息neg': '按项目控制价下浮\d%即为成交价|成交原则|不得确定为(中标|成交)|招标人按下列原则选择中标人|评选成交供应商：|拟邀请供应商|除单一来源采购项目外|单一来源除外|(各.{,5}|尊敬的)(供应商|供货商)[：\s]|竞拍起止时间：|询价结果[\s\n:：]*不公开|本项目已具备招标条件|现对该项目进行招标公告|发布\w{2}结果后\d天内送达|本次\w{2}结果不对外公示|供应商\s*资格要求|成交情况：\s*[流废]标|中标单位：本次招标拟?中标单位\d家',
       # |确定成交供应商[：，\s]
-          '合同公告': '合同(公告|公示|信息|内容)|合同(编号|名称|主体|基本情况|签订日期)|(供应商乙方|乙方供应商)：|合同总?金额',
-          '废标公告': '(终止|中止|废标|流标|失败|作废|异常|撤销)(结果)?(公告|公示|招标|采购|竞价)|(谈判结果为|结果类型)：?废标|((本|该)(项目|标段|合同|合同包|采购包|次)\w{,5})((失败|终止|流标|废标)|予以废标|(按|做|作)?(流标|废标)处理)|(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答|项目)(终止|中止|废标|流标|失败|作废|异常|撤销)',
-          '废标公告2': '(无效|中止|终止|废标|流标|失败|作废|异常|撤销)的?(原因|理由)|本项目因故取消|本(项目|次)(公开)?\w{2}失败|已终止\s*原因：|(人|人数|供应商|单位)(不足|未达\w{,3}数量)|已终止|不足[3三]家|无（废标）|成交情况：\s*[流废]标',
-          '废标公告neg': '超过此报价将作为[废流]标处理|否则按[废流]标处理|终止规则：|视为流标'
+          '合同公告': '合同(公告|公示|信息|内容)|合同(编号|名称|主体|基本情况|完成(日期|时间))|(供应商乙方|乙方供应商)：|合同总?金额|履约信息',
+          '废标公告': '(终止|中止|废标|流标|失败|作废|异常|撤销)(结果)?(公告|公示|招标|采购|竞价)|(谈判结果为|结果类型)：?废标|((本|该)(项目|标段|合同|合同包|采购包|次)\w{,5})((失败|终止|流标|废标)|予以废标|(按|做|作)?(流标|废标|废置)处理)|(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|竞谈|应答|项目)(终止|中止|废标|流标|失败|作废|异常|撤销)',
+          '废标公告2': '(无效|中止|终止|废标|流标|失败|作废|异常|撤销)的?(原因|理由)|本项目因故取消|本(项目|次)(公开)?\w{2}失败|已终止\s*原因：|(人|人数|供应商|单位)(不足|未达\w{,3}数量)|已终止|不足[3三]家|无（废标）|成交情况：\s*[流废]标|现予以废置',
+          '废标公告neg': '超过此报价将作为[废流]标处理|否则按[废流]标处理|终止规则：|成交规则：|视为流标'
       }
       self.title_life_dic = {
           '采购意向': '采购意向|招标意向|选取意向|意向公告|意向公示|意向公开',
           '招标预告': '预公?告|预公示|报建公告|(批前|标前)公示|(供应|招标)计划表?$|(论证|征求|征集)(供应商)?意见|意见征询|需求评审公告|需求(公告|公示|意见)',
           '公告变更': '第[\d一二]次变更|(变更|更正(事项)?|更改|延期|暂停)(招标|采购)?的?(公告|公示|通知)|变更$|更正$',
           '招标答疑': '质疑|澄清|答疑(文件)?|补遗书?|(最高(投标)?限价|控制价|拦标价)(公示|公告|$)',
-          '废标公告': '(终止|中止|废标|废除|流标|失败|作废|异常|撤销|取消成?交?|流拍)(结果|竞价|项目)?的?(公告|公示|$)|(终止|中止)(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|拍卖|招租|交易|出让)',
+          '废标公告': '(终止|中止|废标|废除|废置|流标|失败|作废|异常|撤销|取消成?交?|流拍)(结果|竞价|项目)?的?(公告|公示|$)|(终止|中止)(采购|招标|询价|议价|竞价|比价|比选|遴选|邀请|邀标|磋商|洽谈|约谈|谈判|拍卖|招租|交易|出让)|关于废置',
           '合同公告': '(合同(成交|变更)?|(履约|验收)(结果)?)(公告|公示|信息|公式|公开|签订)|合同备案|合同书|合同$',
           '候选人公示': '候选人(变更)?公示|评标(结果)?公示|中标前?公示|中标预公示',
           '中标信息': '(中标|中选|中价|中租|成交|入选|确认)(候选人|人|供应商|记录|结果|变更)?(公告|公示|结果)|未?入围(公示|公告)|(遴选|采购|招标|竞价|议价|比选|询比?价|评选|谈判|邀标|邀请|洽谈|约谈|评标|发包|遴选|交易)\w{,2}结果|开标(记录|信息|情况)|单一来源|中标通知书|中标$',
@@ -3305,10 +3320,9 @@ class DocChannel():
       def get_life(title, text):
           title = re.sub('[-()（）0-9a-z]|第?[二三四]次公?告?', '', title)
           first_line = text.split()[0] if len(text.split()) > 2 else ''
-          if title.strip()[-2:] not in ['公告', '公示'] and 5 < len(first_line) < 30 and first_line[-2:] in ['公告', '公示']:
+          if title.strip()[-2:] not in ['公告', '公示'] and 5 < len(first_line) < 50 and first_line[-2:] in ['公告', '公示']:
               # print('title: ', title, first_line)
               title += first_line
-              # print('title: ', title)
 
           def count_score(l):
               return len(l) + len(set(l)) * 2
@@ -3366,7 +3380,7 @@ class DocChannel():
                   return '合同公告', msc
               elif life_score.get('中标信息', 0) > 3 or '中标信息' in life_kw_title:
                   return '中标信息', msc
-              elif '招标公告' in life_kw_title and life_score.get('公告变更', 0) < 4:
+              elif '招标公告' in life_kw_title and re.search('变更|更正', title[-4:])==None and life_score.get('公告变更', 0) < 4:
                   return '招标公告', msc
               return '公告变更', msc
           elif '招标答疑' in life_kw_title or '招标答疑' in life_list:
@@ -3463,7 +3477,9 @@ class DocChannel():
           6、预测及原始均在变更、答疑，返回原始类别
           7、预测为采招数据，原始为产权且有关键词，返回原始类别
           8、废标公告原始为招标、预告且标题无废标关键期，返回原始类别
-          9、若预测为非采招数据且源网为采招数据且标题无关键词返回采招数据
+          9、若预测为非采招数据且源网为采招数据且有招标关键词返回采招数据
+          10、招标公告有中标人，且标题有直购关键词，改为中标信息
+          11、预测预告，原始为意向、招标且标题无预告关键词，返回原始类别
           '''
           if result['docchannel']['docchannel'] in ['中标信息', '合同公告'] and origin_dic.get(
                   original_docchannel, '') in ['招标公告', '采购意向', '招标预告', '公告变更'] and is_contain_winner(prem_json)==False:
@@ -3485,7 +3501,12 @@ class DocChannel():
           elif result['docchannel']['docchannel'] in ['招标公告'] and origin_dic.get(
                   original_docchannel, '') in ['采购意向', '招标预告']:
               result['docchannel']['docchannel'] = origin_dic.get(original_docchannel, '')
-              msc += '最终规则修改：预测及原始均在招标、预告、意向，返回原始类别'
+              msc += '最终规则修改：预测为招标，原始为预告、意向，返回原始类别'
+          elif result['docchannel']['docchannel'] in ['招标预告'] and origin_dic.get(
+                  original_docchannel, '') in ['采购意向', '招标公告'] and re.search(
+              self.title_life_dic['招标预告'], title)==None:
+              result['docchannel']['docchannel'] = origin_dic.get(original_docchannel, '')
+              msc += '最终规则修改：预测预告，原始为意向、招标且标题无预告关键词，返回原始类别'
           elif result['docchannel']['docchannel'] in ['招标答疑', '公告变更'] and origin_dic.get(
                   original_docchannel, '') in ['招标答疑', '公告变更']:
               result['docchannel']['docchannel'] = origin_dic.get(original_docchannel, '')
@@ -3499,10 +3520,17 @@ class DocChannel():
                   self.title_life_dic['废标公告'], title) == None:
               result['docchannel']['docchannel'] = origin_dic.get(original_docchannel, '')
               msc += '最终规则修改：废标公告原始为招标、预告且标题无废标关键期，返回原始类别;'
-          elif result['docchannel']['doctype'] != '采招数据' and origin_dic.get(
-                  original_docchannel, '') not in ['产权交易', '土地矿产', '拍卖出让'] and re.search('产权|转让|受让|招租|出租|承租|竞价|资产|挂牌|出让|拍卖|招拍|划拨', title)==None:
+          elif result['docchannel']['docchannel'] in ['招标公告', '招标预告'] and is_contain_winner(
+                  prem_json) and re.search('直购', title):
+              result['docchannel']['docchannel'] = '中标信息'
+              msc += "最终规则修改：预测为招标却有中标人且标题有直购关键词返回中标"
+
+          if result['docchannel']['doctype'] in ['产权交易', '土地矿产', '拍卖出让'] and origin_dic.get(
+                  original_docchannel, '') not in ['产权交易', '土地矿产', '拍卖出让'] \
+                and re.search('产权|转让|受让|招租|招商|出租|承租|竞价|资产|挂牌|出让|拍卖|招拍|划拨|销售', title) == None\
+                and re.search('(采购|招投?标|投标)(信息|内容|项目|公告|数量|人|单位|方式)|(建设|工程|服务|施工|监理|勘察|设计)项目', text):
               result['docchannel']['doctype'] = '采招数据'
-              msc += '最终规则修改：预测为非采招数据，原始为采招数据且无关键词，返回采招数据'
+              msc += ' 最终规则修改：预测为非采招数据，原始为采招数据且有招标关键词，返回采招数据'
 
           '''下面是新格式增加返回字段'''
           if result['docchannel']['docchannel'] != '':  # 预测到生命周期的复制到life_docchannel，否则用数据源结果
@@ -4228,7 +4256,7 @@ class DistrictPredictor():
             except Exception as e:
                 print('解析prem 获取招标人、及地址出错')
             return tenderee, tenderee_address
-        def get_area(text, web_source_name):
+        def get_area(text, web_source_name, not_in_content=True):
             score_l = []
             id_set = set()
 
@@ -4277,14 +4305,17 @@ class DistrictPredictor():
                         w = self.dist_dic[_id]['权重']
                         score = w * 0.2
                         score_l.append([_id, score] + area)
-            area_dic = {'area': '全国', 'province': '全国', 'city': '未知', 'district': '未知'}
+            area_dic = {'area': '全国', 'province': '全国', 'city': '未知', 'district': '未知', "is_in_text": False}
             if len(score_l) == 0:
                 return {'district': area_dic}
             else:
                 df = pd.DataFrame(score_l, columns=['id', 'score', 'province', 'city', 'district'])
+                df['简称'] = df['id'].apply(lambda x: self.dist_dic[x]['地区'])
+                # print('地区评分：')
+                # print(df)
                 df_pro = df.groupby('province').sum().sort_values(by=['score'], ascending=False)
                 pro_id = df_pro.index[0]
-                if df_pro.loc[pro_id, 'score'] < 0.1:  # 省级评分小于0.1的不要
+                if df_pro.loc[pro_id, 'score'] < 0.1 and not_in_content:  # 不是二次全文匹配的 省级评分小于0.1的不要
                     # print('评分低于0.1', df_pro.loc[pro_id, 'score'], self.dist_dic[pro_id]['地区'])
                     return {'district': area_dic}
                 area_dic['province'] = self.dist_dic[pro_id]['地区']
@@ -4305,39 +4336,41 @@ class DistrictPredictor():
                 return {'district': area_dic}
 
         tenderee, tenderee_address = get_ree_addr(prem)
-        project_name = str(project_name).replace(str(tenderee), '')
-        text1 = "{} {} {}".format(project_name, tenderee, tenderee_address)
+        project_name = str(project_name)
+        tenderee = str(tenderee)
+
+        if '##attachment##' in list_articles[0].content:
+            content, attachment = list_articles[0].content.split('##attachment##')
+            if len(content) < 200:
+                content += attachment
+        else:
+            content = list_articles[0].content
+
+        project_name = project_name + title if project_name not in title else project_name
+        project_name = project_name.replace(tenderee, '')
+        text1 = "{0} {1} {2}".format(project_name, tenderee, tenderee_address)
+        ser = re.search('项目所在地区?：(\w{2,8}[省市区县])+', content)
+        if ser:
+            text1 = ser.group(0)
+
         web_source_name = str(web_source_name)  # 修复某些不是字符串类型造成报错
         text1 = re.sub('复合肥|铁路|公路|新会计', ' ', text1)  #预防提取错 合肥 路南 新会 等地区
         rs = get_area(text1, web_source_name)
+
         if rs['district']['province'] == '全国' or rs['district']['city'] == '未知':
-            text2 = title + list_articles[0].content if len(list_articles[0].content)<2000 else title + list_articles[0].content[:1000] + list_articles[0].content[-1000:]
+            text2 = title + content if len(content)<2000 else title + content[:1000] + content[-1000:]
             text2 = re.sub('复合肥|铁路|公路|新会计', ' ', text2)
-            rs2 = get_area(text2, web_source_name)
+            rs2 = get_area(text2, web_source_name, not_in_content=False)
+            rs2['district']['is_in_text'] = True
             if rs['district']['province'] == '全国' and rs2['district']['province'] != '全国':
                 rs = rs2
             elif rs['district']['province'] == rs2['district']['province'] and rs2['district']['city'] != '未知':
                 rs = rs2
         return rs
 
-class TablePremExtractor(object):
-    def __init__(self):
-        '''各要素表头规则'''
-        self.head_rule_dic = {
-            'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|分[包标])编号",
-            'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$",
-            "project_name": "(包[段组件]|标[段包的]|分[包标]|采购|项目|工程)(名称?|内容)",
-            "win_sort": "是否中标|排名|排序|名次|未(中标|成交)原因",
-            "tenderer": "(中标|中选|中价|成交|供货|承包|承建|承租|竞得|受让)(候选)?(人|单位|供应商|公司|企业|厂家|商家?|客户|方)(名称|$)|^(拟定|单一来源)?供应商(名称)?$",
-            "tenderee": "(项目|采购|招标|遴选|寻源|竞价|议价|比选|委托|询比?价|比价|评选|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选)(人|公司|单位|组织|用户|业主|主体|方|部门)(名称|$)",
-            "budget": "最高(投标)?限价|总价限价|控制(价格?|金额|总价)|拦标价|(采购|招标|项目)预算|(预算|招标|采购|计划)金额|挂牌价",
-            "bid_amount": "投标[报总]价|(中标|成交)([金总]额|[报均总]价|价[格款])|承包价",
-        }
-
-        with open(os.path.dirname(__file__)+'/header_set.pkl', 'rb') as f:
-            self.headerset = pickle.load(f)
-
-    def table2list(self, table):
+class TableTag2List():
+    '''把soup table 转化为表格补全后的文本列表[[td, td, td], [td, td, td]]'''
+    def table2list(self, table, text_process=None):
         self._output = []
         row_ind = 0
         col_ind = 0
@@ -4346,17 +4379,21 @@ class TablePremExtractor(object):
             # we should skip
             smallest_row_span = 1
 
+            if len(row.find_all(['td', 'th'], recursive=False)) > 20:
+                log('未补全前表格列数大于20的不做表格处理')
+                return []
+
             for cell in row.children:
                 if cell.name in ('td', 'th'):
                     # check multiple rows
                     # pdb.set_trace()
-                    row_span = int(re.sub('[^0-9]', '', cell.get('rowspan'))) if cell.get('rowspan') and re.search('[0-9]', cell.get('rowspan')) else 1
+                    row_span = int(re.sub('[^0-9]', '', cell.get('rowspan'))) if cell.get('rowspan') and cell.get('rowspan').isdigit() else 1
 
                     # try updating smallest_row_span
                     smallest_row_span = min(smallest_row_span, row_span)
 
                     # check multiple columns
-                    col_span = int(re.sub('[^0-9]', '', cell.get('colspan'))) if cell.get('colspan') and re.search('[0-9]', cell.get('colspan')) else 1
+                    col_span = int(re.sub('[^0-9]', '', cell.get('colspan'))) if cell.get('colspan') and cell.get('colspan').isdigit() else 1
 
                     # find the right index
                     while True:
@@ -4366,15 +4403,19 @@ class TablePremExtractor(object):
 
                     # insert into self._output
                     try:
-                        text = str(cell.get_text()).replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '')
-                        text = re.sub('\s', '', text)[:200] # 只需取前200字即可
+                        if text_process != None:
+                            text = [re.sub('\xa0','',text_process(cell,final=False)),0]
+                        else:
+                            text = str(cell.get_text()).replace("\x06", "").replace("\x05", "").replace("\x07", "").replace('\\', '').replace("(", "（").replace(')', '）').replace('?', '')
+                            text = re.sub('\s', '', text)[:200] # 只需取前200字即可
+                            text = ' ' if text == "" else text
                         self._insert(row_ind, col_ind, row_span, col_span, text)
                     except UnicodeEncodeError:
                         raise Exception( 'Failed to decode text; you might want to specify kwargs transformer=unicode' )
 
                     # update col_ind
                     col_ind += col_span
-                    if col_ind > 50: # 表格列数大于50的去掉
+                    if col_ind > 50 and text_process == None: # 表格要素提取及候选人提取的 表格列数大于50的去掉
                         return []
 
             # update row_ind
@@ -4396,7 +4437,7 @@ class TablePremExtractor(object):
             return True
         if j >= len(self._output[i]):
             return True
-        if self._output[i][j] is None:
+        if self._output[i][j] == "":
             return True
         return False
 
@@ -4415,10 +4456,31 @@ class TablePremExtractor(object):
         if self._output[i][j] == "":
             self._output[i][j] = val
 
+
+class TablePremExtractor(object):
+    def __init__(self):
+        '''各要素表头规则'''
+        self.head_rule_dic = {
+            'project_code': "(项目|招标|采购|计划|公告|包[段组件]|标[段包的]|分[包标])编号",
+            'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$",
+            "project_name": "(包[段组件]|标[段包的]|分[包标]|采购|项目|工程|货物|商品)(名称?|内容)",
+            "win_sort": "是否中标|排名|排序|名次|未(中标|成交)原因",
+            "tenderer": "(中标|中选|中价|成交|供货|承包|承建|承租|竞得|受让)(候选)?(人|单位|供应商|公司|企业|厂家|商家?|客户|方)(名称|$)|^(拟定|单一来源)?供应商(名称)?$",
+            "tenderee": "(项目|采购|招标|遴选|寻源|竞价|议价|比选|委托|询比?价|比价|评选|谈判|邀标|邀请|洽谈|约谈|选取|抽取|抽选)(人|公司|单位|组织|用户|业主|主体|方|部门)(名称|$)",
+            "budget": "最高(投标)?限价|总价限价|控制(价格?|金额|总价)|拦标价|(采购|招标|项目)预算|(预算|招标|采购|计划)金额|挂牌价",
+            "bid_amount": "投标[报总]价|(中标|成交)([金总]额|[报均总]价|价[格款])|承包价",
+        }
+
+        with open(os.path.dirname(__file__)+'/header_set.pkl', 'rb') as f:
+            self.headerset = pickle.load(f)
+
+        self.tb = TableTag2List()
+
+
     def find_header(self, td_list):
         header_dic = dict()
         flag = False
-        if len(set(td_list))>2 and len(set(td_list) & self.headerset)/len(set(td_list))>0.6:
+        if len(set(td_list))>2 and len(set(td_list) & self.headerset)/len(set(td_list))>=0.6:
             flag = True
             for i in range(len(td_list)) :
                 text = td_list[i]
@@ -4429,18 +4491,33 @@ class TablePremExtractor(object):
                 num = 0
                 for k, v in self.head_rule_dic.items():
                     if re.search(v, text):
+                        if k  in ['tenderer'] and re.search('是否', text):
+                            continue
                         header_dic[k] = (i, text)
                         num += 1
                 if num>1:
                     print('表头错误，一个td匹配到两个表头：', header_dic)
                     return flag, dict()
+            if re.search('；金额（万?元）；', '；'.join(td_list)):  # 召回某些表格只写 金额 作为表头，不能识别为招标或中标金额
+                if 'tenderer' in header_dic and 'bid_amount' not in header_dic:
+                    for i in range(len(td_list)):
+                        text = td_list[i]
+                        if  re.search('^金额（万?元）$',text):
+                            header_dic['bid_amount'] = (i, text)
+                            break
+                elif 'tenderee' in header_dic and 'budget' not in header_dic:
+                    for i in range(len(td_list)):
+                        text = td_list[i]
+                        if re.search('^金额（万?元）$', text):
+                            header_dic['budget'] = (i, text)
+                            break
             if ('project_code' in header_dic or 'package_code' in header_dic or 'project_name' in header_dic) and (
                     'budget' in header_dic or 'tenderer' in header_dic):
                 return flag, header_dic
         return flag, dict()
 
     def is_role(self, text):
-        if len(text) > 25 or len(text)<5:
+        if len(text) > 25 or len(text)<4:
             return False
         elif len(re.findall('有限责?任?公司', text)) > 1:
             return False
@@ -4469,6 +4546,10 @@ class TablePremExtractor(object):
             bid_amount_ = df.loc[i, headers['bid_amount'][0]] if "bid_amount" in headers else ""
             win_sort = df.loc[i, headers['win_sort'][0]] if "win_sort" in headers else ""
 
+            if package_code_raw == "" and re.search('第?[0-9一二三四五六七八九十a-zZ-Z]{1,4}(标[段号的包项]|([分子]?包|包[组件号]))$|^(标[段号的包项]|([分子]?包|包[组件号]))号?：?[0-9一二三四五六七八九十a-zZ-Z]{1,4}$', project_name):
+                package_code_raw = project_name
+                project_name = ""
+
             package_code = package_code_raw
             if re.search('合计|总计', package_code+project_code):
                 continue
@@ -4481,7 +4562,7 @@ class TablePremExtractor(object):
                 continue
             if win_sort != "" and re.search('是否中标', headers['win_sort'][1]) and re.search('否', win_sort) == None:
                 continue
-            if win_sort == "" and "tenderer" in headers and re.search('候选|入围', headers['tenderer'][1]) and 'bid_amount' in headers and re.search('(中标|成交)价', headers['bid_amount'][1])==None:
+            if win_sort == "" and "tenderer" in headers and re.search('候选|入围', headers['tenderer'][1]):
                 tenderer = ""
 
             tenderee = tenderee if self.is_role(tenderee) else ""
@@ -4578,9 +4659,10 @@ class TablePremExtractor(object):
     def get_prem(self, soup):
         tables = soup.find_all('table')
         tables.reverse()
+
         rs_dic = {}
         for table in tables:
-            trs = self.table2list(table)
+            trs = self.tb.table2list(table)
             table.extract()
             i = 0
             headers = ""
@@ -4599,7 +4681,7 @@ class TablePremExtractor(object):
                         else:
                             print('表头，内容 列数不一致', len(trs[i]), len(trs[j]))
                             break
-                    if len(table_items) > 1:
+                    if len(table_items) > 0:
                         df = pd.DataFrame(table_items)
                         prem_ = self.extract_from_df(df, headers)
                         rs_dic.update(prem_)
@@ -4615,8 +4697,286 @@ class TablePremExtractor(object):
         prem = self.get_prem(soup)
         if prem == {} and richText:
             prem = self.get_prem(richText)
+        if len(prem) == 1:  # 只有一个包且包号为1 或 长度大于2 的大概率为自动增加编号包，改为Project
+            k = list(prem)[0]
+            if k == '1' or len(k) > 2:
+                prem['Project'] = prem.pop(k)
         return prem
 
+class CandidateExtractor(object):
+    def __init__(self):
+        '''各要素表头规则'''
+        self.head_rule_dic = {
+            'package_code': "(包[段组件]|标[段包]|分[包标])(序?号|$)|包号|^标段$",
+            "win_sort": "排名|排序|名次",
+            'win_or_not': '是否中标|是否入围|是否入库|入围结论',
+            "candidate": "((候选|入围|入选|投标)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|(中标)?公司|(中标)?企业)|(通过)?名单)(名称|名单|全称|\d)?$|^供应商(名称)?$",
+            "bid_amount": "投标[报总]价|(中标|成交)([金总]额|[报均总]价|价[格款])|承包价",
+            "win_tenderer": "第一名|第一(中标|成交)?候选人",
+            "second_tenderer": "第二名|第二(中标|成交)?候选人",
+            "third_tenderer": "第三名|第三(中标|成交)?候选人",
+        }
+        '''非表格候选人正则'''
+        self.p = '((候选|入围|入选|投标)(供应商库)?的?(人|人?单位|机构|供应商|供货商|服务商|投标人|(中标)?公司|(中标)?企业)|(通过)?名单)(名称|名单|全称|\d)?：$'
+        self.tb = TableTag2List()
+        with open(os.path.dirname(__file__)+'/header_set.pkl', 'rb') as f:
+            self.headerset = pickle.load(f)
+
+    def find_header(self, td_list):
+        header_dic = dict()
+        flag = False
+        if len(set(td_list))>=2 and len(set(td_list) & self.headerset)/len(set(td_list))>=0.6:
+            flag = True
+            for i in range(len(td_list)) :
+                text = td_list[i]
+                if len(text) > 15: # 长度大于15 不进行表头匹配
+                    continue
+                if re.search('未(中标|成交)原因', text):  # 不提取此种表格
+                    return flag, dict()
+                num = 0
+                for k, v in self.head_rule_dic.items():
+                    if re.search(v, text):
+                        if k in ['candidate', 'win_tenderer', 'second_tenderer', 'third_tenderer']  and re.search('是否', text):
+                            continue
+                        header_dic[k] = (i, text)
+                        if k != 'candidate': # candidate 可与前三候选重复
+                            num += 1
+                if num>1:
+                    print('表头错误，一个td匹配到两个表头：', header_dic)
+                    return flag, dict()
+            if 'candidate' in header_dic or ('win_tenderer' in header_dic and 'second_tenderer' in header_dic):
+                return flag, header_dic
+        return flag, dict()
+
+    def is_role(self, text):
+        if len(text) > 25 or len(text) < 4:
+            return False
+        elif len(re.findall('有限责?任?公司', text)) > 1:
+            return False
+        elif re.search('[\w（）]{4,}(有限责?任?公司|学校|学院|大学|中学|小学|医院|管理处|办公室|委员会|村委会|纪念馆|监狱|管教所|修养所|社区|农场|林场|羊场|猪场|石场|村|幼儿园|厂|中心|超市|门市|商场|工作室|文印室|城|部|店|站|馆|行|社|处)$', text):
+            return True
+        else:
+            ners = selffool.ner(text)
+            if len(ners[0]) == 1 and ('company' in ners[0][0] or 'org' in ners[0][0]):
+                return True
+        return False
+
+    def money_process(self, money_text, header):
+        '''
+        输入金额文本及金额列表头，返回统一数字化金额及金额单位
+        :param money_text:
+        :param header:
+        :return:
+        '''
+        money = 0
+        money_unit = ""
+        re_price = re.search("[零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]{3,}|\d[\d,]*(?:\.\d+)?万?", money_text)
+        if re_price:
+            money_text = re_price.group(0)
+            if '万元' in header and '万' not in money_text:
+                money_text += '万元'
+            money = float(str(getUnifyMoney(money_text)))
+            if money > 10000000000000:  # 大于万亿的去除
+                money = 0
+            money_unit = '万元' if '万' in money_text else '元'
+        return (money, money_unit)
+
+    def extract_from_df(self, df, headers):
+        prem_dic = {}
+        link_set = set()
+        candidate_set = set()
+        role_dic = dict()  # 保存一二三候选人并排的情况
+        for i in df.index:
+            package_code_raw = df.loc[i, headers['package_code'][0]] if "package_code" in headers else ""
+            candidate_ = df.loc[i, headers['candidate'][0]] if "candidate" in headers else ""
+            win_or_not = df.loc[i, headers['win_or_not'][0]] if "win_or_not" in headers else ""
+            # budget_ = df.loc[i, headers['budget'][0]] if "budget" in headers else ""
+            bid_amount_ = df.loc[i, headers['bid_amount'][0]] if "bid_amount" in headers else ""
+            win_sort = df.loc[i, headers['win_sort'][0]] if "win_sort" in headers else ""
+            win_tenderer = df.loc[i, headers['win_tenderer'][0]] if "win_tenderer" in headers else ""
+            second_tenderer = df.loc[i, headers['second_tenderer'][0]] if "second_tenderer" in headers else ""
+            third_tenderer = df.loc[i, headers['third_tenderer'][0]] if "third_tenderer" in headers else ""
+
+            if candidate_ != "" and win_sort == "" and headers['candidate'][0] > 0: # 修复某些表头不说 排名，直接用候选人代替
+                col_indx = headers['candidate'][0] -1
+                pre_col = df.loc[i, col_indx]
+                if col_indx > 0 and pre_col == candidate_:
+                    pre_col = df.loc[i, col_indx - 1]
+                if re.search('第[一二三]名|第[一二三](中标)?候选人', pre_col):
+                    win_sort = pre_col
+
+            package_code = package_code_raw
+
+            candidate = candidate_ if self.is_role(candidate_) else ""
+            # tenderer = tenderer if self.is_role(tenderer) else ""
+
+            # if len(set([project_code, package_code, project_name, tenderee, tenderer, budget_, bid_amount_])) < 2:
+            #     break
+            if(candidate,win_tenderer, second_tenderer,third_tenderer, bid_amount_) in link_set:
+                continue
+            link_set.add((candidate_, win_tenderer, second_tenderer, third_tenderer, bid_amount_))
+            package = package_code
+            package = uniform_package_name(package) if package !="" else "Project"
+            if candidate_:
+                if win_or_not and re.search('否|未入围', win_or_not):
+                    pass
+                else:
+                    candidate_set.add(candidate)
+
+            if win_tenderer and second_tenderer and third_tenderer:
+                if re.search("(候选人|投标人)名?称?$", df.loc[i, 0]) or re.search("(候选人|投标人)名?称?", df.loc[i, 1]):
+                    for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'],
+                                           [win_tenderer, second_tenderer, third_tenderer]):
+                        if self.is_role(text):
+                            if type not in role_dic:
+                                role_dic[type] = dict()
+                            role_dic[type]['role_text'] = text
+                            if type in ['second_tenderer', 'third_tenderer']:
+                                candidate_set.add(text)
+
+                elif re.search('投标报价|报价$', df.loc[i, 0]) or re.search('投标报价|报价$', df.loc[i, 1]):
+                    header = df.loc[i, 0] if re.search('投标报价|报价$', df.loc[i, 0]) else df.loc[i, 1]
+                    for type, text in zip(['win_tenderer', 'second_tenderer', 'third_tenderer'],
+                                           [win_tenderer, second_tenderer, third_tenderer]):
+                        money, money_unit = self.money_process(text, header)
+                        if money > 0:
+                            if type not in role_dic:
+                                role_dic[type] = dict()
+                            role_dic[type]['money'] = money
+                            role_dic[type]['money_unit'] = money_unit
+                else:
+                    break
+            elif candidate and win_sort:
+                role_type = ""
+                if re.search('第[一1]|^[一1]$', win_sort):
+                    role_type = "win_tenderer"
+                elif re.search('第[二2]|^[二2]$', win_sort):
+                    role_type = "second_tenderer"
+                elif re.search('第[三3]|^[三3]$', win_sort):
+                    role_type = "third_tenderer"
+                if role_type != "":
+                    if package not in prem_dic:
+                        prem_dic[package] = {
+                            'code': '',
+                            'name': '',
+                            'roleList': [],
+                            'tendereeMoney': 0,
+                            'tendereeMoneyUnit': ""
+                        }
+
+                    bid_amount, money_unit  = self.money_process(bid_amount_, df.loc[i, headers['bid_amount'][0]])  if "bid_amount" in headers else (0, "")
+                    prem_dic[package]['roleList'].append({
+                            "address": "",
+                            "linklist": [],
+                            "role_money": {
+                                "discount_ratio": "",
+                                "downward_floating_ratio": "",
+                                "floating_ratio": "",
+                                "money": bid_amount,
+                                "money_unit": money_unit
+                            },
+                            "role_name": role_type,
+                            "role_text": candidate,
+                            "serviceTime": ""
+                    })
+                    if len(prem_dic[package]['roleList']) == 0:  # 只有项目编号和名称的 丢弃
+                        prem_dic.pop(package)
+        if role_dic and prem_dic == dict():
+            if package not in prem_dic:
+                prem_dic[package] = {
+                    'code': '',
+                    'name': '',
+                    'roleList': [],
+                    'tendereeMoney': 0,
+                    'tendereeMoneyUnit': ""
+                }
+            for role_type, v in role_dic.items():
+                role_text = v.get('role_text', '')
+                if role_text == "":
+                    continue
+                money = v.get('money', 0)
+                money_unit = v.get('money_unit', '')
+                prem_dic[package]['roleList'].append({
+                    "address": "",
+                    "linklist": [],
+                    "role_money": {
+                        "discount_ratio": "",
+                        "downward_floating_ratio": "",
+                        "floating_ratio": "",
+                     "money": money,
+                        "money_unit": money_unit
+                    },
+                    "role_name": role_type,
+                    "role_text": role_text,
+                    "serviceTime": ""
+                    })
+            if len(prem_dic[package]['roleList']) == 0:  # 只有项目编号和名称的 丢弃
+                prem_dic.pop(package)
+
+        return prem_dic, candidate_set
+
+    def get_prem(self, soup):
+        tables = soup.find_all('table')
+        tables.reverse()
+        rs_dic = {}
+        candidate_set = set()
+        for table in tables:
+            trs = self.tb.table2list(table)
+            table.extract()
+            i = 0
+            headers = ""
+            while i < len(trs) - 1:
+                flag_, headers_ = self.find_header(trs[i])
+                if flag_ and headers_ != dict():
+                    table_items = []
+                    headers = headers_
+                    for j in range(i + 1, len(trs)):
+                        if len(trs[j]) == len(trs[i]):
+                            flag_, headers_ = self.find_header(trs[j])
+                            if flag_:
+                                break
+                            else:
+                                table_items.append(trs[j])
+                        else:
+                            print('表头，内容 列数不一致', len(trs[i]), len(trs[j]))
+                            break
+                    if len(table_items) > 1:
+                        df = pd.DataFrame(table_items)
+                        prem_, candidate_set_ = self.extract_from_df(df, headers)
+                        rs_dic.update(prem_)
+                        candidate_set.update(candidate_set_)
+                    i = j - 1
+                i += 1
+        return rs_dic, candidate_set
+
+    def get_candidates_from_text(self, list_sentences, list_entitys):
+        candidates = set()
+        sentences = sorted(list_sentences[0], key=lambda x: x.sentence_index)
+        for ent in list_entitys[0]:
+            if ent.entity_type in ['org', 'company']:
+                sen_index = ent.sentence_index
+
+                text = sentences[sen_index].sentence_text
+                b = ent.wordOffset_begin
+                e = ent.wordOffset_end
+                if isinstance(b, int) and isinstance(e, int):
+                    foreword = text[max(0, b - 10):b]
+                    if re.search(self.p, foreword):
+                        candidates.add(ent.entity_text)
+        return candidates
+
+    def predict(self, html, list_sentences, list_entitys):
+        soup = BeautifulSoup(html, 'lxml')
+        richText = soup.find(name='div', attrs={'class': 'richTextFetch'})
+        if richText:
+            richText = richText.extract()  # 过滤掉附件
+        prem, candidate_set = self.get_prem(soup)
+        if prem == {} and richText:
+            prem, candidate_set = self.get_prem(richText)
+        if prem == {} and candidate_set == set():
+            candidate_set = self.get_candidates_from_text(list_sentences, list_entitys)
+        return prem, {'candidate': '，'.join(candidate_set)}
+
 
 def getSavedModel():
     #predictor = FormPredictor()