BIDI_EXTRACTION_AI/BiddingKG/dl/predictors/approval.py

# -*- coding: utf-8 -*-
"""审批信息预测器。

按 ARCHITECTURE.md Phase 5 拆分建议，从 ``interface/predictor.py`` 迁出以下类：
    - ``ApprovalPredictor`` — 审批公告结构化信息提取（项目名称/编号/文号/审批部门
      /建设单位/环评机构/建设地址/建设年限/建设规模/审批结果等）
      （原 predictor.py 第 9323-9833 行）

原 ``from common.Utils import *`` / ``from interface.modelFactory import *``
已替换为显式 import；``os.path.dirname(__file__)`` 路径引用替换为
``predictors._common.INTERFACE_DIR``。

``interface/predictor.py`` 仍保留原定义，老 import 不受影响。
"""
from __future__ import absolute_import

import re

import pandas as pd
from bs4 import BeautifulSoup

from BiddingKG.dl.common.Utils import timeFormat
from BiddingKG.dl.common.attr_utils import turnMoneySource
from BiddingKG.dl.interface.predictor import getPredictor
from BiddingKG.dl.predictors._common import get_role
from BiddingKG.dl.predictors.table_prem import TableTag2List, get_header_line

__all__ = ["ApprovalPredictor"]


class ApprovalPredictor():
    def __init__(self):
        '''
        项目（法人）单位
        '''
        self.other_part = {
            "project_name": "((项目|工程|采购|招标|计划|建设|规划)名称?|生产建设项目|申请项目)：(?P<main>[^：。]{5,50})[，。]([\w（）]{2,15}：|$)?", # 项目名称
            "project_code": "(立案号|项目(统一)?代码|(项目|工程|采购|招标|计划|任务|备案|索引)([编代][号码]|号))：?(?P<main>(\w{2,8})?[（）〔〕【】\[\]a-zA-Z0-9-]{5,30}号?)([\w（）]{2,15}：|$)?", # 项目编号
            "doc_num": "((环评|\w{,3})(审[批查核]|批[复准]|立项|[定知文]书|[公发批]文|用地|决定|备案|核准|许可|确认|受理|申请报告|文[件书]|意见书|办件)[文编证]?号|综合受理号|文书?号|合格书号|申报号|(办件|事项)[编代][号码]|收件号)）?为?：?(?P<main>[（）〔〕【】\[\]0-9]{,8}([\w（）〔〕【】]{2,15})?[（）〔〕【】\[\]a-zA-Z0-9-.]{3,30}号?)[，。]?([\w（）]{2,15}：|$)?", # 文号
            "pro_type": "((申[报请]|审核备|项目|立项)(类型|种类)|项目所属行业|行业(分类|归属)|产业领域|项目行业)：(?P<main>[^：。]{2,30})[，。]([\w（）]{2,15}：|$)?", # 项目类型
            "year_limit": "((建设|工程|服务|项目)(起止|\w{,2})?(年限|期限|时长|工期))：(约|超过|大概|建设工期|共计|合计)?(?P<main>[\d一二三四五六七八九十]+个月|\d{1,3}(日?历?天|小时)|20\d{2}[年/-](\d{1,2}[月/-]?)?(\d{1,2}日?)?([至—-]+20\d{2}[年/-](\d{1,2}[月/-]?)?(\d{1,2}日?)?)?)[（，。]([\w（）]{2,15}：|$)?", # 建设年限
            "construction_scale": "([\d一二三四五六七八九十]{1,2}、|（[\d一二三四五六七八九十]{1,2}）)?(工程|项目|\w{,4})?((建设内容[及和](建设)?规模|建设规模[及和](主要)?(建设)?内容|(建设|招标|采购)）?内容|(建设|工程|项目)(主要)?(规模|内容|概况|面积)([及和](主要)?(规模|内容|概况|面积))?(如下|为)?)|^规模(情况)?)：(?P<main>[^：。]{2,500})[，。]?([\w（）]{2,30}：|$)?", # 建设规模 #56924861 主要环境影响及预防或者减轻不良环境影响的对策和措施：
            "approval_items": "((审[批查核]|批[复准]|申请|监管|受理)(事项|内容|名称)|事项名称|事项审批)：(?P<main>[^：。]{2,150})[，。]([\w（）]{2,15}：|$)?", # 审批事项
            "properties": "((建设|工程|项目)性质)：(?P<main>[^：。]{2,50})[，。]([\w（）]{2,15}：|$)?", # 建设性质
            "approval_result": "((审[批查核]|批[复准]|核[发准]|许可|抽查|备案)(结果|决定|结论|状态|回复|意见)|(办[理件]|，)(状态|意见|结果)|项目(当前|目前)?状态)：(?P<main>[^：。]{2,20})[，。]([\w（）]{2,15}：|$)?", # 审批结果
            "phone": "(联系)?电话：(?P<main>1[3-9][0-9][-—－―]?\d{4}[-—－―]?\d{4}|" # 联系电话
                     '\+86.?1[3-9]\d{9}|'
                     '0[1-9]\d{1,2}[-—－―][2-9]\d{6}\d?[-—－―]\d{1,4}|'
                     '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=1[3-9]\d{9})|'
                     '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=0[1-9]\d{1,2}[-—－―]?[2-9]\d{6}\d?)|'
                     '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?(?=[2-9]\d{6,7})|'
                     '0[1-9]\d{1,2}[-—－―]{0,2}[2-9]\d{6}\d?|'
                     '[\（|\(]0[1-9]\d{1,2}[\）|\)]-?[2-9]\d{6}\d?-?\d{,4}|'
                     '400\d{7}转\d{1,4}|'
                     '[2-9]\d{6,7})[，。]([\w（）]{2,15}：|$)?'
        }

        self.role_type = {
            "declare_company": "(申[请报]|填报|呈报)(人|部门|机关|单位|企业|公司|机构|组织)",  # 申报单位
            "construct_company": "(业主|建设|用地|委托|发包|产权|项目|法人|采购|招标|询价)）?(部门|机关|单位|企业|公司|方|业主|人)|主送机关|法人单位|甲方",  # 建设单位
            "approver": "(审[批查核议图]|许可|批[复准](用地)?|发证|管理|办理|受理|核[发准]|备案|承办)）?(部门|机关|单位|企业|公司|机构)|实施主体",  # 审批部门
            "evaluation_agency": "(环境|环保)?(影响)?(环评|评价|评估)(机构|单位|公司)" , # 环评机构
            "compilation_unit": "编制单位", # 编制单位 20240701加
            "publisher": "(发布|发文|公示|公告)(人|部门|机关|单位|企业|公司|机构|组织)" # 发布机构 20240703加
        }
        self.person_type = {
            "legal_person": "项目法人|法定代表人|企业法人"  # 项目法人
        }
        self.date_type = {
            "time_declare": "(申[请报]|填报|呈报)(时间|日期)", # 申报时间
            "time_commencement": "(开工|动工|(项目|建设|工程|施工)开始)(时间|日期)", # 开工时间
            "time_completion": "(竣工|完工|验收|(项目|建设|工程|施工)(完成|结束))(备案)?(时间|日期)", # 竣工时间
            "time_approval": "(审[批查核查议]|许可|批[复准](用地)?|发证|管理|办理|受理|核[发准]|备案|决定)(时间|日期)", # 审批时间 20240701加
            "time_release": "(发布|发文|公告|生成|成文)(时间|日期)" # 发布时间
        }

        self.addr_type = {
            "project_addr": "((建设|工程|项目|施工|地块|用地)\w{,2}(地址|地点|位置|所在地)|[宗土]地坐落)" # 建设地址
        }

        self.money_type = {
            "total_tendereeMoney": "(项目|概算|投资)金额|项目投资|总投资|总预算|总概算|投资(规模|总额|估算|概算)|批复概算|投资额|项目概算", # 总投资
        }

        self.head_rule_dic = {**self.role_type, **self.person_type, **self.date_type, **self.addr_type, **self.money_type}
        self.head_rule_dic.update({k: v.split('：')[0] for k,v in self.other_part.items()})
        self.tb = TableTag2List()

    def recursive_text(self, tag):
        '''
        递归获取 soup 节点文本
        :param tag:
        :return:
        '''
        texts = []
        for child in tag.children:
            if child.name:
                if child.name in ['p'] and len(child.find_all('br'))>2:
                    texts.extend(self.recursive_text(child))
                if child.name in ["td", "th", "p", "li", "h1", "h2", "h3", "h4", "h5",
                                  "h6"] and child.get_text().strip():
                    texts.append(re.sub('\s', '', child.get_text().strip().replace(':', '：').replace('(', '（').replace(')', '）')))

                else:
                    texts.extend(self.recursive_text(child))
            else:
                if child.strip():
                    texts.append(re.sub('\s', '', child.strip().replace(':', '：').replace('(', '（').replace(')', '）')))
        return texts

    def get_table_info(self, df, nlp_enterprise):
        def get_header_index(datas):
            '''
            根据表格表头判断结果0/1 得到哪些行和列是表头
            :param datas: 表格内容表头判断结果数据[[1,1,1,1],[0,0,0,0]]
            :return: 表头所在的行和列序号
            '''
            header_row = []
            header_col = []
            df_h = pd.DataFrame(datas)  # 表头判断数据 , columns=columns
            for i in df_h.index:
                line = df_h.loc[i].values
                if sum(line) == len(line):
                    header_row.append((i, sum(line) / len(line)))
                elif sum(line) / len(line) > 0.8:
                    header_row.append((i, sum(line) / len(line)))
                elif len(line) > 3 and len(re.findall('11', ''.join([str(it) for it in line]))) > len(
                        re.findall('10', ''.join([str(it) for it in line]))):
                    header_row.append((i, sum(line) / len(line)))
            for i in df_h.columns:
                col = df_h[i].values
                if sum(col) == len(col):
                    header_col.append((i, sum(col) / len(col)))
                elif sum(col) / len(col) > 0.8:
                    header_col.append((i, sum(col) / len(col)))
                elif len(col) > 3 and len(re.findall('11', ''.join([str(it) for it in line]))) > len(
                        re.findall('10', ''.join([str(it) for it in line]))):
                    header_col.append((i, sum(col) / len(col)))
            return header_row, header_col

        def get_header(l, head_rule_dic):
            header_dic = {}
            for i in range(len(l)):
                text = l[i].replace(' ', '') # 修复54969575 项目 名称 被空格分割
                num = 0
                tmp_dic = {}
                for k, v in head_rule_dic.items():
                    if re.search(v, text):
                        tmp_dic[k] = i
                        num += 1
                for k, v in tmp_dic.items():
                    if k not in header_dic:
                        header_dic[k] = v
            return header_dic

        result_l = []
        datas = []
        for i in df.index:
            line = get_header_line(df.loc[i].values)
            datas.append(line)
        header_row, header_col = get_header_index(datas)
        if len(header_col) == 1 and header_col[0][0] > 1: # 列表头不可能在第1列后面开始
            header_col = []
        if len(header_row) >= 1 and len(header_col) == 0:  # 有行表头无列表头
            i = 0
            while i < len(header_row):
                idx, ratio = header_row[i]
                if idx + 1 >= len(df):
                    break
                header_dic = get_header(df.loc[idx].values, self.head_rule_dic)
                i += 1
                range_from = idx + 1
                range_to = len(df)
                if i < len(header_row):
                    next_header = i
                    for j in range(i, len(header_row)):
                        idx2, ratio2 = header_row[j]
                        if idx2 - idx == 1:
                            header_dic2 = get_header(df.loc[idx2].values, self.head_rule_dic)
                            if set(df.loc[idx].values) & set(df.loc[idx2].values) != set():
                                header_dic.update(header_dic2)
                            else:
                                header_dic = header_dic2
                            range_from = idx2 + 1
                            range_to = len(df)
                            next_header = j + 1
                            idx = idx2
                        else:
                            range_from = idx + 1
                            range_to = idx2
                            next_header = j
                            break
                    i = next_header
                if len(header_dic) >= 2 and 'project_name' in header_dic:
                    for index in range(range_from, range_to):
                        if len(set(df.loc[index, :])) <= 2: # 修复 56873031 补全内容跟表头错误连接
                            continue
                        tmp_dic = {}
                        for k, v in header_dic.items():
                            if k.startswith('time_'):
                                try:
                                    content = timeFormat(df.loc[index, v], default_first_day=False) if k in [
                                        'time_completion'] else timeFormat(df.loc[index, v])
                                except:
                                    content = ""
                            elif k in self.role_type:
                                content = get_role(df.loc[index, v], nlp_enterprise)
                            elif k == 'moneysource':
                                content = turnMoneySource(df.loc[index, v])
                            else:
                                content = df.loc[index, v]
                            if content != '':
                                tmp_dic[k] = content
                        if len(tmp_dic) > 1 and 'project_name' in tmp_dic and tmp_dic not in result_l:
                            result_l.append(tmp_dic)
        elif len(header_row) == 0 and len(header_col) >= 1:
            return result_l # 不提取列向表格，容易出错 例 53489774 作多标段
            i = 0
            while i < len(header_col):
                idx, ratio = header_col[i]
                if idx + 1 >= len(df.columns):
                    break
                header_dic = get_header(df[idx].values, self.head_rule_dic)
                i += 1
                range_from = idx + 1
                range_to = len(df.columns)
                if i < len(header_col):
                    next_header = i
                    for j in range(i, len(header_col)):
                        idx2, ratio2 = header_col[j]
                        if idx2 - idx == 1:
                            header_dic2 = get_header(df[idx2].values, self.head_rule_dic)
                            if set(df[idx].values) & set(df[idx2].values) != set():
                                header_dic.update(header_dic2)
                            else:
                                header_dic = header_dic2
                            range_from = idx2 + 1
                            range_to = len(df.columns)
                            next_header = j + 1
                            idx = idx2
                        else:
                            range_from = idx + 1
                            range_to = idx2
                            next_header = j
                            break
                    i = next_header
                if len(header_dic) >= 2 and 'project_name' in header_dic:
                    for index in range(range_from, range_to):
                        if len(set(df.loc[:, index])) <= 2:
                            continue
                        tmp_dic = {}
                        for k, v in header_dic.items():
                            if k.startswith('time_'):
                                content = timeFormat(df.loc[v, index], default_first_day=False) if k in [
                                    'time_completion'] else timeFormat(df.loc[v, index])
                            elif k in self.role_type:
                                content = get_role(df.loc[v, index], nlp_enterprise)
                            elif k == 'moneysource':
                                content = turnMoneySource(df.loc[v, index])
                            else:
                                content = df.loc[v, index]
                            if content != '':
                                tmp_dic[k] = content
                        if len(tmp_dic) > 2 and 'project_name' in tmp_dic and tmp_dic not in result_l:
                            result_l.append(tmp_dic)
        elif len(header_row) == 1 and len(header_col) == 1:
            pass
        return result_l

    def predict_table(self, html, nlp_enterprise=[]):
        html = re.sub("<html>|</html>|<body>|</body>", "", html)
        html = re.sub("##attachment##", "", html)
        soup = BeautifulSoup(html, 'lxml')
        richText = soup.find(name='div', attrs={'class': 'richTextFetch'})
        self.nlp_enterprise = nlp_enterprise
        if richText:
            richText = richText.extract()  # 过滤掉附件
        tables = soup.find_all('table')
        if len(tables) == 0 and richText:
            tables = richText.find_all('table')
        tables.reverse()
        data_list = []
        for table in tables:
            trs = self.tb.table2list(table)
            if len(trs) > 1 and len(set(trs[0])) > 0 and len(set([len(tr) for tr in trs])) == 1: # 表格两行以上且每行列数一样才处理
                df = pd.DataFrame(trs)
                rs_l = self.get_table_info(df, nlp_enterprise)
                for d in rs_l: # 53338603 项目名称+建设内容才是唯一
                    if d not in data_list:
                        data_list.append(d)
                if rs_l:
                    table.extract()
        return data_list

    def predict(self, list_sentences, list_entitys, html, nlp_enterprise=[], span=12):
        tabel_rs = self.predict_table(html, nlp_enterprise)  # 表格多项目提取
        soup = BeautifulSoup(html, 'lxml')
        texts_list = self.recursive_text(soup)
        rs_dic = {k: "" for k in
                  self.other_part.keys() | self.role_type.keys() | self.date_type.keys() | self.addr_type.keys() | self.money_type.keys() | self.person_type.keys()}
        rs_dic['moneysource'] = ""
        sentences = [it.sentence_text for it in sorted(list_sentences[0], key=lambda x: x.sentence_index)]
        entities = [[] for _ in range(len(sentences))]
        rs_l = []
        found_key = 0
        code_name_set = set() # 项目编号、名称集合
        org_set = set() # 保存可能为审批部门的角色
        not_sure_role = '' # 不确定角色， 例：单位名称：长沙驰能新能源开发有限公司眉县分公司
        for entity in list_entitys[0]:
            entities[entity.sentence_index].append(entity)

        for i in range(len(sentences)):
            multi_project = {k: "" for k in
                             self.other_part.keys() | self.role_type.keys() | self.date_type.keys() | self.addr_type.keys() | self.money_type.keys() | self.person_type.keys()}
            multi_project['moneysource'] = ''
            text = sentences[i]
            for entity in entities[i]:
                b, e = entity.wordOffset_begin, entity.wordOffset_end
                if entity.entity_type in ['org', 'company']:
                    flag = 1
                    role_l = []
                    for k, v in self.role_type.items():
                        ser = re.search(v, sentences[entity.sentence_index][max(0, b - span):b])
                        if ser:
                            role_l.append((k, ser.end()))
                    if role_l:
                        role_l = sorted(role_l, key=lambda x: x[1]) # 解决 400064746000 表格某个为空导致两个表头相近提取错误 申报单位名称：备案机关：海门经济技术开发区管理委员会，备案证号：海开审备〔2024〕346号
                        k, _ = role_l[-1]
                        if rs_dic[k] == '':
                            rs_dic[k] = entity.entity_text
                        multi_project[k] = entity.entity_text
                        found_key = 1
                        flag = 0
                        if not_sure_role == entity.entity_text:
                            not_sure_role = ''
                    elif re.search('(，|^)单位名称：', sentences[entity.sentence_index][max(0, b - span):b]):
                        not_sure_role = entity.entity_text
                    if flag and entity.entity_type == "org" and re.search('(局|委员会|委|厅)$', entity.entity_text):
                        org_set.add(entity.entity_text)
                elif entity.entity_type in ['person']:
                    for k, v in self.person_type.items():
                        if re.search(v, sentences[entity.sentence_index][max(0, b - span):b]):
                            if rs_dic[k] == '':
                                rs_dic[k] = entity.entity_text
                            multi_project[k] = entity.entity_text
                            found_key = 1
                            break
                elif entity.entity_type in ['time']:
                    time_l = []
                    for k, v in self.date_type.items():
                        ser = re.search(v, sentences[entity.sentence_index][max(0, b - span):b])
                        if ser:
                            time_l.append((k, ser.end()))
                    if time_l:
                        time_l = sorted(time_l, key=lambda x: x[1])
                        k, end = time_l[-1]
                        time = timeFormat(entity.entity_text, default_first_day=False) if k in [
                            'time_completion'] else timeFormat(entity.entity_text)
                        if time == "":
                            continue
                        if rs_dic[k] == '':
                            rs_dic[k] = time
                        multi_project[k] = time
                        found_key = 1
                elif entity.entity_type in ['location']:
                    for k, v in self.addr_type.items():
                        if re.search(v, sentences[entity.sentence_index][max(0, b - span):b]):
                            if rs_dic[k] == '':
                                rs_dic[k] = entity.entity_text
                            multi_project[k] = entity.entity_text
                            found_key = 1
                elif entity.entity_type in ['money']:
                    for k, v in self.money_type.items():
                        if re.search(v, sentences[entity.sentence_index][max(0, b - span):b]):
                            if rs_dic[k] == '':
                                rs_dic[k] = entity.entity_text
                            multi_project[k] = entity.entity_text
                            found_key = 1
                elif entity.entity_type in ['moneysource']:
                    rs_dic['moneysource'] = turnMoneySource(entity.entity_text)
                    multi_project['moneysource'] = turnMoneySource(entity.entity_text)
                elif entity.entity_type in ['code']:
                    k = 'project_code'
                    v = self.other_part[k].split('：', maxsplit=1)[0]
                    if re.search(v, sentences[entity.sentence_index][max(0, b - span):b]):
                        if rs_dic[k] == '':
                            rs_dic[k] = entity.entity_text
                        multi_project[k] = entity.entity_text
                        found_key = 1
                elif entity.entity_type in ['name']:
                    k = 'project_name'
                    v = self.other_part[k].split('：', maxsplit=1)[0]
                    if re.search(v, sentences[entity.sentence_index][max(0, b - span):b]):
                        if rs_dic[k] == '':
                            rs_dic[k] = entity.entity_text
                        multi_project[k] = entity.entity_text
                        found_key = 1
            for k, v in self.other_part.items(): # 规则提取非实体类信息
                ser = re.search(v, text)
                if ser:
                    if rs_dic[k] == '' or (k == 'project_name' and '，审批事项：' in rs_dic[k]): # 修复 54087410 项目名称包含错误
                        rs_dic[k] = ser.group('main')
                    multi_project[k] = ser.group('main')
                    found_key = 1
            for k, v in self.date_type.items(): # 规则补充时间实体
                if multi_project[k] != '':
                    continue
                ser = re.search(v+'：?(?P<main>20\d{2}-\d{1,2}(-\d{1,2})?|20\d{2}/\d{1,2}(/\d{1,2})?|20\d{2}\.\d{1,2}(\.\d{1,2})?|20\d{2}(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])?)', text)
                if ser:# 规则补充实体识别不到的日期时间
                    time = timeFormat(ser.group('main'), default_first_day=False) if k in ['time_completion'] else timeFormat(ser.group('main'))
                    if time == "":
                        continue
                    if rs_dic[k] == '':
                        rs_dic[k] = time
                    multi_project[k] = time
                    found_key = 1
            for k, v in self.addr_type.items(): # 规则补充地址实体 400063690529 实体不完整 建设地点：湖北省-咸宁市-通城县 通城县大坪乡沙口村15组（通城经济开发区）
                ser = re.search(v + '：?(?P<main>(\w{1,13}(自治[区州县旗]|地区|[省市区县旗盟])[^\w]*)+|\w{2,15}[，。])',text)
                if ser:
                    if rs_dic[k] == '' or len(rs_dic[k]) < len(ser.group('main')):
                        rs_dic[k] = ser.group('main')
                    if len(multi_project[k]) < len(ser.group('main')):
                        multi_project[k] = ser.group('main')
                    found_key = 1
            for k, v in self.role_type.items(): # 规则补充公司实体
                if multi_project[k] != '':
                    continue
                ser = re.search('(%s)：(?P<main>[\w（）]{6,30}(局|发改|超市|棋牌室|店|（个体工商户）))[，。]'%self.role_type[k], text)
                if ser:
                    if rs_dic[k] == '':
                        rs_dic[k] = ser.group('main')
                    multi_project[k] = ser.group('main')

            if (multi_project['project_code'] != "" or multi_project['project_name'] != "") and multi_project['project_code']+multi_project['project_name'] not in code_name_set:
                code_name_set.add(multi_project['project_code']+multi_project['project_name'])
                if len(set([k for k,v in multi_project.items() if v!=''])-set(['project_name', 'project_code']))<2: # 除了包其他要素少于两个的不作为多包
                    continue
                district = getPredictor('district').get_area(
                    multi_project['approver'] + multi_project['project_name'] + multi_project['project_addr'], '')
                if district['district']['province'] != '全国':
                    multi_project['area'] = district['district']['area']
                    multi_project['province'] = district['district']['province']
                    multi_project['city'] = district['district']['city']
                    multi_project['district'] = district['district']['district']
                multi_project = {k: v for k, v in multi_project.items() if v != ''}
                rs_l.append(multi_project)
        if not_sure_role != '' and rs_dic.get('construct_company', '') == '' and not_sure_role not in org_set: # 补充，单位名称：这种作为建设单位 例：400069851014
            rs_dic['construct_company'] = not_sure_role
        if len(tabel_rs) > 1:
            rs_dic_key = [k for k, v in rs_dic.items() if v != '']
            keys = set(["approver", "publisher", "time_release", "phone", "doc_num"]) & set(rs_dic_key) - set(tabel_rs[0].keys())
            if keys:
                for d in tabel_rs:
                    for k in keys:
                        d[k] = rs_dic[k]
            return tabel_rs

        if len(rs_l)>1 and len(set(rs_l[0].keys()))>2 and set(rs_l[0].keys())==set(rs_l[1].keys()):
            for k in self.role_type.keys(): # 多项目无建设单位等通过整篇提取补充
                if rs_dic.get(k, '') != '' and k not in rs_l[0].get(k, '') == '':
                    for d in rs_l:
                        if d.get(k, '') == '':
                            d[k] = rs_dic[k]
            return rs_l
        if found_key == 1:
            district = getPredictor('district').get_area(
                rs_dic['approver'] + rs_dic['project_name'] + rs_dic['project_addr'], '')
            if district['district']['province'] != '全国':
                rs_dic['area'] = district['district']['area']
                rs_dic['province'] = district['district']['province']
                rs_dic['city'] = district['district']['city']
                rs_dic['district'] = district['district']['district']
            if len(org_set) == 1 and rs_dic['approver'] == "":
                rs_dic['approver'] == org_set.pop()

            n = 0
            scale_l = [] # 保存以建设规模开头的文本，如果只有一个且比原来长的替换为此文本，避免提取不完成情况
            for text in texts_list: # 补充纠正内容
                for k, v in self.other_part.items():
                    kw = v.split('：')[0]
                    if re.search('^(%s)$'%kw, text) and rs_dic[k]=='':  # 处理非表格表头内容 排列数据 例：400064764198，web_no: XM0016-5
                        if n >1  and n+2 < len(texts_list) and get_header_line(texts_list[n-2:n+3]) == [1,0,1,0,1]:
                            rs_dic[k] = texts_list[n+1]
                        elif n in [0,1] and n+2 < len(texts_list) and get_header_line(texts_list[n:n+3]) == [1,0,1]:
                            rs_dic[k] = texts_list[n + 1]
                        elif n >1  and n+2 == len(texts_list) and get_header_line(texts_list[n-2:n+2]) == [1,0,1,0]:
                            rs_dic[k] = texts_list[n + 1]
                        elif k == 'construction_scale' and re.search('^（?[一二三四五六七八九十][）、]', text) and n+1 < len(texts_list): # 大纲 例：53375037
                            rs_dic[k] = texts_list[n + 1]
                    if k == 'construction_scale' and len(rs_dic.get(k, '')) < len(text):
                        ser = re.search('^(%s)：(?P<main>.+)'%kw, text)
                        if ser:
                            rs_dic[k] = ser.group('main')

                n += 1
                if 0<len(rs_dic['construction_scale'])<len(text) and rs_dic['construction_scale'][-1] not in ['，', '。'] and text.find(rs_dic['construction_scale'])==0:
                    scale_l.append(text)
            if len(scale_l)==1 and len(scale_l[0])>len(rs_dic['construction_scale']): # 规则补充不完整规模信息 例：53334434
                rs_dic['construction_scale'] = scale_l[0]
            if 0<len(rs_dic['construction_scale'])<8 and re.search('([编代][号码]|名称|时间|日期|金额|单位|机构)$', rs_dic['construction_scale']):
                rs_dic['construction_scale'] = ''

            for k, v in rs_dic.items(): # 限制最大长度
                if len(v)>500:
                    v = v[:500]+'...后面省略%d字'%(len(v)-500)
                    rs_dic[k] = v
                if v == 'null':
                    rs_dic[k] = ''

            rs_dic = {k: v for k, v in rs_dic.items() if v != ''}
            return [rs_dic]
        return []

    def add_ree2approval(self, approval, prem):
        '''
        把招标人补充到审批项目建设单位
        :param approval:
        :param prem:
        :return:
        '''
        ree = ''
        if "Project" in prem:
            for d in prem["Project"]['roleList']:
                if d["role_name"] == "tenderee":
                    ree = d["role_text"]
                    break
        if ree != '':
            for d in approval:
                if d.get('construct_company', '') == '':
                    d['construct_company'] = ree
                else:
                    break
        return approval

    def add_codename2approval(self, approval, codeName):
        if len(approval) == 1 and codeName:  # 根据整个公告项目编号及名称补充审批信息
            if 'project_code' not in approval[0] and codeName[0].get('code', []) != []:
                approval[0]['project_code'] = codeName[0].get('code', [])[0]
            if 'project_name' not in approval[0] and codeName[0].get('name', '') != '':
                approval[0]['project_name'] = codeName[0].get('name', '')
        return approval