# encoding: utf-8
import copy
import json
import logging
import multiprocessing
import os
import re
import sys
import time
import traceback
from concurrent.futures.thread import ThreadPoolExecutor
from datetime import datetime, timedelta
from decimal import Decimal
from pprint import pprint
from uuid import uuid4

from bs4 import BeautifulSoup
from tablestore import BoolQuery, RangeQuery, TermQuery, MatchPhraseQuery, Sort, FieldSort, ColumnReturnType, SortOrder, \
    SearchQuery, ColumnsToGet

sys.path.append(os.path.abspath(os.path.dirname(__file__)) + "/../../../")
from BaseDataMaintenance.common.Utils import getRow_ots, getUnifyMoney
from BaseDataMaintenance.dataSource.source import getConnect_ots, getConnect_ots_capacity
from BaseDataMaintenance.model.ots.approval_project import approval_project
from BaseDataMaintenance.model.ots.document import Document


def log(*args):
    print_str = ""
    print_str += str(globals().get('docid')) + " - "
    for obj in args:
        print_str += str(obj) + ' '
    logging.info(print_str[:-1])


approval_col_dict = {
    "doctitle": "公告标题",
    "page_time": "公告时间",
    "province": "省份",
    "city": "城市",
    "district": "地区",

    "approval_items": "审批事项",
    "approval_result": "审批结果",
    "declare_company": "申报单位",
    "construct_company": "建设单位",
    "evaluation_agency": "环评机构",
    "approver": "审批部门",
    "compilation_unit": "编制单位",
    "publisher": "发布单位",

    "total_tenderee_money": "总投资",
    "construction_scale": "建设规模",
    "proportion": "建筑面积",
    "use_area": "用地面积",

    "doc_num": "审批文号",

    "legal_person": "项目法人",
    "moneysource": "资金来源",
    "money_use": "资金构成",
    "env_invest": "环保投资",
    "phone": "电话",
    "pro_type": "申报类型",
    "project_addr": "项目地址",
    "project_code": "项目编号",
    "project_name": "项目名称",
    "properties": "建设性质",
    "time_commencement": "开工时间",
    "time_completion": "竣工时间",
    "time_declare": "申报时间",

    "year_limit": "建设年限",

    "time_approval": "审批时间",
    "time_release": "发布日期"
}

approval_cols = [
    'uuid',
    'docids',
    'area',
    'province',
    'city',
    'district',
    'source_stage',
    'source_type',
    'approval_items',
    'approval_result',
    'approver',
    'construct_company',
    'construct_company_code',
    'construction_scale',
    'declare_company',
    'doc_nums',
    'evaluation_agency',
    'legal_person',
    'phone',
    'pro_type',
    'properties',
    'time_declare',
    'year_limit',
    'compilation_unit',
    'publisher',
    'time_approval',
    'moneysource',
    'project_addr',
    'project_name',
    'time_commencement',
    'time_completion',
    'time_release',
    'env_invest',
    'proportion',
    'length',
    'use_area',
    'construct_company_province',
    'construct_company_city',
    'construct_company_district',
    'money_use',
    'declare_type',
    'page_time',
    'doctitle',
    'project_codes',
    'total_tenderee_money',
    'total_tenderee_money_unit',

    'docids_cnt',
    'merge_uuids',
]


def get_enterprise_area(ots_client, name):
    bool_query = BoolQuery(must_queries=[
        TermQuery("name", name)
    ])
    rows, next_token, total_count, is_all_succeed = ots_client.search("enterprise", "enterprise_index",
                                                                      SearchQuery(bool_query),
                                                                      ColumnsToGet(["province", "city", "district"],
                                                                                   return_type=ColumnReturnType.SPECIFIED))
    list_data = getRow_ots(rows)
    _d = {}
    if len(list_data) > 0:
        _d["province"] = list_data[0].get("province", "")
        _d["city"] = list_data[0].get("city", "")
        _d["district"] = list_data[0].get("district", "")
    return _d


def area_count(_d):
    keys = ["province", "city", "district"]
    return sum([1 if _d.get(k, "") not in ("", "全国", "未知") else 0 for k in keys])


def fix_area(ots_client, appr):
    if appr.get("district", "") != "":
        return
    declare_company = appr.get("declare_company", "")
    _d = get_enterprise_area(ots_client, declare_company)
    if area_count(_d) > area_count(appr):
        appr.update(_d)

    construct_company = appr.get("construct_company", "")
    _d = get_enterprise_area(ots_client, construct_company)
    print('get_enterprise_area _d', _d)
    if area_count(_d) > area_count(appr):

        appr.update(_d)

    approver = appr.get("approver", "")
    _d = get_enterprise_area(ots_client, approver)
    if area_count(_d) > area_count(appr):
        appr.update(_d)

    compilation_unit = appr.get("compilation_unit", "")
    _d = get_enterprise_area(ots_client, compilation_unit)
    if area_count(_d) > area_count(appr):
        appr.update(_d)

    publisher = appr.get("publisher", "")
    _d = get_enterprise_area(ots_client, publisher)
    if area_count(_d) > area_count(appr):
        appr.update(_d)


class APPredictor:
    def __init__(self):
        pass

    def chinese_to_arabic(self, ch_str):
        chinese_number_dict = {
            '一': 1,
            '二': 2,
            '两': 2,
            '三': 3,
            '四': 4,
            '五': 5,
            '六': 6,
            '七': 7,
            '八': 8,
            '九': 9,
            '十': 10,
            '拾': 10,
            '百': 100,
            '千': 1000,
        }
        no_list = []
        for c in ch_str:
            if c not in chinese_number_dict.keys():
                return None
            no_list.append(chinese_number_dict.get(c))
        arabic_num = 0
        mul_no = None
        for i, no in enumerate(no_list):
            if no in [10, 100, 1000]:
                if mul_no is None:
                    arabic_num += no
                else:
                    arabic_num += no * mul_no
                mul_no = None
            else:
                mul_no = no
        if mul_no:
            arabic_num += mul_no

        return arabic_num

    def standard_properties(self, d):
        # properties 建设性质 未披露 迁建 新建 技术改造 改建 其他 扩建
        # 未分类 改扩建 技改 技改及其他 拆建 装修装饰 迁扩建 其他 其它
        # 新建（迁建）□改建□扩建□技术改造
        # 新建建设类项目 改建建设类项目 新建工程 新建，
        # 新建，建设内容...
        # 扩建，...
        # 新建□改扩建□技改☑
        # （1）新建√（2）改扩建（3）技改（4）迁建
        # 新建，占地面积106.4亩项组织组建力度，今年以来，该区走访以业务关系为纽带，以点带面 未披露
        properties_type_dict = {
            '未披露': '未披露',
            '迁建': '迁建',
            '新建': '新建',
            '技术改造': '技术改造',
            '技改': '技术改造',
            '改建': '改建',
            '其他': '其他',
            '其它': '其他',
            '扩建': '扩建',
            '拆建': '拆建',
            '改扩建': '改建,扩建',
            '迁扩建': '迁建,扩建',
            '装修装饰': '装修装饰',
            '新建（迁建）': '新建,迁建',
            '迁改扩建': '迁建,改建,扩建',
            '装修工程': '装修装饰',
            '旧路改造': '改建',
            '新建（迁扩建）': '新建,迁建,扩建',
            '原规模技改': '技术改造',
            '改扩建（产能核增）': '改扩建'
        }
        p_base_type = ['未披露', '迁建', '新建', '技术改造', '改建', '其他', '扩建', '拆建', '装修装饰']
        p_all_type = list(properties_type_dict.keys())
        p_all_type.sort(key=lambda x: len(x), reverse=True)
        true_char = '[☑■√R◼回]'
        false_char = '[□£☐口]'
        char_reg = '[☑■√R◼回□£☐口]'
        type_reg = '|'.join(p_all_type)
        # p_type_reg_list = [
        #     '(?P<t1>{})(项目|工程|建设类项目|及其他|建设类|（补办）)'.format(type_reg),
        #     f'^[\u4e00-\u9fff]{{2,4}}(?P<t1>{type_reg})$',
        #     f'^(?P<t1>{type_reg})[、](?P<t2>{type_reg})$',
        #     '(?P<t1>{})(（补办）|工程|)[,，;；](?!{}).*'.format(type_reg, type_reg),
        #     f'^（(?P<c1>{char_reg})?）(?P<t1>{type_reg})（(?P<c2>{char_reg})?）(?P<t2>{type_reg})（(?P<c3>{char_reg})?）(?P<t3>{type_reg})$',
        #     f'^(?P<c1>{char_reg}){{0,2}}(?P<t1>{type_reg})(?P<c2>{char_reg}){{0,2}}(?P<t2>{type_reg})?(?P<c3>{char_reg}){{0,2}}(?P<t3>{type_reg})?(?P<c4>{char_reg}){{0,2}}(?P<t4>{type_reg})?(?P<c5>{char_reg}){{0,2}}(?P<t5>{type_reg})?(?P<c6>{char_reg}){{0,2}}(?P<t6>{type_reg})?',
        # ]

        # python3.5 不支持f表达式
        p_type_reg_list = [
            '(?P<t1>{})(项目|工程|建设类项目|及其他|建设类|（补办）)'.format(type_reg),
            '^[\u4e00-\u9fff]{{2,4}}(?P<t1>{})$'.format(type_reg),
            '^(?P<t1>{})[、](?P<t2>{})$'.format(type_reg, type_reg),
            '(?P<t1>{})(（补办）|工程|)[,，;；](?!{}).*'.format(type_reg, type_reg),
            '^（(?P<c1>{})?）(?P<t1>{})（(?P<c2>{})?）(?P<t2>{})（(?P<c3>{})?）(?P<t3>{})$'
                .format(char_reg, type_reg, char_reg, type_reg, char_reg, type_reg),
            '^(?P<c1>{}){{0,2}}(?P<t1>{})(?P<c2>{}){{0,2}}(?P<t2>{})?(?P<c3>{}){{0,2}}(?P<t3>{})?(?P<c4>{}){{0,2}}(?P<t4>{})?(?P<c5>{}){{0,2}}(?P<t5>{})?(?P<c6>{}){{0,2}}(?P<t6>{})?'
                .format(char_reg, type_reg, char_reg, type_reg, char_reg, type_reg, char_reg, type_reg, char_reg, type_reg, char_reg, type_reg),
        ]

        # for d in dict_list:
        properties = str(d.get('properties'))
        properties = re.sub(' ', '', properties)
        match_flag = 0
        if properties not in ['None', None, '']:
            if properties in p_all_type:
                d['properties'] = properties
                match_flag = 1
            else:
                for index, reg in enumerate(p_type_reg_list):
                    match = re.search(reg, properties)
                    if match:
                        match_flag = 1
                        keys = list(match.groupdict().keys())
                        # 普通文本类型
                        if len(keys) == 1 and keys[0] == 't1':
                            t = match.groupdict().get('t1', '')
                            t = properties_type_dict.get(t, '')
                            d['properties'] = t
                            break
                        if len(keys) == 1 and 't' in keys:
                            t = match.groupdict().get(keys[0], '')
                            t = properties_type_dict.get(t, '')
                            d['properties'] = t
                            break
                        if len(keys) == 2 and 't1' in keys and 't2' in keys:
                            t_list = [match.groupdict().get('t1', ''), match.groupdict().get('t2', '')]
                            t_list = [properties_type_dict.get(x, '') for x in t_list]
                            t_list.sort(key=lambda x: x)
                            d['properties'] = ','.join(t_list)
                            break
                        # 括号打勾类型
                        if index == 4:
                            if match.groupdict().get('c1'):
                                t = match.groupdict().get('t1', '')
                            elif match.groupdict().get('c2'):
                                t = match.groupdict().get('t2', '')
                            elif match.groupdict().get('c3'):
                                t = match.groupdict().get('t3', '')
                            else:
                                t = '未披露'
                            t = properties_type_dict.get(t, '')
                            d['properties'] = t
                            break

                        # 选项框类型
                        match = re.finditer(true_char, properties)
                        match = list(match)
                        t_list = []
                        for m in match:
                            for t in p_all_type:
                                if properties[m.end():m.end() + len(t)] == t:
                                    t_list.append(t)
                                    break
                        if t_list:
                            t_list = ','.join([properties_type_dict.get(x) for x in t_list])
                            t_list = t_list.split(',')
                            t_list.sort(key=lambda x: x)
                            d['properties'] = ','.join(t_list)
                        else:
                            d['properties'] = '未披露'
                        break
        if match_flag and ',' not in d.get('properties'):
            t_list = properties_type_dict.get(d.get('properties'), '').split(',')
            t_list.sort(key=lambda x: x)
            d['properties'] = ','.join(t_list)

        if not match_flag and d.get('properties') not in ['None', None, '']:
            # 规则匹配不到，判断是否有选项框
            match = re.finditer(true_char, properties)
            match = list(match)
            t_list = []
            for m in match:
                for t in p_all_type:
                    if properties[m.end():m.end() + len(t)] == t:
                        t_list.append(t)
                        break
            if t_list:
                t_list = ','.join([properties_type_dict.get(x) for x in t_list])
                t_list = t_list.split(',')
                t_list.sort(key=lambda x: x)
                d['properties'] = ','.join(t_list)
            else:
                d['properties'] = ''

        if d.get('properties') in ['None', None, '']:
            d['properties'] = '未披露'
        return d

    def standard_approval_result(self, d):
        # approval_result 审批结果
        # approval_result_type_dict = {
        #     '不予许可/未通过': '不予许可|不准予许可|办结（不通过）|不符合产业政策已撤销|办结（受理不通过）|办结（办理不通过）|不予受理|无需受理|受理不通过|不予备案|拒收|退回|驳回|退件|退办|不合格|不同意',
        #     '准予许可/通过': '准予许可|许可/同意|已审[批核]|审[批核]通过|已审[批核]通过|批准|许可|合格|同意|准予备案',
        #     '在办状态': '在办|审批中|办理中|部门审批中|部门办理|待审批|待补全|待预审|待接件|审批|审查|部门开始办理|待确认中介机构|审[核批查]中|尚未审[核批查]|待联审|待互联网预审|待答复',
        #     '已完成审批': '批复|已办结|办结|批复办结|已批复|审批通过|办结成功|办结（通过）|正常办结|已办结（正式出文）|正式出文|正常|办理完成|结案',
        #     '已受理': '已受理|受理|通过|受理通过|进入受理|已接件|补齐补正受理|预审通过|已登记|已备案|确认备案|收件|接件|已立项|已提交|已处理|登记|备案（未抽中）|已发布',
        #     '其他': '补齐补正通知|补正（开始）|补齐补正受理|等等其他|申报|特别程序|撤销|补办|撤件|补正|补件|归档|送达|挂起|补齐补证',
        # }

        approval_result_type_dict = {
            '不予许可/未通过': '不予许可/未通过|不予许可|不准予许可|办结（不通过）|不符合产业政策已撤销|办结（受理不通过）|办结（办理不通过）|不予受理|无需受理|受理不通过|不予备案|拒收|退回|驳回|退件|退办|不合格|不同意|不符合要求|申请材料虚假|不满足条件|未通过审核|项目终止|资质不符|未获批准|无法办理|申请无效|已驳回|撤销申请|备案已撤销',
            '准予许可/通过': '准予许可/通过|准予许可|许可/同意|已审[批核]|审[批核]通过|已审[批核]通过|批准|许可|合格|同意|准予备案|符合规定|申请成功|符合条件|审核通过|项目获批|资质合格|可以办理|申请有效|办结|准予许可|办结（通过）|办结（准予许可）|确认备案',
            '在办状态': '在办状态|在办|审批中|办理中|部门审批中|部门办理|待审批|待补全|待预审|待接件|审批|审查|部门开始办理|待确认中介机构|审[核批查]中|尚未审[核批查]|待联审|待互联网预审|待答复|申报|特别程序申请|特别程序|待补办|规划条件核实（个人住宅）|收件|补正（结束）|申办|已提交|建设单位提交|待承办|尚未审核|拟同意|已立项|补齐补证|正在审核|审批流程中|办理流程中|等待反馈|需补充材料|正在审查|等待审批结果|处理中|审核中',
            '已完成审批': '已完成审批|批复|已办结|办结|批复办结|已批复|审批通过|办结成功|办结（通过）|正常办结|已办结（正式出文）|正式出文|正常|办理完成|结案|审批结束|审批完成|审批通过并办结|审批流程结束|审批成功|审批办结|审批完结|已发布|符合性判定|已处理|送达',
            '已受理': '已受理|受理|通过|受理通过|进入受理|已接件|补齐补正受理|预审通过|已登记|已备案|确认备案|收件|接件|已立项|已提交|已处理|登记|备案（未抽中）|已发布|已接收申请|申请已受理|受理申请|正在受理|已进入受理流程|申请受理中|受理成功|已受理待审核|已撤件',
            '其他': '补齐补正通知|补正（开始）|补齐补正受理|等等其他|申报|特别程序|撤销|补办|撤件|补正|补件|归档|送达|挂起|补齐补证|暂缓审批|暂不受理|需重新申报|暂停办理|暂无结果|需进一步研究|暂存待查|暂不通过|抽中|未抽中|许可|新设',
        }
        # 未分类： 申报 审查 办结|准予许可 特别程序申请 特别程序 撤销申请 归档 待补办 已撤件 备案已撤销 新设
        # 已驳回 办结（通过） 办结（准予许可） 部门开始办理 规划条件核实（个人住宅） 收件 退件 挂起 批准 合格 不合格 确认备案
        # 待确认中介机构 退办 归档 接件 补正（结束） 许可 抽中 未抽中 同意 补件 申办 结案 已发布 符合性判定
        # 已处理 送达 尚未审核 拟同意 已立项 待联审 已提交 待互联网预审 建设单位提交 待承办 备案（未抽中）
        # 补齐补证 已发布

        # for d in dict_list:
        approval_result = d.get('approval_result')
        new_approval_result = ""
        if approval_result:
            for t, reg in approval_result_type_dict.items():
                if re.search(reg, str(approval_result)):
                    new_approval_result = t
                    break
        d['approval_result'] = new_approval_result
        return d

    def standard_year_limit(self, d):
        # year_limit 建设年限 int类型 单位是月
        year_limit_reg_list = [
            '^(?P<year_num>\d{1,2})年$',
            '^(?P<year_num_ch>[一二三四五六七八九十]{1,2})年$',
            '^(?P<month_num>\d+)个月$',
            '^(?P<month_num_ch>[一二三四五六七八九十]{1,3})个月$',
            '^(?P<day_num>\d+)([天日]|个?日历天)$',
            '^(?P<day_num_ch>[一二三四五六七八九十])([天日]|个?日历天)$',
            '^(?P<y_range1>\d{4,})年?[-至到—一]{1,2}(?P<y_range2>\d{4,})年?$',
            '^(?P<y_range1>\d{4,})[年-]?(?P<m_range1>\d{1,2})[月-]?[-至到—一]{1,2}(?P<y_range2>\d{4,})[年-]?(?P<m_range2>\d{1,2})月?$',
            '^(?P<y_range1>\d{4,})[年-]?(?P<m_range1>\d{1,2})[月-]?(?P<d_range1>\d{1,2})日?[-至到—一]{1,2}(?P<y_range2>\d{4,})[年-]?(?P<m_range2>\d{1,2})[月-]?(?P<d_range2>\d{1,2})日?$',
            '^(?P<y_range1>\d{4,})[年-]?(?P<m_range1>\d{1,2})[月-]?(?P<d_range1>\d{1,2})日?[ ]?(?P<h_range1>\d{1,2})[时:：]?(?P<mi_range1>\d{1,2})[分:：]?(?P<s_range1>\d{1,2})(\.0)?秒?[-至到—一]{1,2}(?P<y_range2>\d{4,})[年-]?(?P<m_range2>\d{1,2})[月-]?(?P<d_range2>\d{1,2})日?[ ]?(?P<h_range2>\d{1,2})[时:：]?(?P<mi_range2>\d{1,2})[分:：]?(?P<s_range2>\d{1,2})(\.0)?秒?$',
        ]

        # for d in dict_list:
        year_limit = d.get('year_limit')
        time_commencement = d.get('time_commencement')
        time_completion = d.get('time_completion')
        if year_limit:
            new_year_limit = 0
            for reg_index, reg in enumerate(year_limit_reg_list):
                match = re.search(reg, str(year_limit))
                if not match:
                    # print('not match!!')
                    continue
                # print('reg_index', reg_index)
                if reg_index == 0:
                    new_year_limit = int(match.group('year_num')) * 12 * 30
                elif reg_index == 1:
                    new_year_limit = self.chinese_to_arabic(match.group('year_num_ch')) * 12 * 30
                elif reg_index == 2:
                    new_year_limit = int(match.group('month_num')) * 30
                elif reg_index == 3:
                    new_year_limit = self.chinese_to_arabic(match.group('month_num_ch')) * 30
                elif reg_index == 4:
                    new_year_limit = int(match.group('day_num'))
                elif reg_index == 5:
                    new_year_limit = self.chinese_to_arabic(match.group('day_num_ch'))
                elif reg_index in [6, 7, 8, 9]:
                    year1, year2 = match.groupdict().get('y_range1'), match.groupdict().get('y_range2')
                    month1, month2 = match.groupdict().get('m_range1'), match.groupdict().get('m_range2')
                    day1, day2 = match.groupdict().get('d_range1'), match.groupdict().get('d_range2')
                    if month1:
                        month1, month2 = [x if len(x) == 2 else '0' + x for x in [month1, month2]]
                    if day1:
                        day1, day2 = [x if len(x) == 2 else '0' + x for x in [day1, day2]]
                        date_str1 = '{}-{}-{}'.format(year1, month1, day1)
                        date1 = datetime.strptime(date_str1, '%Y-%m-%d')
                        date_str2 = '{}-{}-{}'.format(year2, month2, day2)
                        date2 = datetime.strptime(date_str2, '%Y-%m-%d')
                        new_year_limit = (date2 - date1).days
                    elif month1:
                        date_str1 = '{}-{}'.format(year1, month1)
                        date1 = datetime.strptime(date_str1, '%Y-%m')
                        date_str2 = '{}-{}'.format(year2, month2)
                        date2 = datetime.strptime(date_str2, '%Y-%m')
                        new_year_limit = (date2 - date1).days
                    else:
                        date1 = datetime.strptime(year1, '%Y')
                        date2 = datetime.strptime(year2, '%Y')
                        new_year_limit = (date2 - date1).days
                    # 从建设年限补充开工竣工时间
                    if not time_commencement:
                        d['time_commencement'] = datetime.strftime(date1, '%Y-%m-%d')
                    if not time_completion:
                        d['time_completion'] = datetime.strftime(date2, '%Y-%m-%d')
                break
            if new_year_limit > 0:
                d['year_limit'] = new_year_limit
            else:
                d.pop('year_limit')

        if d.get('year_limit') in [None, 'None', '', 0]:
            d['year_limit'] = ''
        return d

    def standard_time_commencement_completion(self, d):
        # time_commencement 开工时间 可以从建设年限year_limit里面提取
        # time_completion 竣工时间 可以从建设年限year_limit里面提取 不能比开工时间早
        # for d in dict_list:
        time_commencement = d.get('time_commencement')
        time_completion = d.get('time_completion')
        if time_commencement and time_completion:
            try:
                date1 = datetime.strptime(time_commencement, '%Y-%m-%d')
                date2 = datetime.strptime(time_completion, '%Y-%m-%d')
            except:
                try:
                    date1 = datetime.strptime(time_commencement, '%Y-%m')
                    date2 = datetime.strptime(time_completion, '%Y-%m')
                    d['time_commencement'] = datetime.strftime(date1, '%Y-%m-%d')
                    d['time_completion'] = datetime.strftime(date2, '%Y-%m-%d')
                except:
                    d['time_commencement'] = ''
                    d['time_completion'] = ''
                    date1, date2 = None, None
            if date1 is not None and date1 > date2:
                d['time_commencement'] = ''
                d['time_completion'] = ''
        return d

    def standard_money_or_area(self, d):
        # 建筑面积、用地面积、长度 float类型两位小数
        # 总投资、环保投资 float类型两位小数
        # for d in dict_list:
        for col in ['proportion', 'use_area', 'length']:
            value = d.get(col)
            if value:
                try:
                    d[col] = round(float(value), 2)
                except:
                    value = re.sub('[\u4e00-\u9fa5]', '', value)
                    value = re.sub('m2', '㎡', value)
                    d[col] = round(float(value[:-1]), 2)

        for col in ['total_tenderee_money', 'env_invest']:
            value = d.get(col)
            if value:
                d[col] = round(float(value), 2)
        return d

    def standard_pro_type(self, d):
        # declare_type 申报类型 从原来项目类型pro_type中拆出
        # 若成功匹配，原来的pro_type置为空
        # for d in dict_list:
        declare_type = d.get('declare_type')
        if declare_type:
            d['pro_type'] = ''
        return d

    def extract_env_invest(self, content):
        """
        环保投资

        :param content:
        :return:
        """
        pattern = "环保投资[大概约为是:：]*(?P<invs>\d+(\.\d+)?万?元)"
        match = re.search(pattern, content)
        if match is not None:
            invest = match.groupdict().get("invs", "")
            money = getUnifyMoney(invest)
            if money > 0:
                return money
        return ""

    def extract_money_use(self, content):
        """
        资金构成

        :param content:
        :return:
        """
        list_sentences = re.split("，|。", content)
        list_data = []
        pattern = "^.{,20}[费用|预备费|费][大概约为是:：]*\d+(\.\d+)?万?元.{,20}$"
        for sentence in list_sentences:
            match = re.search(pattern, sentence)
            if match is not None:
                list_data.append(sentence)
        return "，".join(list_data)

    def extract_use_area(self, content, has_preffix=True):
        """
        用地面积

        :param content:
        :param has_preffix:
        :return:
        """
        if not content:
            return "", ""
        # log("content")
        # log(content)
        suffix = "[大概约为是:：【\[\s]*[\d,]+(\.\d+)?[十百千万亿]*([\]】平方kK千万公㎡mM米里顷亩]+2?))"
        # reg_dict = {
        #     0: "(?P<proportion>(总((用地|占地|使用)(面积|规模)|长|长度))" + suffix,
        #     1: "(?P<proportion>((用地|占地|使用)(面积|规模)|全长)" + suffix,
        #     # 2: "(?P<proportion>((用地|占地|使用)?面积)" + suffix
        #     2: "(?P<proportion>((用地|占地|使用)面积)" + suffix
        # }
        reg_dict = {
            0: "(?P<proportion>(总((用地|占地|使用)(面积|规模)))" + suffix,
            1: "(?P<proportion>((用地|占地|使用)(面积|规模))" + suffix,
            2: "(?P<proportion>((用地|占地|使用)面积)" + suffix
        }

        if not has_preffix:
            reg_dict[3] = "(?P<proportion>" + suffix

        _proportion = ""
        for i in range(len(list(reg_dict.keys()))):
            if _proportion:
                break
            _pattern = reg_dict.get(i)
            # logging.info('content ' + str(content))
            match = re.search(_pattern, str(content))
            if match:
                _proportion = match.groupdict().get("proportion", "")

        if not _proportion:
            return "", ""

        # 统一格式
        multiple_cnt = 1
        digit = ""

        # 确定具体数字
        match = re.search('(?P<d1>[\d,]+)(?P<d2>(\.\d+)?)', _proportion)
        if match:
            # logging.info(str(_proportion) + '  ' + str(match.group()))
            d1 = match.group('d1')
            d2 = match.group('d2')
            try:
                d1 = int(re.sub(',', '', d1))
            except:
                return "", ""
            if d2:
                d2 = Decimal(d2[1:]) / Decimal(str(int(10 ** len(d2[1:]))))
                # print('d1, d2', d1, d2)
                d1 += d2
            digit = d1
        # print('digit', digit)

        # 确定中文倍数
        _proportion2 = re.sub(re.escape(match.group()), '', _proportion)
        match = re.search('[十百千万亿]+', _proportion2)
        _dict = {'十': 10, '百': 100, '千': 1000, '万': 10000, '亿': 100000000}
        if match:
            for c in match.group():
                multiple_cnt *= _dict.get(c)
            _proportion3 = re.sub(re.escape(match.group()), '', _proportion2)
        else:
            _proportion3 = _proportion2
        # print('multiple_cnt2', multiple_cnt)

        # 确定面积/长度
        match = re.search('[平方㎡顷亩]+|[mM]2', _proportion3)
        if match:
            unit = '㎡'
        else:
            unit = 'm'

        # 确定单位倍数
        match = re.search('[平方kK千万公㎡mM米里顷亩]+2?', _proportion3)
        if match:
            if unit == 'm':
                if re.search('[kK千公]', match.group()):
                    multiple_cnt *= 1000
                elif re.search('[里]', match.group()):
                    multiple_cnt *= Decimal(str(500))
            else:
                if '亩' in match.group():
                    multiple_cnt *= Decimal(str(666.67))
                elif '顷' in match.group():
                    multiple_cnt *= 10000
                elif re.search('千米|公里|k[mM㎡]', match.group()):
                    multiple_cnt *= 1000000
        # print('multiple_cnt1', multiple_cnt)

        # 拼接
        digit = str(digit * multiple_cnt) + unit

        return _proportion, digit

    def extract_proportion(self, content, has_preffix=True):
        if not content:
            return "", ""
        # log("content")
        # log(content)
        suffix = "[大概约为是:：【\[\s]*[\d,]+(\.\d+)?[十百千万亿]*([\]】平方kK千万公㎡mM米里顷亩]+2?))"
        reg_dict = {
            0: "(?P<proportion>(总((建筑|建设)(面积|规模)))" + suffix,
            1: "(?P<proportion>((建筑|建设)(面积|规模))" + suffix,
            2: "(?P<proportion>((建筑|建设|区域)?面积|项目规模)" + suffix
        }

        if not has_preffix:
            reg_dict[3] = "(?P<proportion>" + suffix

        _proportion = ""
        for i in range(len(list(reg_dict.keys()))):
            if _proportion:
                break
            _pattern = reg_dict.get(i)
            # logging.info('content ' + str(content))
            match = re.search(_pattern, str(content))
            if match:
                _proportion = match.groupdict().get("proportion", "")

        if not _proportion:
            return "", ""

        # 统一格式
        multiple_cnt = 1
        digit = ""

        # 确定具体数字
        match = re.search('(?P<d1>[\d,]+)(?P<d2>(\.\d+)?)', _proportion)
        if match:
            # logging.info(str(_proportion) + '  ' + str(match.group()))
            d1 = match.group('d1')
            d2 = match.group('d2')
            try:
                d1 = int(re.sub(',', '', d1))
            except:
                return "", ""
            if d2:
                d2 = Decimal(d2[1:]) / Decimal(str(int(10 ** len(d2[1:]))))
                # print('d1, d2', d1, d2)
                d1 += d2
            digit = d1
        # print('digit', digit)

        # 确定中文倍数
        _proportion2 = re.sub(re.escape(match.group()), '', _proportion)
        match = re.search('[十百千万亿]+', _proportion2)
        _dict = {'十': 10, '百': 100, '千': 1000, '万': 10000, '亿': 100000000}
        if match:
            for c in match.group():
                multiple_cnt *= _dict.get(c)
            _proportion3 = re.sub(re.escape(match.group()), '', _proportion2)
        else:
            _proportion3 = _proportion2
        # print('multiple_cnt2', multiple_cnt)

        # 确定面积/长度
        match = re.search('[平方㎡顷亩]+|[mM]2', _proportion3)
        if match:
            unit = '㎡'
        else:
            unit = 'm'

        # 确定单位倍数
        match = re.search('[平方kK千万公㎡mM米里顷亩]+2?', _proportion3)
        if match:
            if unit == 'm':
                if re.search('[kK千公]', match.group()):
                    multiple_cnt *= 1000
                elif re.search('[里]', match.group()):
                    multiple_cnt *= Decimal(str(500))
            else:
                if '亩' in match.group():
                    multiple_cnt *= Decimal(str(666.67))
                elif '顷' in match.group():
                    multiple_cnt *= 10000
                elif re.search('千米|公里|k[mM㎡]', match.group()):
                    multiple_cnt *= 1000000
        # print('multiple_cnt1', multiple_cnt)

        # 拼接
        digit = str(digit * multiple_cnt) + unit

        return _proportion, digit

    def extract_length(self, content, has_preffix=True):
        if not content:
            return "", ""
        # log("content")
        # log(content)
        suffix = "[大概约为是:：【\[\s]*[\d,]+(\.\d+)?[十百千万亿]*([\]】平方kK千万公㎡mM米里顷亩]+2?))"
        reg_dict = {
            0: "(?P<proportion>(总((建筑|建设)长|长度))" + suffix,
            1: "(?P<proportion>((建筑|建设)全长)" + suffix,
            2: "(?P<proportion>((建筑|建设|区域)?全长)" + suffix
        }

        if not has_preffix:
            reg_dict[3] = "(?P<proportion>" + suffix

        _proportion = ""
        for i in range(len(list(reg_dict.keys()))):
            if _proportion:
                break
            _pattern = reg_dict.get(i)
            # logging.info('content ' + str(content))
            match = re.search(_pattern, str(content))
            if match:
                _proportion = match.groupdict().get("proportion", "")

        if not _proportion:
            return "", ""

        # 统一格式
        multiple_cnt = 1
        digit = ""

        # 确定具体数字
        match = re.search('(?P<d1>[\d,]+)(?P<d2>(\.\d+)?)', _proportion)
        if match:
            # logging.info(str(_proportion) + '  ' + str(match.group()))
            d1 = match.group('d1')
            d2 = match.group('d2')
            try:
                d1 = int(re.sub(',', '', d1))
            except:
                return "", ""
            if d2:
                d2 = Decimal(d2[1:]) / Decimal(str(int(10 ** len(d2[1:]))))
                # print('d1, d2', d1, d2)
                d1 += d2
            digit = d1
        # print('digit', digit)

        # 确定中文倍数
        _proportion2 = re.sub(re.escape(match.group()), '', _proportion)
        match = re.search('[十百千万亿]+', _proportion2)
        _dict = {'十': 10, '百': 100, '千': 1000, '万': 10000, '亿': 100000000}
        if match:
            for c in match.group():
                multiple_cnt *= _dict.get(c)
            _proportion3 = re.sub(re.escape(match.group()), '', _proportion2)
        else:
            _proportion3 = _proportion2
        # print('multiple_cnt2', multiple_cnt)

        # 确定面积/长度
        match = re.search('[平方㎡顷亩]+|[mM]2', _proportion3)
        if match:
            unit = '㎡'
        else:
            unit = 'm'

        # 确定单位倍数
        match = re.search('[平方kK千万公㎡mM米里顷亩]+2?', _proportion3)
        if match:
            if unit == 'm':
                if re.search('[kK千公]', match.group()):
                    multiple_cnt *= 1000
                elif re.search('[里]', match.group()):
                    multiple_cnt *= Decimal(str(500))
            else:
                if '亩' in match.group():
                    multiple_cnt *= Decimal(str(666.67))
                elif '顷' in match.group():
                    multiple_cnt *= 10000
                elif re.search('千米|公里|k[mM㎡]', match.group()):
                    multiple_cnt *= 1000000
        # print('multiple_cnt1', multiple_cnt)

        # 拼接
        digit = str(digit * multiple_cnt) + unit

        return _proportion, digit

    def extract_declare_type(self, pro_type):
        """
        申报类型从pro_type项目类型中提取

        :param pro_type:
        :return:
        """
        declare_type_list = ['立项', '审批', '备案', '核准', '未知', ]
        declare_type = ''
        if pro_type:
            reg = '(?P<type>{})'.format('|'.join(declare_type_list))
            match = re.search(reg, str(pro_type))
            if match:
                declare_type = match.group('type')
        return declare_type

    def run_standard(self, rs_dic):
        # 字段标准化 2025-01-13
        # rs_dic = self.standard_data([rs_dic])[0]
        rs_dic = self.standard_properties(rs_dic)
        rs_dic = self.standard_approval_result(rs_dic)
        rs_dic = self.standard_year_limit(rs_dic)
        rs_dic = self.standard_time_commencement_completion(rs_dic)
        rs_dic = self.standard_money_or_area(rs_dic)
        rs_dic = self.standard_pro_type(rs_dic)
        return rs_dic


class APProduction:
    def __init__(self, test_mode=0, update=1, show=0):
        self.ots_client = getConnect_ots()
        self.ots_capacity = getConnect_ots_capacity()
        self.test_mode = test_mode
        self.update = update
        self.show = show

        # 处理到的时间
        self.opertime_txt = os.path.abspath(os.path.dirname(__file__)) + '/opertime.txt'
        with open(self.opertime_txt, 'r') as f:
            self.opertime_last = f.read()

        self.document_cols = ["docid", "doctitle", "project_name", "page_time",
                              "project_code", "approval_json", "extract_json",
                              'total_tenderee_money', 'total_tenderee_money_unit',
                              'time_approval',
                              'construct_company', 'construct_company_code',
                              'compilation_unit', 'evaluation_agency', 'declare_company',
                              'approver', 'publisher'
                              ]

        self.ap_predictor = APPredictor()

        self.none_list = [None, '', '-', '全国', '未知', "None"]

    def ap_data_flow(self, docid):
        flow_start_time = time.time()
        log('process docid ' + str(docid))

        # 获取公告的数据
        # 单个审批公告可能同时发布多个不同的审批项目
        start_time = time.time()
        one_docid_approvals = self.get_approval_data_from_document_by_docid(docid)
        if self.show:
            log('len(one_docid_approvals)', len(one_docid_approvals))
            print('one_docid_approvals', one_docid_approvals)
        # log('time1 ' + str(round(time.time()-start_time, 2)))

        # 跳过单个审批公告多个审批记录的
        if len(one_docid_approvals) > 1:
            log('跳过单个审批公告多个审批记录的')
            print('len(one_docid_approvals)', len(one_docid_approvals))
            return []

        start_time = time.time()
        match_approvals_list = self.get_data_from_document_by_rules(one_docid_approvals)
        if self.show:
            log('len(match_approvals_list)', len(match_approvals_list))
            # print('match_approvals_list', match_approvals_list)
        log('time2 ' + str(round(time.time()-start_time, 2)))
        if match_approvals_list and len(match_approvals_list[0]) == 0:
            log('by_rules cnt too many! return')
            return []

        start_time = time.time()
        merge_approvals_list = self.merge_approval(match_approvals_list, one_docid_approvals)
        if self.show:
            log('len(merge_approvals_list)', len(merge_approvals_list))
        # log('time3 ' + str(round(time.time()-start_time, 2)))

        start_time = time.time()
        approval_project_list = self.generate_project(merge_approvals_list)
        if self.show:
            log('len(approval_project_list)', len(approval_project_list))
            for approval_project in approval_project_list:
                print(approval_project.get('uuid'), approval_project.get('docids_cnt'))
                print(approval_project.get('docids'))
        # log('time4 ' + str(round(time.time()-start_time, 2)))

        start_time = time.time()
        match_projects_list = self.find_exist_projects_by_docid(approval_project_list)
        if self.show:
            log('len(match_projects_list)', len(match_projects_list))
            for match_projects in match_projects_list:
                print([x.get('uuid') for x in match_projects], len(match_projects))
        # log('time5 ' + str(round(time.time()-start_time, 2)))

        start_time = time.time()
        merge_project_list = self.merge_project(match_projects_list)
        if self.show:
            log('len(merge_project_list)', len(merge_project_list))
        # log('time6 ' + str(round(time.time()-start_time, 2)))

        start_time = time.time()
        self.update_project(merge_project_list)
        # log('time7 ' + str(round(time.time()-start_time, 2)))

        log('approval_project_list len:', len(approval_project_list),
            'cost:', round(time.time() - flow_start_time, 2))
        return approval_project_list

    def get_approval_docid_from_document(self, max_cnt=20):
        # 查询这个时间点数量，后续查数据数量需略微大于该时间点数量，否则会死循环
        bool_query = BoolQuery(must_queries=[
            RangeQuery("status", 201, 301),
            TermQuery("persistence_time", self.opertime_last),
            TermQuery("docchannel", 302),
        ])
        total_cnt = self.search_util("document", "document_index",
                                     bool_query, ['docid'], None,
                                     only_return_total_cnt=1)
        if max_cnt <= total_cnt:
            max_cnt = total_cnt + 1
            log('reset max_cnt', str(max_cnt))

        bool_query = BoolQuery(must_queries=[
            RangeQuery("status", 201, 301),
            RangeQuery("persistence_time", self.opertime_last, None),
            TermQuery("docchannel", 302),
        ])
        # 需时间正序执行
        asc_flag = 1
        row_list = self.search_util("document", "document_index", bool_query,
                                    ["docid", "persistence_time"], "persistence_time",
                                    limit=max_cnt, asc=asc_flag)

        id_list = []
        persistence_time_list = []
        for _data in row_list:
            id_list.append(_data.get("docid"))
            persistence_time_list.append(_data.get("persistence_time"))
        return id_list, persistence_time_list

    def get_approval_data_from_document_by_docid(self, docid):
        bool_query = BoolQuery(must_queries=[
            TermQuery("docid", docid),
            RangeQuery('status', 201, 301)
        ])

        if self.test_mode:
            rows = self.search_util("document", "document_index",
                                    bool_query, self.document_cols, 'page_time')
        else:
            self.document_cols += ['persistence_time']
            rows = self.search_util("document", "document_index",
                                    bool_query, self.document_cols, 'persistence_time')

        # 处理下数据，后期可能删除；因为按docid搜只有一个或没有，直接返回
        if len(rows) == 1:
            # 这里会把json里的都处理，可能会产生多个不同审批项目，都返回
            approval_list = self.process_approval_json(rows[0])
        else:
            approval_list = []
        return approval_list

    def get_data_from_document_by_rules(self, approvals):
        def task(approval):
            doc_num = approval.get("doc_num", "")
            doctitle = approval.get("doctitle", "")
            project_name = approval.get("project_name", "")
            project_code = approval.get("project_code", "")
            docid = approval.get("docid")
            should_queries = []
            if doc_num != "":
                should_queries.append(MatchPhraseQuery("doctitle", doc_num))
                should_queries.append(MatchPhraseQuery("doctextcon", doc_num))
                should_queries.append(MatchPhraseQuery("attachmenttextcon", doc_num))
            if doctitle != "":
                should_queries.append(MatchPhraseQuery("doctitle", doctitle))
                should_queries.append(MatchPhraseQuery("doctextcon", doctitle))
                should_queries.append(MatchPhraseQuery("attachmenttextcon", doctitle))
            if project_name != "":
                should_queries.append(MatchPhraseQuery("doctitle", project_name))
                should_queries.append(MatchPhraseQuery("doctextcon", project_name))
                should_queries.append(MatchPhraseQuery("attachmenttextcon", project_name))
            if project_code != "":
                should_queries.append(MatchPhraseQuery("doctitle", project_code))
                should_queries.append(MatchPhraseQuery("doctextcon", project_code))
                should_queries.append(MatchPhraseQuery("attachmenttextcon", project_code))
            _query = BoolQuery(should_queries=should_queries, must_not_queries=[TermQuery("docid", docid)])

            # # 两两组合
            # should_doc_num = []
            # should_doctitle = []
            # should_project_name = []
            # should_project_code = []
            # if doc_num != "":
            #     should_doc_num.append(MatchPhraseQuery("doctitle", doc_num))
            #     should_doc_num.append(MatchPhraseQuery("doctextcon", doc_num))
            #     should_doc_num.append(MatchPhraseQuery("attachmenttextcon", doc_num))
            # if doctitle != "":
            #     should_doctitle.append(MatchPhraseQuery("doctitle", doctitle))
            #     should_doctitle.append(MatchPhraseQuery("doctextcon", doctitle))
            #     should_doctitle.append(MatchPhraseQuery("attachmenttextcon", doctitle))
            # if project_name != "":
            #     should_project_name.append(MatchPhraseQuery("doctitle", project_name))
            #     should_project_name.append(MatchPhraseQuery("doctextcon", project_name))
            #     should_project_name.append(MatchPhraseQuery("attachmenttextcon", project_name))
            # if project_code != "":
            #     should_project_code.append(MatchPhraseQuery("doctitle", project_code))
            #     should_project_code.append(MatchPhraseQuery("doctextcon", project_code))
            #     should_project_code.append(MatchPhraseQuery("attachmenttextcon", project_code))
            #
            # all_should_list = []
            # if should_doc_num and should_doctitle:
            #     q = BoolQuery(must_queries=[
            #             BoolQuery(should_queries=should_doc_num),
            #             BoolQuery(should_queries=should_doctitle)
            #         ])
            #     all_should_list.append(q)
            #
            # if should_doc_num and should_project_name:
            #     q = BoolQuery(must_queries=[
            #             BoolQuery(should_queries=should_doc_num),
            #             BoolQuery(should_queries=should_project_name)
            #         ])
            #     all_should_list.append(q)
            #
            # if should_doc_num and should_project_code:
            #     q = BoolQuery(must_queries=[
            #             BoolQuery(should_queries=should_doc_num),
            #             BoolQuery(should_queries=should_project_code)
            #         ])
            #     all_should_list.append(q)
            #
            # if should_project_code and should_doctitle:
            #     q = BoolQuery(must_queries=[
            #             BoolQuery(should_queries=should_project_code),
            #             BoolQuery(should_queries=should_doctitle)
            #         ])
            #     all_should_list.append(q)
            #
            # if should_project_code and should_project_name:
            #     q = BoolQuery(must_queries=[
            #             BoolQuery(should_queries=should_project_code),
            #             BoolQuery(should_queries=should_project_name)
            #         ])
            #     all_should_list.append(q)
            #
            # if len(all_should_list) == 0:
            #     return [approval]
            #
            # _query = BoolQuery(should_queries=all_should_list)

            bool_query = BoolQuery(must_queries=[
                RangeQuery("status", 201, 301),
                _query
            ])

            # 先查数量
            total_cnt = self.search_util("document", "document_index",
                                         bool_query, ['docid'], None,
                                         only_return_total_cnt=1)
            if total_cnt >= 50:
                log('by_rules total_cnt > 50 ' + str(total_cnt))
                return []

            rows = self.search_util("document", "document_index",
                                    bool_query, self.document_cols, 'page_time')
            return [approval] + rows

        approvals_list = self.multi_thread_util(approvals, task, 3)
        return approvals_list

    def merge_approval(self, match_approvals_list, origin_approvals):
        merge_approvals_list = []
        for index, match_approvals in enumerate(match_approvals_list):
            # 这里match_approvals是从origin_approval通过规则搜出来的一批符合的
            # match_approvals origin_approval 一一对应
            origin_approval = origin_approvals[index]
            merge_approvals = []
            for _d in match_approvals:
                # 这里每个符合的审批都要把json里的都处理一遍，再判断是否跟原审批匹配
                approval_json_list = self.process_approval_json(_d)
                # approval_json_list = [y for x in approval_json_list for y in x]
                merge_approvals += self.check_approval(approval_json_list, origin_approval)
            merge_approvals = [json.loads(x) for x in list(set([json.dumps(x) for x in merge_approvals]))]
            merge_approvals_list.append(merge_approvals)

        return merge_approvals_list

    def generate_project(self, merge_approvals_list):
        approval_project_list = []
        for merge_approvals in merge_approvals_list:
            if not merge_approvals:
                continue

            project_id = str(uuid4())
            merge_approvals.sort(key=lambda x: x.get("page_time", ""), reverse=False)

            merge_approvals[0]['uuid'] = project_id
            _dict = self.get_proper_col_value(merge_approvals)

            _dict["id"] = project_id
            _dict["uuid"] = project_id

            project_dict = {}
            for col in approval_cols + ['docids', 'docids_cnt']:
                project_dict[col] = _dict.get(col, '')
            current_time = datetime.now()
            current_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
            project_dict['update_time'] = current_time
            project_dict['status'] = '1'

            approval_project_list.append(project_dict)
        return approval_project_list

    def find_exist_projects_by_docid(self, project_list):
        match_projects_list = []
        for project in project_list:
            docids = project.get('docids').split(',')
            bool_query = BoolQuery(must_queries=[
                BoolQuery(should_queries=[TermQuery("docids", x) for x in docids]),
                # BoolQuery(must_not_queries=[TermQuery('status', '404')]),
            ])
            columns = ["docid", "doctitle", "project_name", "page_time", "project_code", "approval_json",
                       "extract_json"]
            rows = self.search_util("approval_project", "approval_project_index",
                                    bool_query, columns, None)
            match_projects = [project] + rows
            match_projects_list.append(match_projects)
        return match_projects_list

    def merge_project(self, match_projects_list):
        merge_project_list = []
        for match_projects in match_projects_list:
            merge_uuids = [x.get('uuid') for x in match_projects]
            if len(merge_uuids) > 1:
                # 排除当前项目
                merge_uuids = merge_uuids[1:]
            else:
                merge_uuids = []

            merge_project = self.get_proper_col_value(match_projects)
            merge_project['merge_uuids'] = merge_uuids
            merge_project_list.append(merge_project)
        return merge_project_list

    def update_project(self, project_list):
        if self.update:
            for project in project_list:
                if self.show:
                    print('update_project dict: ')
                    pprint(project)
                _approval_project = approval_project(project)
                _approval_project.update_project(self.ots_client)
            log('update to approval_project success!', len(project_list))
        else:
            if self.show:
                for project in project_list:
                    print('update_project dict: ')
                    pprint(project)
            log('not update to approval_project!')

    def update_opertime(self, p_time_list):
        data_format = "%Y-%m-%d %H:%M:%S"
        opertime_last = datetime.strptime(self.opertime_last, data_format)
        for persistence_time in p_time_list:
            if persistence_time is None:
                continue
            opertime = datetime.strptime(persistence_time, data_format)
            if opertime > opertime_last:
                opertime_last = opertime
        opertime_last = opertime_last.strftime(data_format)
        with open(self.opertime_txt, 'w') as f:
            f.write(opertime_last)
        return opertime_last

    def process_approval_json(self, approval_dict):
        """

        :param approval_dict: 数据dict
        :param check: 是否检查与原来的审批公告属于同一审批项目
        :return:
        """

        _d = approval_dict
        approval_json = _d.get("approval_json")
        partitionkey = _d.get("partitionkey")
        docid = _d.get("docid")
        doctitle = _d.get("doctitle")
        project_name = _d.get("project_name")
        page_time = _d.get("page_time")
        has_standard = _d.get("standard")

        extract_json = _d.get("extract_json")
        # pro_type = extract_json.get("pro_type")
        # approval_result = extract_json.get("approval_result")
        _d_html = {"partitionkey": partitionkey, "docid": docid}
        _html = Document(_d_html)
        _html.fix_columns(self.ots_capacity, ["dochtmlcon"], True)
        dochtml = _html.getProperties().get("dochtmlcon", "")
        doctextcon = BeautifulSoup(dochtml, "lxml").get_text()
        attachmenttextcon = ""
        try:
            _extract = json.loads(extract_json)
            for key in _extract.keys():
                if _d.get(key) in self.none_list and _extract.get(key) not in self.none_list:
                    _d[key] = _extract.get(key)
        except Exception  as e:
            _extract = {}

        # 若要素提取已经标准化过了，不用重新跑
        if has_standard:
            proportion = _d.get('proportion')
            use_area = _d.get('use_area')
            env_invest = _d.get('env_invest')
            money_use = _d.get('money_use')
            declare_type = _d.get('declare_type')
            length = _d.get('length')
        else:
            proportion = _extract.get("pb", {}).get("proportion")
            _, use_area = self.ap_predictor.extract_use_area(doctextcon + attachmenttextcon)
            env_invest = self.ap_predictor.extract_env_invest(doctextcon + attachmenttextcon)
            money_use = self.ap_predictor.extract_money_use(doctextcon + attachmenttextcon)
            declare_type = self.ap_predictor.extract_declare_type(_d.get('pro_type'))
            _, length = self.ap_predictor.extract_length(doctextcon + attachmenttextcon)
            _d["proportion"] = proportion
            _d["use_area"] = use_area
            _d["env_invest"] = env_invest
            _d["money_use"] = money_use
            _d["declare_type"] = declare_type
            _d["length"] = length

            # 标准化
            _d = self.ap_predictor.run_standard(_d)

        approval_list = []
        if approval_json:
            approval_list = json.loads(approval_json)
            for _appr in approval_list:
                for key in _d.keys():
                    if key not in _appr:
                        _appr[key] = _d.get(key)

                fix_area(self.ots_client, _appr)

                # 建设单位信用代码、地区
                if _d.get("construct_company"):
                    new_d = self.get_construct_code_area(_d.get("construct_company"))
                    _appr.update(new_d)

                # 再从各自的项目概况中提取
                construction_scale = _d.get("construction_scale", "")
                proportion, _ = self.ap_predictor.extract_proportion(construction_scale)
                if proportion != "":
                    _appr["proportion"] = proportion
                _, use_area = self.ap_predictor.extract_use_area(construction_scale)
                if use_area != "":
                    _appr["use_area"] = use_area
                env_invest = self.ap_predictor.extract_env_invest(construction_scale)
                if env_invest != "":
                    _appr["env_invest"] = env_invest
                money_use = self.ap_predictor.extract_money_use(construction_scale)
                if money_use != "":
                    _appr["money_use"] = money_use
                # 标准化
                _appr = self.ap_predictor.run_standard(_appr)
        return approval_list

    def get_proper_col_value(self, dict_list):
        """
        获取合适的字段值

        :return:
        """
        merge_dict = {}
        if dict_list:
            merge_dict['uuid'] = dict_list[0].get('uuid')

        # 处理多值字段
        many_value_cols = ['docids', 'doc_nums', 'project_codes']
        for col in many_value_cols:
            for d in dict_list:
                if d.get(col) is not None:
                    continue
                col_values = [str(x.get(col[:-1])) for x in dict_list]
                d[col] = ','.join(col_values)
                if col == 'docids':
                    d['docids_cnt'] = len(col_values)

        # 合并多值字段
        many_value_cols = ['docids', 'doc_nums', 'project_codes', 'properties']
        for col in many_value_cols:
            value_list = []
            for d in dict_list:
                value = d.get(col)
                if value in self.none_list:
                    continue
                value_list += value.split(',')
            value_list = list(set(value_list))
            for v in self.none_list:
                if v in value_list:
                    value_list.remove(v)
            value_list.sort(key=lambda x: x)
            if col == 'properties' and len(value_list) > 1 and '未披露' in value_list:
                value_list.remove('未披露')
            max_cnt = 200
            if col == 'docids' and len(value_list) < max_cnt:
                merge_dict['docids_cnt'] = len(value_list)
            # 限制个数
            value_list = value_list[:max_cnt]
            merge_dict[col] = ','.join(value_list)

        # 数字字段
        num_cols = ['year_limit', 'env_invest', 'proportion', 'length', 'use_area', 'total_tenderee_money']
        for col in num_cols:
            max_value = 0
            for d in dict_list:
                value = d.get(col, 0)
                if value in [None, '', 'None']:
                    value = 0
                if value > max_value:
                    max_value = value
            if max_value != 0:
                merge_dict[col] = max_value
            else:
                merge_dict[col] = ''

        # 日期字段
        time_cols = ['time_commencement', 'time_completion', 'time_release', 'page_time',
                     'time_declare', 'time_approval']
        for col in time_cols:
            # if col in ['update_time']:
            #     format_string = "%Y-%m-%d %H:%M:%S"
            # else:
            format_string = "%Y-%m-%d"

            max_time = None
            for d in dict_list:
                value = d.get(col)
                try:
                    value = datetime.strptime(value, format_string)
                except:
                    continue
                if max_time is None or value > max_time:
                    max_time = value
            if max_time is not None:
                merge_dict[col] = datetime.strftime(max_time, format_string)
            else:
                merge_dict[col] = ''

        # 地区字段
        area_cols = ['area', 'province', 'city', 'district']
        area_list, province_list, city_list, district_list = [[] for _ in range(4)]
        a_p_c_d_dict = {}
        # 有些district是dict
        extra_dict_list = []
        for d in dict_list:
            district = d.get('district')
            if type(district) == dict:
                extra_dict_list.append(district)
                d['district'] = ''

        for d in dict_list + extra_dict_list:
            area = d.get('area')
            province = d.get('province')
            city = d.get('city')
            district = d.get('district')
            district_list.append(district)
            area_list.append(area)
            province_list.append(province)
            city_list.append(city)

            # 重复加1
            a_p_c_d = []
            for v in [area, province, city, district]:
                if v not in self.none_list:
                    a_p_c_d.append(v)
            a_p_c_d = '-'.join(a_p_c_d)
            if a_p_c_d in a_p_c_d_dict.keys():
                a_p_c_d_dict[a_p_c_d] += 1
            else:
                a_p_c_d_dict[a_p_c_d] = 1

        # 包含关系再加1
        keys = list(a_p_c_d_dict.keys())
        for i in range(len(keys)):
            a_p_c_d_1 = keys[i]
            for j in range(i + 1, len(keys)):
                a_p_c_d_2 = keys[j]
                if len(a_p_c_d_1) >= len(a_p_c_d_2) and a_p_c_d_2 in a_p_c_d_1:
                    a_p_c_d_dict[a_p_c_d_1] += a_p_c_d_dict.get(a_p_c_d_2)
                elif len(a_p_c_d_1) < len(a_p_c_d_2) and a_p_c_d_1 in a_p_c_d_2:
                    a_p_c_d_dict[a_p_c_d_2] += a_p_c_d_dict.get(a_p_c_d_1)

        if self.show:
            print('a_p_c_d_dict', a_p_c_d_dict)
        if '' in a_p_c_d_dict.keys():
            a_p_c_d_dict.pop('')

        # 地区匹配次数取最多
        a_p_c_d_list = [[key, value] for key, value in a_p_c_d_dict.items()]
        a_p_c_d_list.sort(key=lambda x: x[1])
        a_p_c_d = a_p_c_d_list[-1][0].split('-')
        for value in a_p_c_d:
            if value in area_list:
                merge_dict['area'] = value
            elif value in province_list:
                merge_dict['province'] = value
            elif value in city_list:
                merge_dict['city'] = value
            elif value in district_list:
                merge_dict['district'] = value

        # 地址字段，取最长的
        address_cols = ['project_addr', 'construction_scale']
        for col in address_cols:
            longer_value = ''
            for d in dict_list:
                value = d.get(col)
                if value in self.none_list:
                    continue
                if len(value) > len(longer_value):
                    longer_value = value
            merge_dict[col] = longer_value

        # 取最多的
        name_cols = ['project_name', 'doctitle']
        for col in name_cols:
            value_dict = {}
            for d in dict_list:
                value = d.get(col)
                if value in self.none_list:
                    continue
                if value in value_dict:
                    value_dict[value] += 1
                else:
                    value_dict[value] = 1
            value_list = [[k, v] for k, v in value_dict.items()]
            if value_list:
                value_list.sort(key=lambda x: (x[1], len(x[0])))
                merge_dict[col] = value_list[-1][0]

        # 剩下的为空就填
        for col in approval_cols:
            if col in many_value_cols + num_cols + time_cols + area_cols + address_cols + name_cols:
                continue
            # 按page_time排序
            dict_list.sort(key=lambda x: x.get('page_time'), reverse=True)
            for d in dict_list:
                if merge_dict.get(col) in [None, '']:
                    merge_dict[col] = d.get(col)

        # 一些改了名字的字段
        # for col in ['moneyuse']:
        #     for d in dict_list:
        #         merge_dict['money_use'] = d.get(col)

        for col in approval_cols:
            if col in area_cols:
                continue
            if merge_dict.get(col) in self.none_list:
                merge_dict[col] = ''
        return merge_dict

    def get_construct_code_area(self, name):
        bool_query = BoolQuery(must_queries=[
            TermQuery("name", name)
        ])

        cols = ["credit_code", "province", "city", "district"]
        data_list = self.search_util("enterprise", "enterprise_index", bool_query,
                                     cols, None, )
        _d = {}
        if len(data_list) > 0:
            _d["construct_company_code"] = data_list[0].get("credit_code", "")
            _d["construct_company_province"] = data_list[0].get("province", "")
            _d["construct_company_city"] = data_list[0].get("city", "")
            _d["construct_company_district"] = data_list[0].get("district", "")
        return _d

    def check_approval(self, approval_list, origin_approval):
        appr1 = origin_approval
        same_approval_list = [origin_approval]
        for appr2 in approval_list:
            check_keys = ["declare_company", "construct_company", "total_tenderee_money", "proportion", "use_area",
                          "doc_num",
                          "project_code"]
            same_count = 0
            for k in check_keys:
                if k in appr1 and k in appr2:
                    if appr1[k] == appr2[k] and appr1[k] is not None and appr1[k] != "":
                        same_count += 1

            if same_count >= 1:
                same_approval_list.append(appr2)
        return same_approval_list

    def search_util(self, table, table_index, bool_query, columns, sort_col, limit=999999, show_total=0, while_next=1,
                    asc=0, only_return_total_cnt=0):
        if sort_col:
            if asc:
                sort = Sort(sorters=[FieldSort(sort_col, SortOrder.ASC)])
            else:
                sort = Sort(sorters=[FieldSort(sort_col, SortOrder.DESC)])
        else:
            sort = None
        return_type = ColumnReturnType.SPECIFIED
        rows, next_token, total_count, is_all_succeed = self.ots_client.search(table, table_index,
                                                                               SearchQuery(bool_query, sort=sort,
                                                                                           limit=100,
                                                                                           get_total_count=True),
                                                                               ColumnsToGet(columns, return_type))
        if only_return_total_cnt:
            return total_count

        list_data = getRow_ots(rows)
        if show_total:
            print('search total_count', total_count)
        if len(list_data) >= limit:
            print('limit ', limit, len(list_data))
            return list_data[:limit]

        if while_next:
            while next_token and len(list_data) < limit:
                rows, next_token, total_count, is_all_succeed = self.ots_client.search(table, table_index,
                                                                                       SearchQuery(bool_query,
                                                                                                   next_token=next_token,
                                                                                                   limit=100,
                                                                                                   get_total_count=True),
                                                                                       ColumnsToGet(columns,
                                                                                                    return_type))
                list_data += getRow_ots(rows)
        return list_data

    def multi_thread_util(self, _list, task, threads):
        result_list = []
        # 创建一个线程池，参数为线程数量
        with ThreadPoolExecutor(max_workers=threads) as executor:
            # 使用线程池映射任务函数和参数
            futures = [executor.submit(task, _list[i]) for i in range(len(_list))]

            # 等待所有任务完成，并获取返回值
            for future in futures:
                result = future.result()  # 获取任务的返回值
                # print(f"Task returned: {result}")
                result_list.append(result)
        return result_list


def run_flow(data, test_mode=0, update=1):
    docid, test_mode, update = data
    ds = APProduction(test_mode, update)
    print('run_flow, test_mode, update', test_mode, update)
    # globals()['docid'] = data
    try:
        aps = ds.ap_data_flow(docid)
    except:
        traceback.print_exc()
        log('error ' + str(docid))
        aps = []
    return aps


if __name__ == "__main__":
    test_mode = 0
    test_docid_list = [400010986401]
    update = 1

    process_num = 7
    sleep_sec = 100
    while True:
        sleep_flag = 1
        ds = APProduction()

        if test_mode:
            tmp_data_list = test_docid_list
            persistence_time_list = []
        else:
            # 获取新增审批公告
            tmp_data_list, persistence_time_list = ds.get_approval_docid_from_document(max_cnt=50)
        log('len(tmp_data_list)', len(tmp_data_list))

        tmp_data_list = [[x, test_mode, update] for x in tmp_data_list]

        # 取不到就等待再循环
        if len(tmp_data_list) <= 0:
            print('sleep for', sleep_sec)
            time.sleep(sleep_sec)
            continue

        # 多进程合并
        mp_start_time = time.time()
        with multiprocessing.Pool(processes=process_num) as pool:
            results = pool.map(run_flow, tmp_data_list)
            all_ap_num = len(results)
            pool.close()
            pool.join()

        # 更新数据时间
        if not test_mode:
            last_time_str = ds.update_opertime(persistence_time_list)
            # 判断时间
            data_format = "%Y-%m-%d %H:%M:%S"
            last_time = datetime.strptime(last_time_str, data_format)
            now = datetime.now()
            print('*' * 30, last_time_str)
            # 执行结束的太快，等待再处理
            if now - last_time >= timedelta(seconds=sleep_sec):
                sleep_flag = 0

        log('mp process data num:', len(tmp_data_list), 'update ap num:', all_ap_num,
            'cost:', round(time.time() - mp_start_time, 2),
            'avg:', round((time.time() - mp_start_time) / len(tmp_data_list), 2))

        # 测试模式不循环
        if test_mode:
            break

        if sleep_flag:
            print('sleep for', sleep_sec)
            time.sleep(sleep_sec)
