BaseDataMaintenance/BaseDataMaintenance/common/Utils.py

'''
Created on 2018年12月20日

@author: User
'''

import numpy as np
import re
import gensim

import os

from threading import RLock

# from pai_tf_predict_proto import tf_predict_pb2
import requests

import time
from bs4 import BeautifulSoup


model_w2v = None
lock_model_w2v = RLock()

USE_PAI_EAS = False

Lazy_load = False

ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]|\x00')

import smtplib
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.utils import formataddr

ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')

def getLegal_str(_str):
    if _str is not None:
        return ILLEGAL_CHARACTERS_RE.sub("",str(_str))

import traceback
dict_server = {}

def getServer(host,username,password,reconnect=False):
    key = "%s-%s-%s"%(host,username,password)
    if key in dict_server:
        server = dict_server[key]
        if reconnect:
            server = smtplib.SMTP_SSL(host, 465)
            server.login(username,password)
    else:
        server = smtplib.SMTP_SSL(host, 465)
        server.login(username,password)
        dict_server[key] = server
    return server
from email.mime.text import MIMEText
def sendEmail(host,username,password,receivers,subject="数据导出",content="",attachs=[]):
    try:
        #处理附件
        msg = MIMEMultipart()
        msg["From"] = formataddr(["广州比地数据科技有限公司",username])
        msg["To"] = formataddr(["客户",receivers[0]])
        msg["Subject"] = subject
        message = MIMEText(content, 'plain', 'utf-8')
        for at in attachs:
            xlsfile = MIMEApplication(open(at,"rb").read())
            xlsfile.add_header("Content-Disposition","attachment",filename=('gbk', '', at.split("/")[-1]))
            log(at.split("/")[-1])
            msg.attach(xlsfile)
        server = getServer(host,username,password)
        server.sendmail(username,receivers,msg.as_string())
        log("发送邮件成功%s"%str(attachs))

    except smtplib.SMTPServerDisconnected as e:
        server = getServer(host,username,password,reconnect=True)
        server.sendmail(username,receivers,msg.as_string())
        log("发送邮件成功%s"%str(attachs))

    except Exception as e:
        traceback.print_exc()
        log("发送邮件错误%s"%str(e))
    finally:
        server.close()

mobile_pattern = re.compile("^1\d{10}$")
def recog_likeType(phone):
    if re.search(mobile_pattern,phone) is not None:
        return "mobile"
    else:
        return "phone"

def article_limit(soup,limit_words=30000):
    sub_space = re.compile("\s+")
    def soup_limit(_soup,_count,max_count=30000,max_gap=500):
        """
        :param _soup: soup
        :param _count: 当前字数
        :param max_count: 字数最大限制
        :param max_gap: 超过限制后的最大误差
        :return:
        """
        _gap = _count - max_count
        _is_skip = False
        next_soup = None
        while len(_soup.find_all(recursive=False)) == 1 and \
                _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
            _soup = _soup.find_all(recursive=False)[0]
        if len(_soup.find_all(recursive=False)) == 0:
            _soup.string = str(_soup.get_text())[:max_count-_count]
            _count += len(re.sub(sub_space, "", _soup.string))
            _gap = _count - max_count
            next_soup = None
        else:
            for _soup_part in _soup.find_all(recursive=False):
                if not _is_skip:
                    _count += len(re.sub(sub_space, "", _soup_part.get_text()))
                    if _count >= max_count:
                        _gap = _count - max_count
                        if _gap <= max_gap:
                            _is_skip = True
                        else:
                            _is_skip = True
                            next_soup = _soup_part
                            _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
                            continue
                else:
                    _soup_part.decompose()
        return _count,_gap,next_soup

    text_count = 0
    have_attachment = False
    attachment_part = None
    _attachment = soup.find("div", attrs={"class": "richTextFetch"})
    if _attachment is not None:
        _attachment.insert_before("##attachment##")
        attachment_part = _attachment
        have_attachment = True
    if not have_attachment:
        # 无附件
        if len(re.sub(sub_space, "", soup.get_text())) > limit_words:
            text_count,gap,n_soup = soup_limit(soup,text_count,max_count=limit_words,max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)

    else:
        # 有附件
        _text = re.sub(sub_space, "", soup.get_text())
        _text_split = _text.split("##attachment##")
        if len(_text_split[0])>limit_words:
            main_soup = attachment_part.parent
            main_text = main_soup.find_all(recursive=False)[0]
            text_count, gap, n_soup = soup_limit(main_text, text_count, max_count=limit_words, max_gap=500)
            while n_soup:
                text_count, gap, n_soup = soup_limit(n_soup, text_count, max_count=limit_words, max_gap=500)
        if len(_text_split[1])>limit_words:
            # attachment_html纯文本，无子结构
            if len(attachment_part.find_all(recursive=False))==0:
                attachment_part.string = str(attachment_part.get_text())[:limit_words]
            else:
                attachment_text_nums = 0
                attachment_skip = False
                for part in attachment_part.find_all(recursive=False):
                    if not attachment_skip:
                        if part.name == 'div' and 'filemd5' in part.attrs:
                            for p_part in part.find_all(recursive=False):
                                last_attachment_text_nums = attachment_text_nums
                                attachment_text_nums = attachment_text_nums + len(
                                    re.sub(sub_space, "", p_part.get_text()))
                                if not attachment_skip:
                                    if attachment_text_nums >= limit_words:
                                        p_part.string = str(p_part.get_text())[:limit_words - last_attachment_text_nums]
                                        attachment_skip = True
                                else:
                                    p_part.decompose()
                        else:
                            last_attachment_text_nums = attachment_text_nums
                            attachment_text_nums = attachment_text_nums + len(re.sub(sub_space, "", part.get_text()))
                            if attachment_text_nums >= limit_words and not attachment_skip:
                                part.string = str(part.get_text())[:limit_words - last_attachment_text_nums]
                                attachment_skip = True
                    else:
                        part.decompose()
    soup = str(soup).replace("##attachment##","")
    return soup

def soup_limit(_soup,_count,max_count=30000,max_gap=500,sub_space = re.compile("\s+")):
    """
    :param _soup: soup
    :param _count: 当前字数
    :param max_count: 字数最大限制
    :param max_gap: 超过限制后的最大误差
    :return:
    """
    _gap = _count - max_count
    _is_skip = False
    next_soup = None
    while len(_soup.find_all(recursive=False)) == 1 and \
            _soup.get_text(strip=True) == _soup.find_all(recursive=False)[0].get_text(strip=True):
        _soup = _soup.find_all(recursive=False)[0]
    if len(_soup.find_all(recursive=False)) == 0:
        _soup.string = str(_soup.get_text())[:max_count-_count]
        _count += len(re.sub(sub_space, "", _soup.string))
        _gap = _count - max_count
        next_soup = None
    else:
        for _soup_part in _soup.find_all(recursive=False):
            if not _is_skip:
                _count += len(re.sub(sub_space, "", _soup_part.get_text()))
                if _count >= max_count:
                    _gap = _count - max_count
                    if _gap <= max_gap:
                        _is_skip = True
                    else:
                        _is_skip = True
                        next_soup = _soup_part
                        _count -= len(re.sub(sub_space, "", _soup_part.get_text()))
                        continue
            else:
                _soup_part.decompose()
    return _count,_gap,next_soup

def cut_str(text_list, only_text_list, max_bytes_length=2000000):
    try:

        # 计算有格式总字节数
        bytes_length = 0
        for text in text_list:
            bytes_length += len(bytes(text, encoding='utf-8'))


        # 小于直接返回
        if bytes_length < max_bytes_length:
            return text_list

        # 全部文件连接，重新计算无格式字节数
        all_text = ""
        bytes_length = 0
        for text in only_text_list:
            bytes_length += len(bytes(text, encoding='utf-8'))
            all_text += text

        # 小于直接返回
        if bytes_length < max_bytes_length:
            return only_text_list

        # 截取字符
        all_text = all_text[:max_bytes_length//3]

        return [all_text]
    except Exception as e:
        logging.info("cut_str " + str(e))
        return text_list

def getLegal_str(_str):
    if _str is not None:
        return ILLEGAL_CHARACTERS_RE.sub("",str(_str))
    return ""

def getRow_ots_primary(row):
    _dict = dict()
    if row is None:
        return None
    for part in row.attribute_columns:
        _dict[part[0]] = part[1]
    for part in row.primary_key:
        _dict[part[0]] = part[1]
    return _dict

def timeAdd(_time,days,format="%Y-%m-%d",minutes=0):
    a = time.mktime(time.strptime(_time,format))+86400*days+60*minutes

    _time1 = time.strftime(format,time.localtime(a))
    return _time1

def getRow_ots(rows):
    list_dict = []
    for row in rows:
        _dict = dict()
        for part in row:
            for v in part:
                _dict[v[0]] = v[1]
        list_dict.append(_dict)
    return list_dict

def getw2vfilepath():
    w2vfile = os.path.dirname(__file__)+"/../wiki_128_word_embedding_new.vector"
    if os.path.exists(w2vfile):
        return w2vfile
    return "wiki_128_word_embedding_new.vector"

def getLazyLoad():
    global Lazy_load
    return Lazy_load

def get_file_name(url, headers):
    filename = ''
    if 'Content-Disposition' in headers and headers['Content-Disposition']:
        disposition_split = headers['Content-Disposition'].split(';')
        if len(disposition_split) > 1:
            if disposition_split[1].strip().lower().startswith('filename='):
                file_name = disposition_split[1].split('=')
                if len(file_name) > 1:
                    filename = file_name[1]
    if not filename and os.path.basename(url):
        filename = os.path.basename(url).split("?")[0]
    if not filename:
        return time.time()
    return filename

model_word_file = os.path.dirname(__file__)+"/../singlew2v_model.vector"
model_word = None
lock_model_word = RLock()

from decimal import Decimal
import logging
logging.basicConfig(level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
import pickle
import os

import json

#自定义jsonEncoder
class MyEncoder(json.JSONEncoder):

    def __init__(self):
        import numpy as np
        global np

    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        elif isinstance(obj, bytes):
            return str(obj, encoding='utf-8')
        elif isinstance(obj, (np.float_, np.float16, np.float32,
                              np.float64)):
            return float(obj)
        elif isinstance(obj,(np.int64,np.int32)):
            return int(obj)
        return json.JSONEncoder.default(self, obj)

vocab_word = None
vocab_words = None

file_vocab_word = "vocab_word.pk"
file_vocab_words = "vocab_words.pk"

selffool_authorization = "NjlhMWFjMjVmNWYyNzI0MjY1OGQ1M2Y0ZmY4ZGY0Mzg3Yjc2MTVjYg=="
selffool_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/selffool_gpu"
selffool_seg_authorization = "OWUwM2Q0ZmE3YjYxNzU4YzFiMjliNGVkMTA3MzJkNjQ2MzJiYzBhZg=="
selffool_seg_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/selffool_seg_gpu"
codename_authorization = "Y2M5MDUxMzU1MTU4OGM3ZDk2ZmEzYjkxYmYyYzJiZmUyYTgwYTg5NA=="
codename_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/codename_gpu"

form_item_authorization = "ODdkZWY1YWY0NmNhNjU2OTI2NWY4YmUyM2ZlMDg1NTZjOWRkYTVjMw=="
form_item_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/form"
person_authorization = "N2I2MDU2N2Q2MGQ0ZWZlZGM3NDkyNTA1Nzc4YmM5OTlhY2MxZGU1Mw=="
person_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/person"
role_authorization = "OWM1ZDg5ZDEwYTEwYWI4OGNjYmRlMmQ1NzYwNWNlZGZkZmRmMjE4OQ=="
role_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/role"
money_authorization = "MDQyNjc2ZDczYjBhYmM4Yzc4ZGI4YjRmMjc3NGI5NTdlNzJiY2IwZA=="
money_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/money"
codeclasses_authorization = "MmUyNWIxZjQ2NjAzMWJlMGIzYzkxMjMzNWY5OWI3NzJlMWQ1ZjY4Yw=="
codeclasses_url = "http://pai-eas-vpc.cn-beijing.aliyuncs.com/api/predict/codeclasses"

def viterbi_decode(score, transition_params):
    """Decode the highest scoring sequence of tags outside of TensorFlow.

    This should only be used at test time.

    Args:
      score: A [seq_len, num_tags] matrix of unary potentials.
      transition_params: A [num_tags, num_tags] matrix of binary potentials.

    Returns:
      viterbi: A [seq_len] list of integers containing the highest scoring tag
          indices.
      viterbi_score: A float containing the score for the Viterbi sequence.
    """
    trellis = np.zeros_like(score)
    backpointers = np.zeros_like(score, dtype=np.int32)
    trellis[0] = score[0]

    for t in range(1, score.shape[0]):
        v = np.expand_dims(trellis[t - 1], 1) + transition_params
        trellis[t] = score[t] + np.max(v, 0)
        backpointers[t] = np.argmax(v, 0)

    viterbi = [np.argmax(trellis[-1])]
    for bp in reversed(backpointers[1:]):
        viterbi.append(bp[viterbi[-1]])
    viterbi.reverse()

    viterbi_score = np.max(trellis[-1])
    return viterbi, viterbi_score

import ctypes
import inspect

def _async_raise(tid, exctype):
    """raises the exception, performs cleanup if needed"""
    tid = ctypes.c_long(tid)
    if not inspect.isclass(exctype):
        exctype = type(exctype)
    res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
    if res == 0:
        raise ValueError("invalid thread id")
    elif res != 1:
        ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
        raise SystemError("PyThreadState_SetAsyncExc failed")

def stop_thread(thread):
    _async_raise(thread.ident, SystemExit)

def limitRun(sess,list_output,feed_dict,MAX_BATCH=1024):
    len_sample = 0
    if len(feed_dict.keys())>0:
        len_sample = len(feed_dict[list(feed_dict.keys())[0]])
    if len_sample>MAX_BATCH:
        list_result = [[] for _ in range(len(list_output))]
        _begin = 0
        while(_begin<len_sample):
            new_dict = dict()
            for _key in feed_dict.keys():
                new_dict[_key] = feed_dict[_key][_begin:_begin+MAX_BATCH]
            _output = sess.run(list_output,feed_dict=new_dict)
            for _index in range(len(list_output)):
                list_result[_index].extend(_output[_index])
            _begin += MAX_BATCH
    else:
        list_result = sess.run(list_output,feed_dict=feed_dict)
    return list_result



def get_values(response,output_name):
        """
        Get the value of a specified output tensor
        :param output_name: name of the output tensor
        :return: the content of the output tensor
        """
        output = response.outputs[output_name]
        if output.dtype == tf_predict_pb2.DT_FLOAT:
            _value = output.float_val
        elif output.dtype == tf_predict_pb2.DT_INT8 or output.dtype == tf_predict_pb2.DT_INT16 or \
                output.dtype == tf_predict_pb2.DT_INT32:
            _value = output.int_val
        elif output.dtype == tf_predict_pb2.DT_INT64:
            _value = output.int64_val
        elif output.dtype == tf_predict_pb2.DT_DOUBLE:
            _value = output.double_val
        elif output.dtype == tf_predict_pb2.DT_STRING:
            _value = output.string_val
        elif output.dtype == tf_predict_pb2.DT_BOOL:
            _value = output.bool_val
        return np.array(_value).reshape(response.outputs[output_name].array_shape.dim)

def vpc_requests(url,authorization,request_data,list_outputs):
    
    
    headers = {"Authorization": authorization}
    dict_outputs = dict()
    
    response = tf_predict_pb2.PredictResponse()
    resp = requests.post(url, data=request_data, headers=headers)
    
    
    if resp.status_code != 200:
        print(resp.status_code,resp.content)
        log("调用pai-eas接口出错,authorization:"+str(authorization))
        return None
    else:
        response = tf_predict_pb2.PredictResponse()
        response.ParseFromString(resp.content)
        for _output in list_outputs:
            dict_outputs[_output] = get_values(response, _output)
        return dict_outputs

def encodeInput(data,word_len,word_flag=True,userFool=False):
    result = []
    out_index = 0
    for item in data:
        if out_index in [0]:
            list_word = item[-word_len:]
        else:
            list_word = item[:word_len]
        temp = []
        if word_flag:
            for word in list_word:
                if userFool:
                    temp.append(getIndexOfWord_fool(word))
                else:
                    temp.append(getIndexOfWord(word))
            list_append = []
            temp_len = len(temp)
            while(temp_len<word_len):
                if userFool:
                    list_append.append(0)
                else:
                    list_append.append(getIndexOfWord("<pad>"))
                temp_len += 1
            if out_index in [0]:
                temp = list_append+temp
            else:
                temp = temp+list_append
        else:
            for words in list_word:
                temp.append(getIndexOfWords(words))
                
            list_append = []
            temp_len = len(temp)
            while(temp_len<word_len):
                list_append.append(getIndexOfWords("<pad>"))
                temp_len += 1
            if out_index in [0,1]:
                temp = list_append+temp
            else:
                temp = temp+list_append
        result.append(temp)
        out_index += 1
    return result

def encodeInput_form(input,MAX_LEN=30):
    x = np.zeros([MAX_LEN])
    for i in range(len(input)):
        if i>=MAX_LEN:
            break
        x[i] = getIndexOfWord(input[i])
    return x
    

def getVocabAndMatrix(model,Embedding_size = 60):
    '''
    @summary:获取子向量的词典和子向量矩阵
    '''
    vocab = ["<pad>"]+model.index2word
    
    embedding_matrix = np.zeros((len(vocab),Embedding_size))
    for i in range(1,len(vocab)):
        embedding_matrix[i] = model[vocab[i]]
    
    return vocab,embedding_matrix

def getIndexOfWord(word):
    global vocab_word,file_vocab_word
    if vocab_word is None:
        if os.path.exists(file_vocab_word):
            vocab = load(file_vocab_word)
            vocab_word = dict((w, i) for i, w in enumerate(np.array(vocab)))
        else:
            model = getModel_word()
            vocab,_ = getVocabAndMatrix(model, Embedding_size=60)
            vocab_word = dict((w, i) for i, w in enumerate(np.array(vocab)))
            save(vocab,file_vocab_word)
    if word in vocab_word.keys():
        return vocab_word[word]
    else:
        return vocab_word['<pad>']
        
def getIndexOfWords(words):
    global vocab_words,file_vocab_words
    if vocab_words is None:
        if os.path.exists(file_vocab_words):
            vocab = load(file_vocab_words)
            vocab_words = dict((w, i) for i, w in enumerate(np.array(vocab)))
        else:
            model = getModel_w2v()
            vocab,_ = getVocabAndMatrix(model, Embedding_size=128)
            vocab_words = dict((w, i) for i, w in enumerate(np.array(vocab)))
            save(vocab,file_vocab_words)
    if words in vocab_words.keys():
        return vocab_words[words]
    else:
        return vocab_words["<pad>"]

def isCellphone(phone):
    if phone is not None and re.search("^1\d{10}$",str(phone)) is not None:
        return True
    return False

def popNoneFromDict(_dict):
    list_pop = []
    for k,v in _dict.items():
        if v is None or v=="":
            list_pop.append(k)
    for k in list_pop:
        _dict.pop(k)
    return _dict


pattern_attachment = re.compile("\.(?P<attachment>jpg|jpeg|png|swf|tif|pdf|doc|docx|xls|xlsx|zip|rar|tar|7z|wim)$")
def getAttachmentTypeFromUrl(url):
    _match = re.search(pattern_attachment,url)
    if _match is not None:
        return _match.groupdict().get("attachment")
    return None

def getAttachmentUrls(sourceHtml):
    list_urls = []
    _soup = BeautifulSoup(sourceHtml,"lxml")
    set_types = set()
    list_a = _soup.find_all("a")
    for _a in list_a:
        _url = _a.attrs.get("href","")
        _type = getAttachmentTypeFromUrl(_url)
        if _type is not None:
            list_urls.append({"url":_url,"type":_type})
    list_img = _soup.find_all("img")
    for _img in list_img:
        _url = _img.attrs.get("src","")
        _type = getAttachmentTypeFromUrl(_url)
        if _type is not None:
            list_urls.append({"url":_url,"type":_type})
    return list_urls

def getCurrent_date(format="%Y-%m-%d %H:%M:%S"):
    _time = time.strftime(format,time.localtime())
    return _time

def log_tofile(filename):
    logging.basicConfig(filename=filename,level = logging.INFO,format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    logger = logging.getLogger(__name__)

def log(msg):
    '''
    @summary:打印信息
    '''
    logger.info(msg)

def debug(msg):
    '''
    @summary:打印信息
    '''
    logger.debug(msg)


def save(object_to_save, path):
    '''
    保存对象
    @Arugs:
        object_to_save: 需要保存的对象

    @Return:
        保存的路径
    '''
    with open(path, 'wb') as f:
        pickle.dump(object_to_save, f)

def load(path):
    '''
    读取对象
    @Arugs:
        path: 读取的路径

    @Return:
        读取的对象
    '''
    with open(path, 'rb') as f:
        object1 = pickle.load(f)
        return object1


def uniform_num(num):
    d1 = {'一': '1', '二': '2', '三': '3', '四': '4', '五': '5', '六': '6', '七': '7', '八': '8', '九': '9', '十': '10'}
    # d2 = {'A': '1', 'B': '2', 'C': '3', 'D': '4', 'E': '5', 'F': '6', 'G': '7', 'H': '8', 'I': '9', 'J': '10'}
    d3 = {'Ⅰ': '1', 'Ⅱ': '2', 'Ⅲ': '3', 'Ⅳ': '4', 'Ⅴ': '5', 'Ⅵ': '6', 'Ⅶ': '7'}
    if num.isdigit():
        if re.search('^0[\d]$', num):
            num = num[1:]
        return num
    elif re.search('^[一二三四五六七八九十]+$', num):
        _digit = re.search('^[一二三四五六七八九十]+$', num).group(0)
        if len(_digit) == 1:
            num = d1[_digit]
        elif len(_digit) == 2 and _digit[0] == '十':
            num = '1'+ d1[_digit[1]]
        elif len(_digit) == 2 and _digit[1] == '十':
            num = d1[_digit[0]] + '0'
        elif len(_digit) == 3 and _digit[1] == '十':
            num = d1[_digit[0]] + d1[_digit[2]]
    elif re.search('[ⅠⅡⅢⅣⅤⅥⅦ]', num):
        num = re.search('[ⅠⅡⅢⅣⅤⅥⅦ]', num).group(0)
        num = d3[num]
    return num

def uniform_package_name(package_name):
    '''
    统一规范化包号。数值类型统一为阿拉伯数字，字母统一为大写，包含施工监理等抽到前面, 例 A包监理一标段 统一为 监理A1 ; 包Ⅱ 统一为 2
    :param package_name: 字符串类型 包号
    :return:
    '''
    package_name_raw = package_name
    package_name = re.sub('pdf|doc|docs|xlsx|rar|\d{4}年', ' ', package_name)
    package_name = package_name.replace('标段（包）', '标段').replace('№', '')
    package_name = re.sub('\[|【', '', package_name)
    kw = re.search('(施工|监理|监测|勘察|设计|劳务)', package_name)
    name = ""
    if kw:
        name += kw.group(0)
    if re.search('^[a-zA-Z0-9-]{5,}$', package_name):   # 五个字符以上编号
        _digit = re.search('^[a-zA-Z0-9-]{5,}$', package_name).group(0).upper()
        # print('规范化包号1', _digit)
        name += _digit
    elif re.search('(?P<eng>[a-zA-Z])包[：）]?第?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))标段?', package_name): # 处理类似 A包2标段
        ser = re.search('(?P<eng>[a-zA-Z])包[：）]?第?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))标段?', package_name)
        # print('规范化包号2', ser.group(0))
        _char = ser.groupdict().get('eng')
        if _char:
            _char = _char.upper()
        _digit = ser.groupdict().get('num')
        _digit = uniform_num(_digit)
        name += _char.upper() + _digit
    elif re.search('第?(?P<eng>[0-9a-zA-Z-]{1,4})?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))(标[段号的包项]?|合同[包段]|([分子]?[包标]))', package_name): # 处理类似 A包2标段
        ser = re.search('第?(?P<eng>[0-9a-zA-Z-]{1,4})?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))(标[段号的包项]?|合同[包段]|([分子]?[包标]))', package_name)
        # print('规范化包号3', ser.group(0))
        _char = ser.groupdict().get('eng')
        if _char:
            _char = _char.upper()
        _digit = ser.groupdict().get('num')
        _digit = uniform_num(_digit)
        if _char:
            name += _char.upper()
        name += _digit
    elif re.search('(标[段号的包项]?|项目|子项目?|([分子]?包|包[组件号]))编?号?[:：]?(?P<eng>[0-9a-zA-Z-]{1,4})?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))', package_name):  # 数字的统一的阿拉伯数字
        ser = re.search('(标[段号的包项]?|项目|子项目?|([分子]?包|包[组件号]))编?号?[:：]?(?P<eng>[0-9a-zA-Z-]{1,4})?(?P<num>([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4}))',package_name)
        # print('规范化包号4', ser.group(0))
        _char = ser.groupdict().get('eng')
        if _char:
            _char = _char.upper()
        _digit = ser.groupdict().get('num')
        _digit = uniform_num(_digit)
        if _char:
            name += _char.upper()
        name += _digit
    elif re.search('(标[段号的包项]|([分子]?包|包[组件号]))编?号?[:：]?(?P<eng>[a-zA-Z-]{1,5})', package_name):  # 数字的统一的阿拉伯数字
        _digit = re.search('(标[段号的包项]|([分子]?包|包[组件号]))编?号?[:：]?(?P<eng>[a-zA-Z-]{1,5})', package_name).group('eng').upper()
        # print('规范化包号5', _digit)
        name += _digit
    elif re.search('(?P<eng>[a-zA-Z]{1,4})(标[段号的包项]|([分子]?[包标]|包[组件号]))', package_name):  # 数字的统一的阿拉伯数字
        _digit = re.search('(?P<eng>[a-zA-Z]{1,4})(标[段号的包项]|([分子]?[包标]|包[组件号]))', package_name).group('eng').upper()
        # print('规范化包号6', _digit)
        name += _digit
    elif re.search('^([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4})$', package_name):  # 数字的统一的阿拉伯数字
        _digit = re.search('^([0-9]{1,4}|[一二三四五六七八九十]{1,4}|[ⅠⅡⅢⅣⅤⅥⅦ]{1,4})$', package_name).group(0)
        # print('规范化包号7', _digit)
        _digit = uniform_num(_digit)
        name += _digit
    elif re.search('^[a-zA-Z0-9-]+$', package_name):
        _char = re.search('^[a-zA-Z0-9-]+$', package_name).group(0)
        # print('规范化包号8', _char)
        name += _char.upper()
    if name == "":
        return package_name_raw
    else:
        if name.isdigit():
            name = str(int(name))
        # print('原始包号：%s, 处理后：%s'%(package_name, name))
        return name

def getIndexOfWord_fool(word):
    
    if word in fool_char_to_id.keys():
        return fool_char_to_id[word]
    else:
        return fool_char_to_id["[UNK]"]


def find_index(list_tofind,text):
    '''
    @summary: 查找所有词汇在字符串中第一次出现的位置
    @param:
        list_tofind:待查找词汇
        text:字符串
    @return: list,每个词汇第一次出现的位置
    
    '''
    result = []
    for item in list_tofind:
        index = text.find(item)
        if index>=0:
            result.append(index)
        else:
            result.append(-1)
    return result


def combine(list1,list2):
    '''
    @summary:将两个list中的字符串两两拼接
    @param:
        list1:字符串list
        list2:字符串list
    @return:拼接结果list
    '''
    result = []
    for item1 in list1:
        for item2 in list2:
            result.append(str(item1)+str(item2))
    return result


def getDigitsDic(unit):
    '''
    @summary:拿到中文对应的数字
    '''
    DigitsDic = {"零":0, "壹":1, "贰":2, "叁":3, "肆":4, "伍":5, "陆":6, "柒":7, "捌":8, "玖":9,
                 "〇":0, "一":1, "二":2, "三":3, "四":4, "五":5, "六":6, "七":7, "八":8, "九":9}
    return DigitsDic.get(unit)

def getMultipleFactor(unit):
    '''
    @summary:拿到单位对应的值
    '''
    MultipleFactor = {"兆":Decimal(1000000000000),"亿":Decimal(100000000),"万":Decimal(10000),"仟":Decimal(1000),"千":Decimal(1000),"佰":Decimal(100),"百":Decimal(100),"拾":Decimal(10),"十":Decimal(10),"元":Decimal(1),"圆":Decimal(1),"角":round(Decimal(0.1),1),"分":round(Decimal(0.01),2)}
    return MultipleFactor.get(unit)

def getUnifyMoney(money):
    '''
    @summary:将中文金额字符串转换为数字金额
    @param:
        money:中文金额字符串
    @return: decimal,数据金额
    '''

    MAX_MONEY = 1000000000000
    MAX_NUM = 12
    #去掉逗号
    money = re.sub("[，,]","",money)
    money = re.sub("[^0-9.零壹贰叁肆伍陆柒捌玖拾佰仟萬億圆十百千万亿元角分]","",money)
    result = Decimal(0)
    chnDigits = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"]
    # chnFactorUnits = ["兆", "亿", "万", "仟", "佰", "拾","圆","元","角","分"]
    chnFactorUnits = ["兆", "亿", "万", "仟", '千', "佰", '百', "拾", '十',"圆", "元", "角", "分"]  # 20240611 修复大写提取错误 '陆拾陆亿伍千柒佰零叁万肆千叁佰陆拾伍元' Decimal('11607430365')

    LowMoneypattern = re.compile("^[\d,]+(\.\d+)?$")
    BigMoneypattern = re.compile("^零?(?P<BigMoney>[%s])$"%("".join(chnDigits)))
    try:
        if re.search(LowMoneypattern,money) is not None:
            return Decimal(money)
        elif re.search(BigMoneypattern,money) is not None:
            return getDigitsDic(re.search(BigMoneypattern,money).group("BigMoney"))
        for factorUnit in chnFactorUnits:
            if re.search(re.compile(".*%s.*"%(factorUnit)),money) is not None:
                subMoneys = re.split(re.compile("%s(?!.*%s.*)"%(factorUnit,factorUnit)),money)
                if re.search(re.compile("^(\d+)(\.\d+)?$"),subMoneys[0]) is not None:
                    if MAX_MONEY/getMultipleFactor(factorUnit)<Decimal(subMoneys[0]):
                        return Decimal(0)
                    result += Decimal(subMoneys[0])*(getMultipleFactor(factorUnit))
                elif len(subMoneys[0])==1:
                    if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[0]) is not None:
                        result += Decimal(getDigitsDic(subMoneys[0]))*(getMultipleFactor(factorUnit))
                # subMoneys[0]中无金额单位，不可再拆分
                elif subMoneys[0]=="":
                    result += 0
                elif re.search(re.compile("[%s]"%("".join(chnFactorUnits))),subMoneys[0]) is None:
                    # print(subMoneys)
                    # subMoneys[0] = subMoneys[0][0]
                    result += Decimal(getUnifyMoney(subMoneys[0])) * (getMultipleFactor(factorUnit))
                else:
                    result += Decimal(getUnifyMoney(subMoneys[0]))*(getMultipleFactor(factorUnit))
                if len(subMoneys)>1:
                    if re.search(re.compile("^(\d+(,)?)+(\.\d+)?[百千万亿]?\s?(元)?$"),subMoneys[1]) is not None:
                        result += Decimal(subMoneys[1])
                    elif len(subMoneys[1])==1:
                        if re.search(re.compile("^[%s]$"%("".join(chnDigits))),subMoneys[1]) is not None:
                            result += Decimal(getDigitsDic(subMoneys[1]))
                    else:
                        result += Decimal(getUnifyMoney(subMoneys[1]))
                break
    except Exception as e:
        # traceback.print_exc()
        return Decimal(0)
    return result




def getModel_w2v():
    '''
    @summary:加载词向量
    '''
    global model_w2v,lock_model_w2v
    with lock_model_w2v:
        if model_w2v is None:
            model_w2v = gensim.models.KeyedVectors.load_word2vec_format(getw2vfilepath(),binary=True)
        return model_w2v

def getModel_word():
    '''
    @summary:加载字向量
    '''

    global model_word,lock_model_w2v
    with lock_model_word:
        if model_word is None:
            model_word = gensim.models.KeyedVectors.load_word2vec_format(model_word_file,binary=True)
        return model_word

# getModel_w2v()
# getModel_word()

def formatArea(area):
    if area is not None and len(area)>=3:
        return re.sub("[省市区县]","",area)
    return area

def findAllIndex(substr,wholestr):
    '''
    @summary: 找到字符串的子串的所有begin_index
    @param:
        substr:子字符串
        wholestr:子串所在完整字符串
    @return: list,字符串的子串的所有begin_index
    '''
    copystr = wholestr
    result = []
    indexappend = 0
    while(True):
        index = copystr.find(substr)
        if index<0:
            break
        else:
            result.append(indexappend+index)
            indexappend += index+len(substr)
            copystr = copystr[index+len(substr):]
    return result
    
  
def spanWindow(tokens,begin_index,end_index,size,center_include=False,word_flag = False,use_text = False,text = None):
    '''
    @summary:取得某个实体的上下文词汇
    @param:
        tokens:句子分词list
        begin_index:实体的开始index
        end_index:实体的结束index
        size:左右两边各取多少个词
        center_include:是否包含实体
        word_flag:词/字,默认是词
    @return: list,实体的上下文词汇
    '''  
    if use_text:
        assert text is not None
    length_tokens = len(tokens)
    if begin_index>size:
        begin = begin_index-size
    else:
        begin = 0
    if end_index+size<length_tokens:
        end = end_index+size+1
    else:
        end = length_tokens
    result = []
    if not word_flag:
        result.append(tokens[begin:begin_index])
        if center_include:
            if use_text:
                result.append(text)
            else:
                result.append(tokens[begin_index:end_index+1])
        result.append(tokens[end_index+1:end])
    else:
        result.append("".join(tokens[begin:begin_index]))
        if center_include:
            if use_text:
                result.append(text)
            else:
                result.append("".join(tokens[begin_index:end_index+1]))
        result.append("".join(tokens[end_index+1:end]))
    #print(result)
    return result

#根据规则补全编号或名称两边的符号
def fitDataByRule(data):
    symbol_dict = {"(":")",
                   "（":"）",
                   "[":"]",
                   "【":"】",
                   ")":"(",
                   "）":"（",
                   "]":"[",
                   "】":"【"}
    leftSymbol_pattern = re.compile("[\(（\[【]")
    rightSymbol_pattern = re.compile("[\)）\]】]")
    leftfinds = re.findall(leftSymbol_pattern,data)
    rightfinds = re.findall(rightSymbol_pattern,data)
    result = data
    if len(leftfinds)+len(rightfinds)==0:
        return data
    elif len(leftfinds)==len(rightfinds):
        return data
    elif abs(len(leftfinds)-len(rightfinds))==1:
        if len(leftfinds)>len(rightfinds):
            if symbol_dict.get(data[0]) is not None:
                result = data[1:]
            else:
                #print(symbol_dict.get(leftfinds[0]))
                result = data+symbol_dict.get(leftfinds[0])
        else:
            if symbol_dict.get(data[-1]) is not None:
                result = data[:-1]
            else:
                result = symbol_dict.get(rightfinds[0])+data
    result = re.sub("[。]","",result)
    return  result

def embedding(datas,shape):
    '''
    @summary:查找词汇对应的词向量
    @param:
        datas:词汇的list
        shape:结果的shape
    @return: array,返回对应shape的词嵌入
    '''
    model_w2v = getModel_w2v()
    embed = np.zeros(shape)
    length = shape[1]
    out_index = 0
    #print(datas)
    for data in datas:
        index = 0
        for item in data:
            item_not_space = re.sub("\s*","",item)
            if index>=length:
                break
            if item_not_space in model_w2v.vocab:
                embed[out_index][index] = model_w2v[item_not_space]
                index += 1
            else:
                #embed[out_index][index] = model_w2v['unk']
                index += 1
        out_index += 1
    return embed

def embedding_word(datas,shape):
    '''
    @summary:查找词汇对应的词向量
    @param:
        datas:词汇的list
        shape:结果的shape
    @return: array,返回对应shape的词嵌入
    '''
    model_w2v = getModel_word()
    embed = np.zeros(shape)
    length = shape[1]
    out_index = 0
    #print(datas)
    for data in datas:
        index = 0
        for item in str(data)[-shape[1]:]:
            if index>=length:
                break
            if item in model_w2v.vocab:
                embed[out_index][index] = model_w2v[item]
                index += 1
            else:
                # embed[out_index][index] = model_w2v['unk']
                index += 1
        out_index += 1
    return embed

def formEncoding(text,shape=(100,60),expand=False):
    embedding = np.zeros(shape)
    word_model = getModel_word()
    for i in range(len(text)):
        if i>=shape[0]:
            break
        if text[i] in word_model.vocab:
            embedding[i] = word_model[text[i]]
    if expand:
        embedding = np.expand_dims(embedding,0)
    return embedding

def partMoney(entity_text,input2_shape = [7]):
    '''
    @summary:对金额分段
    @param:
        entity_text:数值金额
        input2_shape:分类数
    @return: array,分段之后的独热编码
    '''
    money = float(entity_text)
    parts = np.zeros(input2_shape)
    if money<100:
        parts[0] = 1
    elif money<1000:
        parts[1] = 1
    elif money<10000:
        parts[2] = 1
    elif money<100000:
        parts[3] = 1
    elif money<1000000:
        parts[4] = 1
    elif money<10000000:
        parts[5] = 1
    else:
        parts[6] = 1
    return parts

def recall(y_true, y_pred):
    from keras import backend as K
    '''
    计算召回率
    @Argus:
        y_true: 正确的标签
        y_pred: 模型预测的标签

    @Return
        召回率
    '''
    c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    c3 = K.sum(K.round(K.clip(y_true, 0, 1)))
    if c3 == 0:
        return 0
    recall = c1 / c3
    return recall


def f1_score(y_true, y_pred):
    from keras import backend as K
    '''
    计算F1

    @Argus:
        y_true: 正确的标签
        y_pred: 模型预测的标签

    @Return
        F1值
    '''

    c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    c2 = K.sum(K.round(K.clip(y_pred, 0, 1)))
    c3 = K.sum(K.round(K.clip(y_true, 0, 1)))
    precision = c1 / c2
    if c3 == 0:
        recall = 0
    else:
        recall = c1 / c3
    f1_score = 2 * (precision * recall) / (precision + recall)
    return f1_score


def precision(y_true, y_pred):
    from keras import backend as K
    '''
    计算精确率

    @Argus:
        y_true: 正确的标签
        y_pred: 模型预测的标签

    @Return
        精确率
    '''
    c1 = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    c2 = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = c1 / c2
    return precision

# def print_metrics(history):
#     '''
#     制作每次迭代的各metrics变化图片
#
#     @Arugs:
#         history: 模型训练迭代的历史记录
#     '''
#     import matplotlib.pyplot as plt
#
#     # loss图
#     loss = history.history['loss']
#     val_loss = history.history['val_loss']
#     epochs = range(1, len(loss) + 1)
#     plt.subplot(2, 2, 1)
#     plt.plot(epochs, loss, 'bo', label='Training loss')
#     plt.plot(epochs, val_loss, 'b', label='Validation loss')
#     plt.title('Training and validation loss')
#     plt.xlabel('Epochs')
#     plt.ylabel('Loss')
#     plt.legend()
#
#     # f1图
#     f1 = history.history['f1_score']
#     val_f1 = history.history['val_f1_score']
#     plt.subplot(2, 2, 2)
#     plt.plot(epochs, f1, 'bo', label='Training f1')
#     plt.plot(epochs, val_f1, 'b', label='Validation f1')
#     plt.title('Training and validation f1')
#     plt.xlabel('Epochs')
#     plt.ylabel('F1')
#     plt.legend()
#
#     # precision图
#     prec = history.history['precision']
#     val_prec = history.history['val_precision']
#     plt.subplot(2, 2, 3)
#     plt.plot(epochs, prec, 'bo', label='Training precision')
#     plt.plot(epochs, val_prec, 'b', label='Validation pecision')
#     plt.title('Training and validation precision')
#     plt.xlabel('Epochs')
#     plt.ylabel('Precision')
#     plt.legend()
#
#     # recall图
#     recall = history.history['recall']
#     val_recall = history.history['val_recall']
#     plt.subplot(2, 2, 4)
#     plt.plot(epochs, recall, 'bo', label='Training recall')
#     plt.plot(epochs, val_recall, 'b', label='Validation recall')
#     plt.title('Training and validation recall')
#     plt.xlabel('Epochs')
#     plt.ylabel('Recall')
#     plt.legend()
#
#     plt.show()


if __name__=="__main__":
    print(fool_char_to_id[">"])
    # model = getModel_w2v()
    # vocab,matrix = getVocabAndMatrix(model, Embedding_size=128)
    # save([vocab,matrix],"vocabMatrix_words.pk")