BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/interface/123.py

from dl.common.Utils import findAllIndex
from dl.interface.Entitys import PREM
import re
import copy
import math

def getTheRole(entity,role_list):
    '''
    @summary:根据实体名称拿到index
    @param:
        entity:实体名称
        role_list:角色list
    @return:该实体所在下标
    '''
    for role_index in range(len(role_list)):
        if entity in role_list[role_index]:
            return role_index
    return None

dict_role_id = {"0":"tenderee",
                "1":"agency",
                "2":"win_tenderer",
                "3":"second_tenderer",
                "4":"third_tenderer"}

def getPackage(packageList,sentence_index,begin_index):
    '''
    @param:
        packageList:文章的包的信息
        sentence_index:实体所在的句子
        begin_index:实体所在句子的起始位置
    @return:公司实体所属的包

    '''
    if len(packageList)==0:
        return None
    before_index = None
    after_index = None
    equal_index = None
    equal_count = 0


    for pack_index in range(len(packageList)):
        if packageList[pack_index][1]>sentence_index and after_index is None:
            after_index = pack_index
        if packageList[pack_index][1]<sentence_index:
            before_index = pack_index
        if packageList[pack_index][1]==sentence_index and equal_index is None:
            equal_index = pack_index
    #当前句子和之前句子未找到包
    if before_index is None and equal_index is None:
        return None
    else:
        if after_index is None:
            end_index = len(packageList)
        else:
            end_index = after_index
        #只在当前句子找到一个包号
        if end_index-max((before_index if before_index is not None else -1,equal_index if equal_index is not None else -1))==1:
            return packageList[end_index-1][0]
        else:
            for i in range(max((before_index if before_index is not None else -1,equal_index if equal_index is not None else -1)),end_index):
                if packageList[i][2]>int(begin_index):
                    if packageList[i-1][4]:
                        return packageList[i-1][0]
                    else:
                        if packageList[i][4]:
                            return packageList[i-1][0]
                        else:
                            return packageList[i][0]
            return packageList[end_index-1][0]
    return None

#生成合法的组合
def get_legal_comba(list_entity,dict_role_combination):

    #拿到一个包中所有合法的组合
    def recursive_package(_dict_legal_combination,set_legal_entity,dict_one_selution,list_all_selution):
        last_layer = False
        #若是空组合则放回空
        if len(_dict_legal_combination.keys())==0:
            return []
        #递归到最后一层则修改状态
        if len(_dict_legal_combination.keys())==1:
            last_layer = True
        #取一个角色开始进行遍历
        _key_role = list(_dict_legal_combination.keys())[0]
        for item in _dict_legal_combination[_key_role]:
            copy_dict_one_selution = copy.copy(dict_one_selution)
            copy_dict_legal_combination = {}
            copy_set_legal_entity = copy.copy(set_legal_entity)

            #复制余下的所有角色，进行下一轮递归
            for _key in _dict_legal_combination.keys():
                if _key!=_key_role:
                    copy_dict_legal_combination[_key] = _dict_legal_combination[_key]

            #修改为招标人和代理人可以为同一个
            if item !="":
                _flag = True
                if str(_key_role) in ["0","1"]:
                    for _key_flag in copy_dict_one_selution:
                        if _key_flag not in ["0","1"] and copy_dict_one_selution[_key_flag]==item:
                            _flag = False
                else:
                    for _key_flag in copy_dict_one_selution:
                        if copy_dict_one_selution[_key_flag]==item:
                            _flag = False
                if _flag:
                    copy_dict_one_selution[_key_role] = item

            '''
            if item not in copy_set_legal_entity:
                if item !="":
                    copy_dict_one_selution[_key_role] = item
            '''
            copy_set_legal_entity.add(item)
            if last_layer:
                list_all_selution.append(copy_dict_one_selution)
            else:
                recursive_package(copy_dict_legal_combination,copy_set_legal_entity,copy_dict_one_selution,list_all_selution)


    #递归匹配各个包的结果
    def recursive_packages(_dict_legal_combination,dict_one_selution,list_all_selution):
        last_layer = False
        if len(_dict_legal_combination.keys())==0:
            return []
        if len(_dict_legal_combination.keys())==1:
            last_layer = True
        _key_pack = list(_dict_legal_combination.keys())[0]
        for item in _dict_legal_combination[_key_pack]:
            copy_dict_one_selution = copy.copy(dict_one_selution)
            copy_dict_legal_combination = {}
            for _key in _dict_legal_combination.keys():
                if _key!=_key_pack:
                    copy_dict_legal_combination[_key] = _dict_legal_combination[_key]
            for _key_role in item.keys():
                copy_dict_one_selution[_key_pack+"$$"+_key_role] = item[_key_role]
            if last_layer:
                list_all_selution.append(copy_dict_one_selution)
            else:
                recursive_packages(copy_dict_legal_combination,copy_dict_one_selution,list_all_selution)
        return list_all_selution

    #循环获取所有包组合
    def circle_pageages(_dict_legal_combination):
        list_all_selution = []
        for _key_pack in _dict_legal_combination.keys():
            list_key_selution = []
            for item in _dict_legal_combination[_key_pack]:
                _dict = dict()
                for _key_role in item.keys():
                    _dict[_key_pack+"$$"+_key_role] = item[_key_role]
                list_key_selution.append(_dict)
            if len(list_all_selution)==0:
                list_all_selution = list_key_selution
            else:
                _list_all_selution = []
                for item_1 in list_all_selution:
                    for item_2 in list_key_selution:
                        _list_all_selution.append(dict(item_1,**item_2))
                list_all_selution = _list_all_selution
        return list_all_selution

    #拿到各个包解析之后的结果
    _dict_legal_combination = {}
    for packageName in dict_role_combination.keys():
        _list_all_selution = []
        recursive_package(dict_role_combination[packageName], set(), {}, _list_all_selution)
        '''
        print("===1")
        print(packageName)
        for item in _list_all_selution:
            print(item)
        print("===2")
        '''
        #去除包含子集
        list_all_selution_simple = []
        _list_set_all_selution = []
        for item_selution in _list_all_selution:
            item_set_selution = set()
            for _key in item_selution.keys():
                item_set_selution.add((_key,item_selution[_key]))
            _list_set_all_selution.append(item_set_selution)
        if len(_list_set_all_selution)>1000:
            _dict_legal_combination[packageName] = _list_all_selution
            continue
        for i in range(len(_list_set_all_selution)):

            be_included = False
            for j in range(len(_list_set_all_selution)):
                if i!=j:
                    if len(set(_list_set_all_selution[i])&set(_list_set_all_selution[j]))==len(_list_set_all_selution[i]) and len(_list_set_all_selution[i])!=len(_list_set_all_selution[j]):
                        be_included = True
            if not be_included:
                list_all_selution_simple.append(_list_all_selution[i])
        _dict_legal_combination[packageName] = list_all_selution_simple

    _list_final_comba = []
    #对各个包的结果进行排列组合
    _comba_count = 1
    for _key in _dict_legal_combination.keys():
        _comba_count *= len(_dict_legal_combination[_key])
    #如果过大，则每个包只取概率最大的那个
    if _comba_count>250:
        new_dict_legal_combination = dict()
        for _key_pack in _dict_legal_combination.keys():
            MAX_PROB = -1000
            _MAX_PROB_COMBA = None
            for item in _dict_legal_combination[_key_pack]:
                _dict = dict()
                for _key in item.keys():
                    _dict[str(_key_pack)+"$$"+str(_key)] = item[_key]
                _prob = getSumExpectation(list_entity, _dict)
                if _prob>MAX_PROB:
                    MAX_PROB = _prob
                    _MAX_PROB_COMBA = [item]
            new_dict_legal_combination[_key_pack] = _MAX_PROB_COMBA
        _dict_legal_combination = new_dict_legal_combination
    #recursive_packages(_dict_legal_combination, {}, _list_final_comba)
    _list_final_comba = circle_pageages(_dict_legal_combination)

    #除了Project包(招标人和代理人)，其他包是不会有冲突的
    #查看是否有一个实体出现在了Project包和其他包中，如有，要进行裁剪
    _list_real_comba = []
    for dict_item in _list_final_comba:
        set_project = set()
        set_other = set()
        for _key in list(dict_item.keys()):
            if _key.split("$$")[0]=="Project":
                set_project.add(dict_item[_key])
            else:
                set_other.add(dict_item[_key])
        set_common = set_project&set_other
        if len(set_common)>0:
            dict_project = {}
            dict_not_project = {}
            for _key in list(dict_item.keys()):
                if dict_item[_key] in set_common:
                    if str(_key.split("$$")[0])=="Project":
                        dict_project[_key] = dict_item[_key]
                    else:
                        dict_not_project[_key] = dict_item[_key]
                else:
                    dict_project[_key] = dict_item[_key]
                    dict_not_project[_key] = dict_item[_key]

            _list_real_comba.append(dict_project)
            _list_real_comba.append(dict_not_project)
        else:
            _list_real_comba.append(dict_item)

    return _list_real_comba

#计算合计期望
def getSumExpectation(list_entity,combination,on_value=0.5):
    '''
    expect = 0
    for entity in list_entity:
        if entity.entity_type in ['org','company']:
            values = entity.values
            role_prob = float(values[int(entity.label)])
            _key = entity.packageName+"$$"+str(entity.label)
            if role_prob>on_value and str(entity.label)!="5":
                if _key in combination.keys() and combination[_key]==entity.entity_text:
                    expect += math.pow(role_prob,4)
                else:
                    expect -= math.pow(role_prob,4)
    '''
    #修改为同一个实体只取对应包-角色的最大的概率值
    expect = 0
    dict_entity_prob = {}
    for entity in list_entity:
        if entity.entity_type in ['org','company']:
            values = entity.values
            role_prob = float(values[int(entity.label)])
            _key = entity.packageName+"$$"+str(entity.label)
            if role_prob>=on_value and str(entity.label)!="5":
                if _key in combination.keys() and combination[_key]==entity.entity_text:
                    _key_prob = _key+entity.entity_text
                    if _key_prob in dict_entity_prob.keys():
                        if dict_entity_prob[_key_prob]<role_prob:
                            dict_entity_prob[_key_prob] = role_prob
                    else:
                        dict_entity_prob[_key_prob] = role_prob
                else:
                    _key_prob = _key+entity.entity_text
                    if _key_prob in dict_entity_prob.keys():
                        if dict_entity_prob[_key_prob]>-role_prob:
                            dict_entity_prob[_key_prob] = -role_prob
                    else:
                        dict_entity_prob[_key_prob] = -role_prob
    for _key in dict_entity_prob.keys():
        symbol = 1 if dict_entity_prob[_key]>0 else -1
        expect += symbol*math.pow(dict_entity_prob[_key],2)
    return expect


def getRoleList(list_sentence,list_entity,on_value = 0.5):
    '''
    @summary: 搜索树，得到所有不矛盾的角色组合，取合计期望值最大的作为结果返回
    @param:
        list_sentence:文章所有的sentence
        list_entity:文章所有的实体
        on_value:概率阈值
    @return:文章的角色list
    '''

    pack = getPackagesFromArticle(list_sentence)
    if pack is None:
        return None
    PackageList,PackageSet,dict_PackageCode = pack

    #拿到所有可能的情况
    dict_role_combination = {}

    #拿到各个实体的packageName,packageCode
    for entity in list_entity:
        if entity.entity_type in ['org','company']:
            values = entity.values
            role_prob = float(values[int(entity.label)])
            if role_prob>=on_value and str(entity.label)!="5":
                if str(entity.label) in ["0","1"]:
                    packageName = "Project"
                else:
                    if len(PackageSet)>1:
                        packageName = getPackage(PackageList,entity.sentence_index,entity.end_index)
                        if packageName is None:
                            #continue
                            packageName = "Project"
                    else:
                        packageName = "Project"
                    find_flag = False
                    role_name = dict_role_id.get(str(entity.label))
                    if packageName in dict_PackageCode.keys():
                        packageCode = dict_PackageCode[packageName]
                    else:
                        packageCode = ""
                    entity.packageName = packageName
                    entity.packageCode = packageCode
                    entity.roleName = role_name
                if entity.packageName in dict_role_combination.keys():
                    if str(entity.label) in dict_role_combination[entity.packageName].keys():
                        dict_role_combination[entity.packageName][str(entity.label)].add(entity.entity_text)
                    else:
                        dict_role_combination[entity.packageName][str(entity.label)] = set([entity.entity_text])
                else:
                    dict_role_combination[entity.packageName] = {}
                    #初始化空值
                    roleIds = [0,1,2,3,4]
                    for _roleId in roleIds:
                        dict_role_combination[entity.packageName][str(_roleId)] = set([""])
                    dict_role_combination[entity.packageName][str(entity.label)].add(entity.entity_text)

    list_real_comba = get_legal_comba(list_entity,dict_role_combination)

    #拿到最大期望值的组合
    max_index = 0
    max_expect = -100
    _index = 0
    for item_combination in list_real_comba:
        expect = getSumExpectation(list_entity, item_combination)
        if expect>max_expect:
            max_index = _index
            max_expect = expect
        _index += 1

    RoleList = []
    RoleSet = set()
    if len(list_real_comba)>0:
        for _key in list_real_comba[max_index].keys():
            packageName = _key.split("$$")[0]
            label = _key.split("$$")[1]
            role_name = dict_role_id.get(str(label))
            entity_text = list_real_comba[max_index][_key]
            if packageName in dict_PackageCode.keys():
                packagecode = dict_PackageCode.get(packageName)
            else:
                packagecode = ""
            RoleList.append(PREM(packageName,packagecode,role_name,entity_text,0,0,0.0,[]))
            RoleSet.add(entity_text)

    return RoleList,RoleSet,PackageList,PackageSet

def getPackagesFromArticle(list_sentence):
    '''
    @param:
        list_sentence:文章的句子list
    @return: type:list if [包号,句子index,词偏移,标段号] meaning:文章的包/标段信息
    '''

    if len(list_sentence)==0:
        return None

    PackageList = []
    PackageSet = set()
    dict_packageCode = dict()

    package_name_pattern = re.compile("((标[段号的包]|分包)的?(名[称单]?|包名))[:：]?([^:：]{3,30}?)，{1}")
    package_N_name_pattern = re.compile("(分?包|标段|标|包|包组|项目)编?号?[:：]?[\(（]?([0-9A-Za-z一二三四五六七八九十]){1,2}，{1}")
    package_number_pattern = re.compile("((包|标[段号的包]|分?包|包组|项目)编?号?[:：]?[\(（]?[0-9A-Za-z一二三四五六七八九十]{1,4})|(第?[0-9A-Za-z一二三四五六七八九十]{1,4}(包号|标[段号的包]|分?包))")
    other_package_pattern = re.compile('(项目名称|物资名称|设备名称|场次名称|标段名称)[:：](.{,20}?)(，|项目)')  # 新正则识别标段
    number_pattern = re.compile("[0-9A-Za-z一二三四五六七八九十]{1,4}")

    package_code_pattern = re.compile("(?:编号[:：]?\s*)([-\dA-Za-z]+)")

    def changeIndexFromWordToWords(tokens,word_index):
        '''
        @summary:转换某个字的字偏移为词偏移
        '''
        before_index = 0
        after_index = 0
        for i in range(len(tokens)):
            after_index = after_index+len(tokens[i])
            if before_index<=word_index and after_index>=word_index:
                return i
            before_index = after_index
    package_names = []

    def extractPackageCode(tokens,word_index,size=20,pattern = package_code_pattern):
        '''
        @summary:抽取包附近的标段号
        @param:
            tokens:包所在句子的分词
            word_index:包所在字偏移
            size:左右各取多少个词
            pattern:提取标段号的正则
        @return: type:string,meaning:标段号
        '''
        index = changeIndexFromWordToWords(tokens,word_index)
        if index<size:
            begin = index
        else:
            begin = index-size
        if index+size>len(tokens):
            end = len(tokens)
        else:
            end = index+size
        #拿到左右两边的词语组成短语
        text = "".join(tokens[begin:end])
        #在短语中的字偏移
        new_word_index = word_index-len("".join(tokens[:begin]))
        min_distance = len(text)
        packageCode = None
        for the_iter in re.finditer(pattern,text):
            #算出最小距离
            distance = min([abs(new_word_index-the_iter.span()[0]),abs(new_word_index-the_iter.span()[1])])
            if distance<min_distance:
                min_distance = distance
                packageCode = the_iter.group(1)
        return packageCode

    #从标段介绍表格中提取包名和包号
    for i in range(len(list_sentence)):
        content = list_sentence[i].sentence_text
        names = re.findall(package_name_pattern,content)
        if names == []:
            names = re.findall(other_package_pattern, content)
        N_names = re.findall(package_N_name_pattern,content)
        if len(names)==1 and len(N_names)==1:
            package_names.append([names[0][-1],N_names[0][-1]])
    for i in range(len(list_sentence)):
        PackageList_item = []
        content = list_sentence[i].sentence_text
        tokens = list_sentence[i].tokens
        for name in package_names:
            for index in findAllIndex(name[0],content):
                temp_package_number = re.findall(number_pattern,name[1])[0]
                PackageList_item.append([temp_package_number,i,changeIndexFromWordToWords(tokens,index),index])
                code = extractPackageCode(tokens, index)
                if code is not None:
                    dict_packageCode[temp_package_number] = code
                PackageSet.add(temp_package_number)
        for iter in re.finditer(package_number_pattern,content):
            temp_package_number = re.findall(number_pattern,content[iter.span()[0]:iter.span()[1]])[0]
            PackageList_item.append([temp_package_number,i,changeIndexFromWordToWords(tokens,iter.span()[0]),iter.span()[0]])
            code = extractPackageCode(tokens, iter.span()[0])
            if code is not None:
                dict_packageCode[temp_package_number] = code
            PackageSet.add(temp_package_number)
        if PackageList_item == []:  # 原有正则没有识别到标段和包号时增加以下正则识别
            for iter in re.finditer(other_package_pattern,content):
                temp_package_number = iter.group(2)
                PackageList_item.append([temp_package_number,i,changeIndexFromWordToWords(tokens,iter.span()[0]),iter.span()[0]])
                code = extractPackageCode(tokens, iter.span()[0])
                if code is not None:
                    dict_packageCode[temp_package_number] = code
                PackageSet.add(temp_package_number)
        PackageList_item.sort(key=lambda x:x[2])
        PackageList = PackageList+PackageList_item
    pattern_punctuation = "[:：（）\(\),，。；;]"
    for i in range(len(list_sentence)):
        for j in range(len(PackageList)):
            if i==PackageList[j][1]:
                _flag = False
                left_str = list_sentence[i].sentence_text[PackageList[j][3]-30:PackageList[j][3]]
                right_str = list_sentence[i].sentence_text[PackageList[j][3]:PackageList[j][3]+30]
                _left_find = re.findall(pattern_punctuation,left_str)
                _right_find = re.findall(pattern_punctuation,right_str)
                if len(_left_find)>0 and _left_find[-1] in [":","："]:
                    _flag = True
                if len(_right_find)>0 and _right_find[0] in [":","："]:
                    _flag = True
                PackageList[j].append(_flag)
    return PackageList,PackageSet,dict_packageCode


def findAttributeAfterEntity(roleList,roleSet,PackageList,PackageSet,list_entity,on_value = 0.5,on_value_person=0.5,sentence_len=4):
    '''
    @param:
        roleList:文章角色list
        roleSet:文章所有角色的公司名称
        PackageList:文章的包信息
        PackageSet:文章所有包的名称
        list_entity:文章所有经过模型处理的实体
        on_value:金额模型的阈值
        on_value_person:联系人模型的阈值
        sentence_len:公司和属性间隔句子的最大长度
    @return:添加了属性信息的角色list
    '''

    #根据roleid添加金额到rolelist中
    def addMoneyByRoleid(RoleList,packageName,roleid,money,money_prob):
        for i in range(len(RoleList)):
            if RoleList[i].packageName==packageName and RoleList[i].role_name==dict_role_id.get(str(roleid)):
                if money_prob>RoleList[i].money_prob:
                    RoleList[i].money = money
                    RoleList[i].money_prob = money_prob
        return RoleList

    #根据实体名称添加金额到rolelist中
    def addMoneyByEntity(RoleList,packageName,entity,money,money_prob):
        for i in range(len(RoleList)):
            if RoleList[i].packageName==packageName and RoleList[i].entity_text==entity:
                if money_prob>RoleList[i].money_prob:
                    RoleList[i].money = money
                    RoleList[i].money_prob = money_prob
        return RoleList

    #根据实体名称得到角色
    def getRoleWithText(roleList,entity_text):
        for i in range(len(roleList)):
            if roleList[i].entity_text==entity_text:
                return roleList[i].role_name

    def doesEntityOrLinkedEntity_inRoleSet(entity,RoleSet):
        _list_entitys = [entity]+entity.linked_entitys
        for _entity in _list_entitys:
            if _entity.entity_text in RoleSet:
                return True

    p_entity = 0

    set_tenderer_role = set()
    set_tenderer_money = set()
    #遍历所有实体
    while(p_entity<len(list_entity)):
        entity = list_entity[p_entity]
        if entity.entity_type=="money":
            if entity.values[entity.label]>=on_value:
                if str(entity.label)=="1":
                    set_tenderer_money.add(float(entity.entity_text))
                if str(entity.label)=="0":
                    packageName = "Project"
                    addMoneyByRoleid(roleList, packageName, "0", entity.entity_text, entity.values[entity.label])
        if entity.entity_type=="person":
            if entity.values[entity.label]>=on_value_person:
                if str(entity.label)=="1":
                    for i in range(len(roleList)):
                        if roleList[i].role_name=="tenderee":
                            roleList[i].linklist.append((entity.entity_text,entity.person_phone))
                elif str(entity.label)=="2":
                    for i in range(len(roleList)):
                        if roleList[i].role_name=="agency":
                            roleList[i].linklist.append((entity.entity_text,entity.person_phone))

        #如果实体属于角色集合，则往后找属性
        if doesEntityOrLinkedEntity_inRoleSet(entity, roleSet):

            p_entity += 1
            #循环查找符合的属性
            while(p_entity<len(list_entity)):

                entity_after = list_entity[p_entity]
                if entity_after.sentence_index-entity.sentence_index>=sentence_len:
                    p_entity -= 1
                    break
                #若是遇到公司实体，则跳出循环
                if entity_after.entity_type in ['org','company']:
                    p_entity -= 1
                    break
                if entity_after.values is not None:
                    if entity_after.entity_type=="money":
                        if entity_after.values[entity_after.label]>=on_value:
                            if str(entity_after.label)=="0":
                                packageName = "Project"
                                addMoneyByRoleid(roleList, packageName, "0", entity_after.entity_text, entity_after.values[entity_after.label])
                            elif str(entity_after.label)=="1":
                                _list_entitys = [entity]+entity.linked_entitys
                                for _entity in _list_entitys:
                                    if getRoleWithText(roleList, _entity.entity_text) in ['tenderee','agency']:
                                        packageName_entity = "Project"
                                    else:
                                        if len(PackageSet)>1:
                                            packageName_entity = getPackage(PackageList,_entity.sentence_index,_entity.begin_index)
                                            if packageName_entity is None:
                                                continue
                                        else:
                                            packageName_entity = "Project"
                                    if str(_entity.label) in ["2","3","4"]:
                                        addMoneyByEntity(roleList, packageName_entity, _entity.entity_text, entity_after.entity_text, entity_after.values[entity_after.label])
                        '''
                    if entity_after.entity_type=="person":
                        if entity_after.values[entity_after.label]>=on_value_person:
                            if str(entity_after.label)=="1":
                                for i in range(len(roleList)):
                                    if roleList[i].role_name=="tenderee":
                                        roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                            elif str(entity_after.label)=="2":
                                for i in range(len(roleList)):
                                    if roleList[i].role_name=="agency":
                                        roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                            elif str(entity_after.label)=="3":
                                _list_entitys = [entity]+entity.linked_entitys
                                for _entity in _list_entitys:
                                    for i in range(len(roleList)):
                                        if roleList[i].entity_text==_entity.entity_text:
                                            if entity_after.sentence_index-_entity.sentence_index>1 and len(roleList[i].linklist)>0:
                                                break
                                            roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                    '''

                p_entity += 1

        p_entity += 1

    ''''''
    #删除一个机构有多个角色的数据
    # 1）第一个公司实体的招标人，则看看下一个实体是否为代理人，如果是则联系人错位连接 。2）在同一句中往后找联系人。3）连接不上在整个文章找联系人。
    temp_ent_list = []  # 临时列表，记录0,1角色及3联系人
    other_person = []  # 阈值以上的联系人列表
    link_person = []   # 有电话没联系上角色的person列表
    other_ent = []
    link_ent = []
    found_person = False
    ent_list = []
    for entity in list_entity:
        if entity.entity_type in ['org','company','person']:
            ent_list.append(entity)
    #for list_index in range(len(ent_list)):
    #if ent_list[list_index].entity_type in ['org','company'] and ent_list[list_index].label == 0 and list_index+2<len(ent_list) and \
    #ent_list[list_index+1].entity_type in ['org','company'] and ent_list[list_index+1].label == 1 and ent_list[list_index+2].entity_type in ['person']:
    #ent_list[list_index+1], ent_list[list_index+2] = ent_list[list_index+2], ent_list[list_index+1]
    for index in range(len(ent_list)):
        entity = ent_list[index]
        if entity.entity_type=="person":
            if entity.values[entity.label]>on_value_person:
                if str(entity.label)=="1":
                    for i in range(len(roleList)):
                        if roleList[i].role_name=="tenderee":
                            roleList[i].linklist.append((entity.entity_text,entity.person_phone))
                            link_person.append(entity.entity_text)
                            link_ent.append(roleList[i].entity_text)
                elif str(entity.label)=="2":
                    for i in range(len(roleList)):
                        if roleList[i].role_name=="agency":
                            roleList[i].linklist.append((entity.entity_text,entity.person_phone))
                            link_person.append(entity.entity_text)
                            link_ent.append(roleList[i].entity_text)
                elif str(entity.label)=="3":
                    #not_link_person.append((entity_after.entity_text,entity_after.person_phone))
                    other_person.append(entity.entity_text)
                    temp_ent_list.append((entity.entity_text,entity.person_phone))

        #if entity.entity_text in roleSet:
        if entity.entity_text in set([ent.entity_text for ent in roleList]):
            if entity.label in [0,1]:
                other_ent.append(entity.entity_text)
                temp_ent_list.append((entity.entity_text, entity.label))
            for behind_index in range(index+1, len(ent_list)):
                entity_after = ent_list[behind_index]
                if entity_after.sentence_index-entity.sentence_index>=1 or entity_after.entity_type in ['org','company']:   # 只在本句中找联系人
                    break
                if entity_after.values is not None:
                    if entity_after.entity_type=="person":
                        if entity_after.values[entity_after.label]>on_value_person:
                            if str(entity_after.label)=="1":
                                for i in range(len(roleList)):
                                    if roleList[i].role_name=="tenderee":
                                        roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                                        link_person.append(entity_after.entity_text)
                                        link_ent.append(roleList[i].entity_text)
                            elif str(entity_after.label)=="2":
                                for i in range(len(roleList)):
                                    if roleList[i].role_name=="agency":
                                        roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                                        link_person.append(entity_after.entity_text)
                                        link_ent.append(roleList[i].entity_text)
                            elif str(entity_after.label)=="3":
                                for i in range(len(roleList)):
                                    if roleList[i].entity_text==entity.entity_text:
                                        #if entity_after.sentence_index-entity.sentence_index>1 and len(roleList[i].linklist)>0:
                                        #break
                                        roleList[i].linklist.append((entity_after.entity_text,entity_after.person_phone))
                                        link_person.append(entity_after.entity_text)
    not_link_person = [person for person in other_person if person not in link_person]
    not_link_ent = [ent for ent in other_ent if ent not in link_ent]
    if len(not_link_person) > 0 and len(not_link_ent) > 0 :
        item = temp_ent_list
        for i in range(len(item)):
            if item[i][0] in not_link_ent and item[i][1] == 0 and i+3 < len(item):
                if item[i+1][0] in other_ent and item[i+1][1] == 1 and item[i+2][0] in other_person and item[i+3][0] in other_person:
                    item[i+1], item[i+2] = item[i+2], item[i+1]
        for i in range(len(item)-1, -1, -1):
            if item[i][0] in not_link_ent:
                for role in roleList:
                    if role.entity_text == item[i][0] and len(role.linklist) < 1:
                        for j in range(i+1, len(item)):
                            if item[j][0] in not_link_person:
                                role.linklist.append(item[j])
                                break
                            else:
                                break

    for i in range(len(roleList)):
        if roleList[i].role_name=="win_tenderer":
            set_tenderer_role.add(roleList[i])
    if len(set_tenderer_money)==1 and len(set_tenderer_role)==1:
        list(set_tenderer_role)[0].money = list(set_tenderer_money)[0]

                                #删除一个机构有多个角色的数据
    #删除重复人、概率不回传
    final_roleList = []
    for i in range(len(roleList)):
        item = roleList[i].getString(roleList)
        if item:
            final_roleList.append(item)
    return final_roleList

def getPackageRoleMoney(list_sentence,list_entity):
    '''
    @param:
        list_sentence:文章的句子list
        list_entity:文章的实体list
    @return: 拿到文章的包-标段号-角色-实体名称-金额-联系人-联系电话
    '''

    theRole = getRoleList(list_sentence,list_entity)
    if not theRole:
        return []
    RoleList,RoleSet,PackageList,PackageSet = theRole
    RoleList = findAttributeAfterEntity(RoleList, RoleSet, PackageList, PackageSet, list_entity)
    return RoleList

def getPREMs(list_sentences,list_entitys,list_articles):
    '''
    @param:
        list_sentence:所有文章的句子list
        list_entity:所有文章的实体list
    @return:list of dict which include文章的包-角色-实体名称-金额-联系人-联系电话
    '''
    result = []
    for list_sentence,list_entity,list_article in zip(list_sentences,list_entitys,list_articles):

        RoleList = getPackageRoleMoney(list_sentence,list_entity)
        result.append([list_article.id,{"prem":RoleList}])
    return result


if __name__=="__main__":
    '''
    conn = getConnection()
    cursor = conn.cursor()
    #sql = " select distinct A.doc_id from entity_mention A,test_predict_role B where A.entity_id=B.entity_id limit 200"
    sql = " select B.doc_id,B.prem from articles_processed A, articles_validation B where A.id=B.doc_id "
    
    result = []
    
    cursor.execute(sql)
    rows = cursor.fetchall()
    count = 0
    for row in rows:
        
        count += 1
        print(count)
        doc_id = row[0]
        
        roleList = getPackageRoleMoney(doc_id)
        result.append([doc_id,str(roleList),row[1]])
        ''''''
    with codecs.open("getAttribute.html","w",encoding="utf8") as f:
        f.write('<html><head>\
        <meta http-equiv="Content-Type"\
        content="text/html; charset=UTF-8">\
        </head>\
        <body bgcolor="#FFFFFF">\
        <table border="1">\
        <tr>\
        <td>doc_id</td>\
        <td>角色</td>\
        </tr>')
        for item in result:
            f.write("<tr>"+"<td>"+item[0]+"</td>"+"<td>"+item[1]+"</td>"+"<td>"+item[2]+"</td>"+"</tr>")
        f.write("</table></body>")
    '''