ORGS_FUSION/metrics.py

import tensorflow as tf
import numpy as np
import scipy
import json
from fuzzywuzzy import fuzz
import gc
from utils import loadfile
import heapq
import time
import os


dir = os.getcwd()+"\\data1\\100000\\zh_en\\"


def masked_softmax_cross_entropy(preds, labels, mask):
    """Softmax cross-entropy loss with masking."""
    loss = tf.nn.softmax_cross_entropy_with_logits(logits=preds, labels=labels)
    mask = tf.cast(mask, dtype=tf.float32)
    mask /= tf.reduce_mean(mask)
    loss *= mask
    return tf.reduce_mean(loss)


def masked_accuracy(preds, labels, mask):
    """Accuracy with masking."""
    correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(labels, 1))
    accuracy_all = tf.cast(correct_prediction, tf.float32)
    mask = tf.cast(mask, dtype=tf.float32)
    mask /= tf.reduce_mean(mask)
    accuracy_all *= mask
    return tf.reduce_mean(accuracy_all)


def get_placeholder_by_name(name):
    try:
        return tf.get_default_graph().get_tensor_by_name(name+":0")
    except:
        return tf.placeholder(tf.int32, name=name)


#
def align_loss(outlayer, ILL, gamma, k, AE=True):
    left = ILL[:, 0]
    right = ILL[:, 1]
    t = len(ILL)
    negative_ILL = loadfile(dir+"ref_ent_ids_neg", 2)

    # 正样本
    # embedding_lookup 就是返回 outlayer的第left个Tensor，但不只是查表，同样也会更新参数
    left_x = tf.nn.embedding_lookup(outlayer, left)
    right_x = tf.nn.embedding_lookup(outlayer, right)

    # reduce_sum：计算指定维度的和
    A = tf.reduce_sum(tf.abs(left_x - right_x), 1)

    # 负样本
    # 如果是AE则使用随机选取负样本，如果是SE则人工筛选负样本
    if AE:
        neg_left = get_placeholder_by_name("neg_left") #tf.placeholder(tf.int32, [t * k], "neg_left")
        neg_right = get_placeholder_by_name("neg_right") #tf.placeholder(tf.int32, [t * k], "neg_right")
    else:
        negative_ILL = negative_ILL[:t*k]
        np.random.shuffle(negative_ILL)
        negative_ILL = np.array(negative_ILL)
        neg_left = negative_ILL[:, 0]
        neg_right = negative_ILL[:, 1]

    neg_l_x = tf.nn.embedding_lookup(outlayer, neg_left)
    neg_r_x = tf.nn.embedding_lookup(outlayer, neg_right)
    B = tf.reduce_sum(tf.abs(neg_l_x - neg_r_x), 1)
    C = - tf.reshape(B, [t, k])
    D = A + gamma
    L1 = tf.nn.relu(tf.add(C, tf.reshape(D, [t, 1])))
    neg_left = get_placeholder_by_name("neg2_left") #tf.placeholder(tf.int32, [t * k], "neg2_left")
    neg_right = get_placeholder_by_name("neg2_right") #tf.placeholder(tf.int32, [t * k], "neg2_right")
    neg_l_x = tf.nn.embedding_lookup(outlayer, neg_left)
    neg_r_x = tf.nn.embedding_lookup(outlayer, neg_right)
    B = tf.reduce_sum(tf.abs(neg_l_x - neg_r_x), 1)
    C = - tf.reshape(B, [t, k])
    L2 = tf.nn.relu(tf.add(C, tf.reshape(D, [t, 1])))
    return (tf.reduce_sum(L1) + tf.reduce_sum(L2)) / (2.0 * k * t)


def get_hits(vec, test_pair, top_k=(1, 10, 30, 50)):
    # print("vec ===================")
    # print(vec.shape)

    # 取每个在test_pair中vec对应的的左右两个向量
    Lvec = np.array([vec[e1] for e1, e2 in test_pair])
    Rvec = np.array([vec[e2] for e1, e2 in test_pair])
    # print("Lvec ===================")
    # print(Lvec.shape)
    # print("Rvec ===================")
    # print(Rvec.shape)
    # 计算两个向量的相似度
    sim = scipy.spatial.distance.cdist(Lvec, Rvec, metric='cityblock')
    # print("sim ====================")
    # print(len(sim))
    # print(len(sim[0]))
    # 计算
    top_lr = [0] * len(top_k)
    for i in range(Lvec.shape[0]):
        # argsort: 从小到大排序，并只取排序好的索引
        rank = sim[i, :].argsort()
        rank_index = np.where(rank == i)[0][0]

        for j in range(len(top_k)):
            if rank_index < top_k[j]:
                top_lr[j] += 1

    # print("rank ====================")
    # print(np.where(rank == 0)[0])
    #
    # print("top_lr ====================")
    # print(len(top_lr))

    top_rl = [0] * len(top_k)
    for i in range(Rvec.shape[0]):
        rank = sim[:, i].argsort()
        rank_index = np.where(rank == i)[0][0]
        for j in range(len(top_k)):
            if rank_index < top_k[j]:
                top_rl[j] += 1
    # print("top_rl ====================")
    # print(top_rl)

    print('For each left:')
    for i in range(len(top_lr)):
        print('Hits@%d: %.2f%%' % (top_k[i], top_lr[i] / len(test_pair) * 100))
    print('For each right:')
    for i in range(len(top_rl)):
        print('Hits@%d: %.2f%%' % (top_k[i], top_rl[i] / len(test_pair) * 100))


def get_combine_hits(se_vec, ae_vec, beta, test_pair, top_k=(1, 10, 30, 50)):
    vec = np.concatenate([se_vec*beta, ae_vec*(1.0-beta)], axis=1)
    get_hits(vec, test_pair, top_k)


def predict(vec, test_pair):
    # 读取Modelid 和 RealId 映射字典
    map_dict = loadDict(dir+"ModelId2RealId")
    id_list = file2Data(dir+"ent_ids_1_real")

    # 反转key-value，方便查找
    map_dict_reverse = {}
    [map_dict_reverse.update({str(v): int(k)}) for k, v in map_dict.items()]

    # ID list 转 dict，方便查找
    id_dict = {}
    for i in range(len(id_list)):
        ss = id_list[i][:-1].split("\t")
        id_dict[str(ss[0])] = ss[1]

    # 取test_pair左右两个ID
    Lid = np.array([[e1] for e1, e2 in test_pair])
    Rid = np.array([[e2] for e1, e2 in test_pair])
    # 取每个在test_pair中vec对应的的左右两个向量
    Lvec = np.array([vec[e1] for e1, e2 in test_pair])
    Rvec = np.array([vec[e2] for e1, e2 in test_pair])
    # 计算两个向量的相似度
    sim = scipy.spatial.distance.cdist(Lvec, Rvec, metric='cityblock')

    # model ID对应similar分数输出字典
    ModelId_sim_dict = {}
    RealId_sim_dict = {}
    Org_sim_dict = {}
    for i in range(len(sim)):
        for j in range(len(sim[i])):
            # 跳过自己与自己的相似度
            if int(Lid[i]) == int(Rid[j]):
                continue
            ModelId_sim_dict[(int(Lid[i]), int(Rid[j]))] = float(sim[i][j])
            # 取真正ID
            Lid_real = map_dict_reverse[str(Lid[i][0])]
            Rid_real = map_dict_reverse[str(Rid[j][0])]
            # 根据ID取公司名
            Lorg = id_dict[str(Lid_real)]
            Rorg = id_dict[str(Rid_real)]
            RealId_sim_dict[(str(Lid_real)+"\t"+str(Rid_real), Lorg+"\t"+Rorg)] = float(sim[i][j])
            Org_sim_dict[(Lorg, Rorg)] = float(sim[i][j])

    RealId_sim_sorted_list = sorted(RealId_sim_dict.items(), key=lambda x: x[1])
    Org_sim_sorted_list = sorted(Org_sim_dict.items(), key=lambda x: x[1])

    for i in range(20):
        print(Org_sim_sorted_list[i])



    # 根据阈值取符合条件的ID对
    # for i in range(len(RealId_sim_sorted_list)):
    #     length = RealId_sim_sorted_list[i][1]
    #     if length
    return


def predict_new(vec):
    dir_new_align = "C:\\Users\\admin\\Desktop\\Predict_Align_10w"

    with open(dir_new_align, "r+", encoding='UTF-8') as f:
        f.truncate()
        f.close()

    # 读取Modelid 和 RealId 映射字典
    id_list2 = file2Data(dir + "ent_ids_1")
    map_dict = loadDict(dir + "ModelId2RealId")
    id_list = file2Data(dir + "ent_ids_1_real")
    align_list = file2Data(dir + "ref_ent_ids_real")

    # 反转key-value，方便查找
    map_dict_reverse = {}
    [map_dict_reverse.update({str(v): int(k)}) for k, v in map_dict.items()]

    # ID list 转 dict，方便查找
    id_dict = {}
    for i in range(len(id_list)):
        ss = id_list[i][:-1].split("\t")
        if len(ss) < 2:
            continue
        id_dict[str(ss[0])] = ss[1]

    id_list3 = []
    for i in range(len(id_list2)):
        ss = id_list2[i][:-1].split("\t")
        if ss[1][0] == "O":
            # print(ss[1])
            id_list3.append(ss[0])

    # 取ID
    id = np.array([[e] for e in id_list3])

    # 取ID对应向量计算相似度
    id_vec = np.array([vec[int(k)] for k in id_list3])

    # 清内存
    del vec
    del id_list2
    del id_list3
    del map_dict
    del id_list
    gc.collect()

    # 分割成 2k * 2k 计算相似度
    splited = 2000
    print("len(id_vec)", len(id_vec))
    for i in range(0, len(id_vec), splited):
        splited_vec_left = id_vec[i:i+splited]
        if len(id_vec) - i < splited:
            splited_vec_left = id_vec[i:]

        for j in range(0, len(id_vec), splited):
            print("i,j", i, j)
            splited_vec_right = id_vec[j:j+splited]
            if len(id_vec) - j < splited:
                splited_vec_right = id_vec[j:]

            print("计算相似度")
            sim = scipy.spatial.distance.cdist(splited_vec_left, splited_vec_right, metric='cityblock')
            print("计算完成！")

            # print("转换：ID-相似度")
            # start_time = time.time()
            # RealId_sim_dict = {}
            # for k in range(len(sim)):
            #     # print("k", k)
            #     for m in range(len(sim[k])):
            #
            #         if i+k >= len(id) or j+m >= len(id):
            #             # print("i, k, m", i, k, m)
            #             continue
            #
            #         # 跳过自己与自己的相似度
            #         if int(id[i+k]) == int(id[j+m]):
            #             continue
            #
            #         # 跳过相似度距离大于1的
            #         if float(sim[k][m]) > 0.6:
            #             # print(k, m, sim[k][m])
            #             continue
            #
            #         # 取真正ID
            #         Lid_real = map_dict_reverse[str(id[i+k][0])]
            #         Rid_real = map_dict_reverse[str(id[j+m][0])]
            #         # print("Lid_real/Rid_real", Lid_real, Rid_real)
            #         # 根据ID取公司名
            #         Lorg = id_dict[str(Lid_real)]
            #         Rorg = id_dict[str(Rid_real)]
            #         RealId_sim_dict[(str(Lid_real)+"\t"+str(Rid_real), Lorg+"\t"+Rorg)] = float(sim[k][m])
            #
            # print("len(RealId_sim_dict)", len(RealId_sim_dict))
            # print("排序")
            # RealId_sim_sorted_list = sorted(RealId_sim_dict.items(), key=lambda x: x[1])
            # RealId_sim_sorted_list = RealId_sim_sorted_list[0:200]
            # print("sort", time.time()-start_time)
            # print(RealId_sim_sorted_list[:5])

            start_time = time.time()
            c_dict = {}
            for k in range(len(sim)):
                if i+k >= len(id):
                    continue

                # print("取前10")
                c = heapq.nsmallest(10, range(len(sim[k])), sim[k].take)
                # print("转换：ID-相似度")
                for index in range(len(c)):
                    Lid_real = map_dict_reverse[str(id[i+k][0])]
                    Rid_real = map_dict_reverse[str(id[j+c[index]][0])]
                    Lorg = id_dict[str(Lid_real)]
                    Rorg = id_dict[str(Rid_real)]
                    if Lid_real == Rid_real:
                        continue
                    c_dict[(str(Lid_real)+"\t"+str(Rid_real), Lorg+"\t"+Rorg)] = float(sim[k][c[index]])
            print("字典去重")
            c_sort = sorted(c_dict.items(), key=lambda x: x[1])
            c_sort = c_sort[0:200]
            print("headq", time.time()-start_time)
            # print(c_sort[:5])

            # 清内存
            print("清内存")
            del sim
            # del RealId_sim_dict
            del c_dict
            gc.collect()

            print("计算编辑距离")
            new_align_list = []
            for k in range(len(c_sort)):

                # print(RealId_sim_sorted_list[0:5])

                ids = c_sort[k][0][0]
                id1 = ids.split("\t")[0]
                id2 = ids.split("\t")[1]
                orgs = c_sort[k][0][1]
                org1 = orgs.split("\t")[0]
                org2 = orgs.split("\t")[1]
                length = c_sort[k][1]

                # 包含关系直接放入
                if (org1 in org2) or (org2 in org1):
                    if int(id1) <= int(id2):
                        new_align_list.append(id1 + "\t" + id2 + "\t" + org1 + "\t"
                                              + org2 + "\t" + str(length) + "\n")
                    else:
                        new_align_list.append(id2 + "\t" + id1 + "\t" + org2 + "\t"
                                              + org1 + "\t" + str(length) + "\n")
                # if org2 in org1:
                #     if int(id1) <= int(id2):
                #         new_align_list.append(ids + "\t" + orgs + str(length) + "\n")
                #     else:
                #         new_align_list.append(id2 + "\t" + id1 + "\t" + org2 + "\t"
                #                               + org1 + "\t" + str(length) + "\n")

                # 计算编辑距离
                sim = fuzz.ratio(org1, org2)

                if sim > 85:
                    print(org1, org2, sim)
                    for m in range(len(align_list)):

                        if int(id1) <= int(id2):
                            new_align_list.append(id1 + "\t" + id2 + "\t" + org1 + "\t"
                                                  + org2 + "\t" + str(length) + "\n")
                        else:
                            new_align_list.append(id2 + "\t" + id1 + "\t" + org2 + "\t"
                                                  + org1 + "\t" + str(length) + "\n")
            print("计算完成！")

            print("去重")
            new_align_list = list(set(new_align_list))
            print("len(new_align_list)", len(new_align_list))

            print("写入")
            data2FileAppend(new_align_list, dir_new_align)

            print("Finished right", j+splited)

        print("Finished left", i+splited)

    # 计算模型对齐的ID对在人工对齐的ID中的比例
    accuracy_num = 0
    new_num = 0
    model_align_list = file2Data(dir_new_align)

    model_align_list2 = []
    for i in range(len(model_align_list)):
        ss = model_align_list[i].split("\t")
        if int(ss[0]) > int(ss[1]):
            model_align_list2.append(ss[1]+"\t"+ss[0])
        else:
            model_align_list2.append(ss[0]+"\t"+ss[1])
    model_align_list2 = list(set(model_align_list2))

    for i in range(len(model_align_list2)):
        ss = model_align_list2[i].split("\t")
        ids = ss[0] + "\t" + ss[1]
        ids_reverse = ss[1] + "\t" + ss[0]

        for ids2 in align_list:
            if ids == ids2[:-1] or ids_reverse == ids2[:-1]:
                # print("right predict", model_align_list[i])
                accuracy_num += 1
                break

        if ids != ids2[:-1] and ids_reverse != ids2[:-1]:
            print("new predict", model_align_list[i])
            new_num += 1

    print("====================================")
    print("预测正确率", accuracy_num/len(model_align_list))
    print("predict right number", accuracy_num)
    print("predict new number", new_num)
    print("predict all number", len(model_align_list))
    print("====================================")



def loadDict(filename):
    with open(filename, "r") as json_file:
        dic = json.load(json_file)
    return dic


def file2Data(filename):
    with open(filename, 'r', encoding='UTF-8') as f:
        _list = f.readlines()
        f.close()
    return _list


def data2File(_list, filename):
    with open(filename, 'w', encoding='UTF-8') as f:
        f.writelines(_list)
        f.close()


def data2FileAppend(_list, filename):
    with open(filename, 'a+', encoding='UTF-8') as f:
        f.writelines(_list)
        f.close()