BIDI_EXTRACTION_AI/BiddingKG/dl/role/train.py

'''
Created on 2019年4月22日

@author: User
'''
# DEPRECATED(Phase1): 本文件中的 PostgreSQL 硬编码连接（host=192.168.*,
# password=postgres 等）将在后续 training/ Phase 迁移到 BiddingKG.dl.infra.db。
# 迁移完成前可临时使用：from BiddingKG.dl.infra.db import get_connection;
# conn = get_connection("<dbname>")
# 详见 ARCHITECTURE.md 第 11 章 Phase 1 与 REFACTOR_LOG.md。

import sys
import os
sys.path.append(os.path.abspath("../../.."))
from BiddingKG.dl.common.Utils import *
from keras.callbacks import ModelCheckpoint
from BiddingKG.dl.common.models import *
import pandas as pd
import keras
import numpy as np

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""





def loadTrainData(percent=0.9,line=False):
    # files = ["id_token_text_begin_end_label.pk","id_token_text_begin_end_label.pk1","id_token_text_begin_end_label-selffool.pk1"]
    # files = ["id_token_text_begin_end_label.pk","id_token_text_begin_end_label.pk1"]
    files = ["id_token_text_begin_end_label-moreTrue.pk"]
    data_x = []
    data_y = []
    #data_id = []
    test_x = []
    test_y = []
    test_id = []
    #_,_,_,_,_,test_id_before = load("all_data_selffool.pk_line")
    #test_id_before = set(test_id_before)
    dict_label_item = dict()
    #统计数据分布
    for file in files:
        data = load(file)
        for row in data:
            id = row[0]
            label = int(row[5])
            if label not in dict_label_item:
                dict_label_item[label] = set()
            dict_label_item[label].add(id)
    dict_label_num = dict()
    for _key in dict_label_item.keys():
        dict_label_num[_key] = int(len(dict_label_item[_key])*(1-percent))

    for file in files:
        data = load(file)
        _count = 0
        for row in data:
            #item_x = embedding_word(spanWindow(tokens=row[1],begin_index=row[3],end_index=row[4],size=100,center_include=True,word_flag=True), shape=(3,100,60))
            _span = spanWindow(tokens=row[1],begin_index=row[3],end_index=row[4],size=10,center_include=True,word_flag=True,text=row[2])
            item_x = encodeInput(_span, word_len=50, word_flag=True,userFool=False)

            if line:
                item_x = item_x[0]+item_x[1]+item_x[2]
            
            item_y = np.zeros([6])
            label = int(row[5])
            print(_span,label)
            _count += 1
            if label not in [0,1,2,3,4,5]:
                continue
            item_y[label] = 1
            if np.random.random()>0.5 and dict_label_num[label]>0:
                dict_label_num[label] -= 1
                test_x.append(item_x)
                test_y.append(item_y)
                test_id.append(row[0])
            else:
                data_x.append(item_x)
                data_y.append(item_y)
                #data_id.append(row[0])

            # if np.random.random()>percent:
            # # if row[0] not in test_id_before:
            #     data_x.append(item_x)
            #     data_y.append(item_y)
            #     #data_id.append(row[0])
            # else:
            #     test_x.append(item_x)
            #     test_y.append(item_y)
            #     test_id.append(row[0])
            
    print(np.shape(np.array(data_x)),np.shape(np.array(test_x)))
    print(dict_label_num)
    if line:
        return np.array(data_x),np.array(data_y),np.array(test_x),np.array(test_y),None,test_id
    else:
        return np.transpose(np.array(data_x),(1,0,2)),np.array(data_y),np.transpose(np.array(test_x),(1,0,2)),np.array(test_y),None,test_id

def train():
    # data_pk = "all_data_selffool_before-10.pk"
    data_pk = "all_data_selffool_moretrue-10.pk"
    # data_pk = "all_data_selffool_all-10.pk"
    if os.path.exists(data_pk):
        train_x,train_y,test_x,test_y,_,test_id = load(data_pk)
    else:
        train_x,train_y,test_x,test_y,_,test_id = loadTrainData()
        save((train_x,train_y,test_x,test_y,_,test_id),data_pk)
    with tf.Session(graph=tf.Graph()).as_default() as sess:
        with sess.graph.as_default():
          # dict_key_value = load("dict_key_value.pk")
          # model = getBiLSTMModel(input_shape=(3,50,256), vocab=fool_char_to_id.keys(), embedding_weights=dict_key_value["bert/embeddings/word_embeddings:0"], classes=6)
          vocab,matrix = getVocabAndMatrix(getModel_word())
          # model = getBiLSTMModel(input_shape=(3,50,60), vocab=vocab, embedding_weights=matrix, classes=6)
          model = getBiLSTMModel_entity(input_shape=(3,50,60), vocab=vocab, embedding_weights=matrix, classes=6)
          # model = getTextCNNModel(input_shape=(2,50,60), vocab=vocab, embedding_weights=matrix, classes=6)

          '''
          
          for k,v in dict_key_value.items():
              if re.search("encoder",k) is not None:
                  sess.run(tf.assign(sess.graph.get_tensor_by_name(k[13:]),v))
                  print(k)
          '''
          #model = getTextCNNModel(input_shape=(3,50,60), vocab=vocab, embedding_weights=weights, classes=6)
          # model.load_weights("log/ep044-loss0.142-val_loss0.200-f1_score0.934.h5",skip_mismatch=True,by_name=True)
          model.load_weights("log/min_val_loss_ep027-loss0.112-val_loss0.109-f1_score0.963.h5")

          #model.summary()
          #print("11111111111",sess.run(sess.graph.get_tensor_by_name("encoder/layer_0/attention/self/query/kernel:0")))
          callback = ModelCheckpoint(filepath="log/"+"min_val_loss_ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}-f1_score{val_f1_score:.3f}.h5",monitor="val_loss",save_best_only=True, save_weights_only=True, mode="min")
          callback1 = ModelCheckpoint(filepath="log/"+"min_loss_ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}-f1_score{val_f1_score:.3f}.h5",monitor="loss",save_best_only=True, save_weights_only=True, mode="min")

          history_model = model.fit(x=[train_x[0],train_x[1],train_x[2]],y=train_y,validation_data=([test_x[0],test_x[1],test_x[2]],test_y),epochs=600,batch_size=96,shuffle=True,callbacks=[callback,callback1])
          # history_model = model.fit(x=[train_x[0],train_x[1]],y=train_y,validation_data=([test_x[0],test_x[1]],test_y),epochs=600,batch_size=128,shuffle=True,callbacks=[callback,callback1])
          # history_model = model.fit(x=train_x,y=train_y,validation_data=(test_x,test_y),epochs=600,batch_size=128,shuffle=True,callbacks=[callback])
          #print("2222222222222",sess.run(sess.graph.get_tensor_by_name("encoder/layer_0/attention/self/query/kernel:0")))
    
def test():
  
    _span = ['：预算金额1000000元，中标金额', '1df元', '；']
    _input = encodeInput(_span, word_len=50, word_flag=True,userFool=True)
    print(_input)
    print(len(_input))
    print(len(_input[0]))
    print(len(_input[1]))
    print(len(_input[2]))

def statis():
    df = pd.read_excel("测试数据_role-biws-biw0.xls")
    result = {"正确-词":0,
              "错误-词":0,
              "正确-字":0,
              "错误-字":0}
    for i in range(6):
        result["正确-词"+str(i)] = 0
        result["错误-词"+str(i)] = 0
        result["正确-字"+str(i)] = 0
        result["错误-字"+str(i)] = 0
    
    for label_ws,prob_ws,label_w,prob_w,label_true in zip(df["list_newlabel"],df["list_newprob"],df["list_newlabel_cnn"],df["list_newprob_cnn"],df["label_true"]):
        if int(label_ws)==int(label_true):
            key = "正确-词"
            result[key] += 1
            result[key+str(int(label_ws))]+=1
        else:
            key = "错误-词"
            result[key] += 1
            result[key+str(int(label_ws))]+=1
        if int(label_w)==int(label_true):
            key = "正确-字"
            result[key] += 1
            result[key+str(int(label_w))]+=1
        else:
            key = "错误-字"
            result[key] += 1
            result[key+str(int(label_w))]+=1
    data = []
    for key in result.keys():
        data.append([key,result[key]])
    data.sort(key=lambda x:x[0])
    for item in data:
        print(item)

def val():
    data_pk = "all_data_selffool.pk_line"
    train_x,train_y,test_x,test_y,_,test_id = load(data_pk)
    vocab,matrix = getVocabAndMatrix(getModel_word())
    model = getBiLSTMModel(input_shape=(1,150,60), vocab=vocab, embedding_weights=matrix, classes=6)
    model.load_weights("log/ep064-loss0.585-val_loss0.634-f1_score0.927.h5")

    # predict_y = np.argmax(model.predict([test_x[0],test_x[1],test_x[2]]),-1)
    predict_y = np.argmax(model.predict(test_x),-1)
    dict_notTrue = dict()
    for _y,Y,_id in zip(predict_y,np.argmax(test_y,-1),test_id):
        if _y!=Y:
            dict_notTrue[_id] = [_y,Y]

    token_data = load("id_token_text_begin_end_label-selffool.pk1")
    test_before = []
    test_center = []
    test_after = []
    test_label = []
    test_predict = []
    for item in token_data:
        if item[0] in dict_notTrue:
            token = item[1]
            text = item[2]
            begin = item[3]
            end = item[4]
            predict,label = dict_notTrue[item[0]]
            _span = spanWindow(tokens=token,begin_index=begin,end_index=end,size=10,center_include=True,word_flag=True,text=text)
            before,center,after = _span
            test_before.append(before)
            test_center.append(center)
            test_after.append(after)
            test_label.append(label)
            test_predict.append(predict)
    data = {"test_before":test_before,"test_center":test_center,"test_after":test_after,"test_label":test_label,"test_predict":test_predict}
    df = pd.DataFrame(data)
    df.to_excel("val_bert_position.xls",columns=["test_before","test_center","test_after","test_label","test_predict"])


def get_savedmodel():
    with tf.Session(graph=tf.Graph()).as_default() as sess:
        with sess.graph.as_default():
            vocab,matrix = getVocabAndMatrix(getModel_word(),Embedding_size=60)
            # model = getBiLSTMModel(input_shape=(3,50,60), vocab=vocab, embedding_weights=matrix, classes=6)
            model = getBiLSTMModel_entity(input_shape=(3,50,60), vocab=vocab, embedding_weights=matrix, classes=6)
            # model = getTextCNNModel(input_shape=(2,50,60), vocab=vocab, embedding_weights=matrix, classes=6)
            # filepath = "log/ep001-loss0.087-val_loss0.172-f1_score0.944.h5"
            filepath = "../../dl_dev/role/log/min_val_loss_ep034-loss0.070-val_loss0.068-f1_score0.975.h5"
            model.load_weights(filepath)
            tf.saved_model.simple_save(sess,
                                       "role_savedmodel/",
                                       inputs={"input0":model.input[0],
                                               "input1":model.input[1],
                                               "input2":model.input[2]},
                                       outputs={"outputs":model.output}
                                       )


def get_tensorboard():
    with tf.Session(graph=tf.Graph()) as sess:
        tf.saved_model.loader.load(sess,export_dir="role_savedmodel",tags=["serve"])
        writer = tf.summary.FileWriter(graph=sess.graph,logdir="log2")

def relabel():
    list_id = []
    list_before = []
    list_center = []
    list_after = []
    list_label = []
    files = ["id_token_text_begin_end_label.pk","id_token_text_begin_end_label.pk1"]
    for file in files:
        data = load(file)
        _count = 0
        for row in data:
            #item_x = embedding_word(spanWindow(tokens=row[1],begin_index=row[3],end_index=row[4],size=100,center_include=True,word_flag=True), shape=(3,100,60))
            _span = spanWindow(tokens=row[1],begin_index=row[3],end_index=row[4],size=15,center_include=True,word_flag=True,text=row[2])
            _label = row[5]
            if int(_label) in [3,4]:
                list_id.append(row[0])
                list_before.append(_span[0])
                list_center.append(_span[1])
                list_after.append(_span[2])
                list_label.append(str(_label))
    df = pd.DataFrame({"list_id":list_id,
                       "list_before":list_before,
                       "list_center":list_center,
                       "list_after":list_after,
                       "list_label":list_label})
    df.to_excel("relabel_1.xls",columns=["list_id","list_before","list_center","list_after","list_label"])

def generate_data():
    file_before = "D:\\myProject\\traindata\\"
    files = ["id_token_text_begin_end_label.pk","id_token_text_begin_end_label.pk1"]
    data = load(file_before+"id_token_text_begin_end_label-selffool.pk1")
    df = pd.read_excel(file_before+"relabel_1.xls")
    set_id = set(df["list_id"])
    for file in files:
        temp_data = load(file_before+file)
        for row in temp_data:
            if row[0] in set_id:
                # print(row)
                data.append(row)
    save(data,file_before+"id_token_text_begin_end_label-moreTrue.pk")


if __name__=="__main__":
    # loadTrainData()
    train()
    # relabel()
    # generate_data()
    test()
    #statis()
    # val()
    # get_savedmodel()
    # get_tensorboard()
    pass