BIDI_ML_INFO_EXTRACTION/BiddingKG/dl/money/money_keras.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Author  : bidikeji
# @Time    : 2021/7/27 0027 15:05 

import os
import sys
import h5py
from keras import models,layers,losses,optimizers
sys.path.append(os.path.abspath("../../.."))
import pandas as pd
import math
from keras.callbacks import ModelCheckpoint
from BiddingKG.dl.common.Utils import *
import tensorflow as tf
from keras.models import load_model

lb = ['招标金额','中标金额','其他金额']
id2lb = {k:v for k,v in enumerate(lb)}
lb2id = {v:k for k,v in id2lb.items()}
seq_len = 30

def labeling(label, out_len=3):
    out = np.zeros((out_len))
    out[label] = 1
    return out


def getTrainData(percent=0.9):
    df = pd.read_excel('traindata/2兼职标注数据_test22.xlsx')
    df2 = pd.read_excel('traindata/原金额模型标注数据.xls')
    df = df.append(df2, ignore_index=True)
    df.dropna(subset=['left'], inplace=True)
    df.fillna('', inplace=True)
    if 'relabel' in df.columns:
        df['label'] = df.apply(lambda x:x['relabel'] if x['relabel']!="" else x['label'], axis=1)
        print('更新标注完成')
    for i in df.index:
        if df.loc[i, 'label'] not in lb:
            print('标签错误：',df.loc[i, 'label'])
    df['label'] = df['label'].apply(lambda x:lb2id.get(x, 0))

    print('总样本：', len(df))
    train_x = []
    train_y = []
    test_x = []
    test_y = []

    for before, text, after, label in zip(df["left"], df["center"], df["right"], df["label"]):
        before = str(before) if str(before) != "nan" else ""
        text = str(text)
        after = str(after) if str(after) != "nan" else ""

        x = encodeInput([before, text, after], word_len=seq_len, word_flag=True, userFool=False)
        y = labeling(label)
        if np.random.random() < percent:
            train_x.append(x)
            train_y.append(y)
        else:
            test_x.append(x)
            test_y.append(y)
    return np.transpose(np.array(train_x), (1, 0, 2)), np.array(train_y), np.transpose(np.array(test_x),
                                                                                       (1, 0, 2)), np.array(test_y)
def word2id(df):
    train_x = []
    train_y = []
    test_x = []
    test_y = []

    for before, text, after, label in zip(df["left"], df["center"], df["right"], df["label"]):
        before = str(before) if str(before) != "nan" else ""
        text = str(text)
        after = str(after) if str(after) != "nan" else ""

        x = encodeInput([before, text, after], word_len=seq_len, word_flag=True, userFool=False)
        y = labeling(label)
        train_x.append(x)
        train_y.append(y)
    return np.transpose(np.array(train_x), (1, 0, 2)), np.array(train_y)


def train():
    # pk_file = "traindata/all_data.pk"
    # if os.path.exists(pk_file):
    #     train_x, train_y, test_x, test_y = load(pk_file)
    # else:
    #     train_x, train_y, test_x, test_y = getTrainData()
    #     save([train_x, train_y, test_x, test_y], pk_file)

    df_train = pd.read_excel('traindata/df_train.xlsx')
    df_test = pd.read_excel('traindata/df_test.xlsx')
    train_x, train_y = word2id(df_train)
    test_x, test_y = word2id(df_test)
    with tf.Session() as sess:
        vocab, matrix = getVocabAndMatrix(getModel_word())
        model = getBiLSTMModel(input_shape=(3, seq_len, 60), vocab=vocab, embedding_weights=matrix, classes=3)
        print("loading weights")
        # model.load_weights("log/ep378-loss0.178-val_loss0.117-f1_score0.965.h5",by_name=True, skip_mismatch=True)

        callback = ModelCheckpoint(
            filepath="log/" + "ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}-f1_score{val_f1_score:.3f}.h5",
            monitor="val_loss", save_best_only=True, save_weights_only=True, mode="min")
        model.fit(x=[train_x[0], train_x[1], train_x[2]], y=train_y, batch_size=128, epochs=600,callbacks=[callback],
                  validation_data=[[test_x[0], test_x[1], test_x[2]], test_y]) #


def test(_span = ['：预算金额1000000元，中标金额', '1151元', '；']):
    input = encodeInput(_span, word_len=seq_len, word_flag=True, userFool=False)
    print(input)
    graph = tf.get_default_graph()
    with graph.as_default():
        sess = tf.Session(graph=graph)
        with sess.as_default():
            vocab, matrix = getVocabAndMatrix(getModel_word())
            model = getBiLSTMModel(input_shape=(3, seq_len, 60), vocab=vocab, embedding_weights=matrix,
                                   classes=3)
            model.load_weights("log/ep007-loss0.079-val_loss0.099-f1_score0.966.h5", by_name=True, skip_mismatch=True)
            logit = model.predict([np.array([input[0]]), np.array([input[1]]), np.array([input[2]])])
            print(logit)
            return logit

def get_savedModel():
    sess = tf.Session(graph=tf.Graph())
    with sess.as_default():
        with sess.graph.as_default():
            vocab, matrix = getVocabAndMatrix(getModel_word())
            model = getBiLSTMModel(input_shape=(3, seq_len, 60), vocab=vocab, embedding_weights=matrix, classes=3)
            sess.run(tf.global_variables_initializer())
            # model.load_weights(filepath="log/ep009-loss0.057-val_loss0.076-f1_score0.978.h5")
            # model.load_weights(filepath="log/ep007-loss0.079-val_loss0.099-f1_score0.966.h5") # 2021/7/27调整模型30字最优模型
            model.load_weights(filepath="../../dl_dev/money/log/ep029-loss0.081-val_loss0.094-f1_score0.971.h5") # 2021/08/06 调整模型30字最优模型
            tf.saved_model.simple_save(session=sess,
                                       # export_dir="money_savedmodel20210727_3",
                                       export_dir="money_savedmodel20210806",
                                       inputs={"input0": model.input[0],
                                               "input1": model.input[1],
                                               "input2": model.input[2]},
                                       outputs={"outputs": model.output})

def tensorboard_model():
    with tf.Session(graph=tf.Graph()).as_default() as sess:
        with sess.graph.as_default():
            tf.saved_model.loader.load(sess, tags=["serve"], export_dir="money_savedmodel1")
            tf.summary.FileWriter(graph=sess.graph, logdir="log2")

def getBiLSTMModel(input_shape,vocab,embedding_weights,classes,use_am=False):
    assert len(input_shape)==3
    list_input = []
    for i in range(input_shape[0]):
        list_input.append(layers.Input(shape=(input_shape[1],),dtype=tf.int32,name="input%d"%(i)))
    print("list_input",list_input)
    list_embedding = []

    embedding_input = list_input
    embedding = layers.Embedding(len(vocab),input_shape[2],
                                 weights=[embedding_weights] if embedding_weights is not None else None,
                                 trainable=True,name="char_embeding")
    for i in range(len(embedding_input)):
        print(i)
        list_embedding.append(embedding(embedding_input[i]))
        print(list_embedding)

    list_w2v = list_embedding
    list_lstm = []

    list_lstm.append(layers.Bidirectional(layers.LSTM(32, dropout=0.5, recurrent_dropout=0.5))(list_w2v[0]))
    list_lstm.append(layers.Bidirectional(layers.LSTM(8, dropout=0.5, recurrent_dropout=0.5))(list_w2v[1]))
    list_lstm.append(layers.Bidirectional(layers.LSTM(16, dropout=0.5, recurrent_dropout=0.5))(list_w2v[2]))

    concat = layers.concatenate(list_lstm)
    dropout = layers.Dropout(0.5)(concat)

    out = layers.Dense(classes,activation="softmax")(dropout)

    model = models.Model(list_input,out)
    model.compile(optimizer=optimizers.Adam(lr=0.001),loss=losses.categorical_crossentropy,metrics=[precision,recall,f1_score])
    model.summary()

    return model
def verification():
    graph = tf.get_default_graph()
    with graph.as_default():
        sess = tf.Session(graph=graph)
        with sess.as_default():
            vocab, matrix = getVocabAndMatrix(getModel_word())
            model = getBiLSTMModel(input_shape=(3, seq_len, 60), vocab=vocab, embedding_weights=matrix,
                                   classes=3)
            model.load_weights("log/ep029-loss0.081-val_loss0.094-f1_score0.971.h5", by_name=True, skip_mismatch=True)

            df_val = pd.read_excel('traindata/df_val_predict.xlsx')
            val_x, val_y = word2id(df_val)
            logit = model.predict([val_x[0], val_x[1], val_x[2]])
            lg = np.argmax(logit, axis=-1)
            df_val['pred_kera'] = pd.DataFrame(lg)
            df_val['prob_kera'] = pd.DataFrame(np.amax(logit, axis=1))
            df_val['tf=kera'] = df_val.apply(lambda x:1 if x['pred_kera']==x['pred_tf'] else 0, axis=1)
            df_val['tf=lb'] = df_val.apply(lambda x:1 if x['label']==x['pred_tf'] else 0, axis=1)
            df_val['kera=lb'] = df_val.apply(lambda x:1 if x['pred_kera']==x['label'] else 0, axis=1)
            df_val.to_excel('traindata/df_val_predict2.xlsx')



            df = pd.read_excel('traindata/2兼职标注数据_test22.xlsx')
            df.fillna('', inplace=True)
            df.reset_index(drop=True, inplace=True)
            preds = []
            if 'relabel' in df.columns:
                df['label'] = df.apply(lambda x:x['relabel'] if x['relabel']!="" else x['label'], axis=1)
                print('更新标注完成')
            for left, center, right, label in zip(df['left'], df['center'], df['right'], df['label']):
                _span=[left, center, right]
                input = encodeInput(_span, word_len=seq_len, word_flag=True, userFool=False)
                logit = model.predict([np.array([input[0]]), np.array([input[1]]), np.array([input[2]])])
                lg = np.argmax(logit, axis=-1)[0]
                prob = logit[0][lg]
                lg = id2lb.get(lg, '')
                preds.append(lg)
                # if lg != label:
                #     print(left, '###', center, '###', right)
                #     print('预测类别：%s, 预测：%.4f, 标签：%s'%(lg, prob, label))
                #     print()
            df['pred'] = pd.DataFrame(preds)
            df.to_excel('traindata/2兼职标注数据_test22_predict.xlsx')

if __name__ == "__main__":
    # train()
    verification()
    # test(_span=['预算金额：50万，中标金额：','100.600万','元，'])
    # get_savedModel()
    # tensorboard_model()