import sys import os sys.path.append(os.path.abspath("../..")) # sys.path.append('/data/python_znj/znj/BIDI_ML_INFO_EXTRACTION/') import pandas as pd import re import psycopg2 from keras.callbacks import ModelCheckpoint from keras import layers,models,optimizers,losses from keras.layers import * from BiddingKG.dl.common.Utils import * from BiddingKG.dl.common.models import * from sklearn.metrics import classification_report from sklearn.utils import shuffle,class_weight import matplotlib.pyplot as plt import random input_shape = (2,30,60) input_shape2 = (2,40,128) # output_shape = [4] time_label_dict = { 'time': 0, 'time_release': 1, #发布时间 'time_bidopen': 2, #开标时间 'time_bidclose': 3, #截标时间 'time_bidstart': 12, #投标（开始）时间、响应文件接收（开始）时间 'time_publicityStart': 4, #公示开始时间（公示时间、公示期） 'time_publicityEnd': 5, #公示截止时间 'time_getFileStart': 6, #文件获取开始时间（文件获取时间） 'time_getFileEnd': 7, #文件获取截止时间 'time_registrationStart': 8, #报名开始时间（报名时间） 'time_registrationEnd': 9, #报名截止时间 'time_earnestMoneyStart': 10, #保证金递交开始时间（保证金递交时间） 'time_earnestMoneyEnd': 11, #保证金递交截止时间 'time_commencement': 13, #开工日期 'time_completion': 14 #竣工日期 } output_shape = [len(time_label_dict)] def get_data(): data_load = pd.read_csv("newdata_30_prc.csv", index_col=0) id_set = set() for id in data_load['document_id']: id_set.add(id) conn = psycopg2.connect(dbname="iepy", user="postgres", password="postgres", host="192.168.2.103") sql = "SELECT A.human_identifier,A.sentences,A.tokens,A.offsets_to_text,B.value " \ "FROM corpus_iedocument A,brat_bratannotation B " \ "WHERE A.human_identifier = '%s' " \ "AND A.human_identifier = B.document_id " db_data = [] count = 0 for id in list(id_set): count+=1 print(count) cur1 = conn.cursor() cur1.execute(sql % (id)) db_data.extend(cur1.fetchall()) cur1.close() conn.close() columns = ['document_id','sentences','tokens','offsets_to_text','value'] df = pd.DataFrame(db_data, columns=columns) df = df[df['value'].str.contains('time')] df = df.reset_index(drop=True) print(len(df)) time_label = df['value'].str.split(expand=True) time_label.columns = ['_', 'label_type', 'begin_index', 'end_index', 'entity_text'] time_label = time_label.drop('_', axis=1) df = pd.concat([df, time_label], axis=1) print(df.info()) df['tokens'] = [token[2:-2].split("', '") for token in df['tokens']] df['sentences'] = [eval(sentence) for sentence in df['sentences']] # df['sentences'] = [sentence[1:-1].split(", ") for sentence in df['sentences']] # df['sentences'] = [[int(s) for s in sentence] for sentence in df['sentences']] df['offsets_to_text'] = [eval(offset) for offset in df['offsets_to_text']] # df['offsets_to_text'] = [offset[1:-1].split(", ") for offset in df['offsets_to_text']] # df['offsets_to_text'] = [[int(o) for o in offset] for offset in df['offsets_to_text']] save(df,'db_time_data.pk') def getModel(): ''' @summary: 时间分类模型 ''' L_input = layers.Input(shape=input_shape2[1:], dtype='float32') R_input = layers.Input(shape=input_shape2[1:], dtype='float32') L_lstm = layers.Bidirectional(layers.LSTM(40,return_sequences=True,dropout=0.1))(L_input) # L_lstm = layers.LSTM(32,return_sequences=True,dropout=0.2)(L_input) avg_l = layers.GlobalAveragePooling1D()(L_lstm) R_lstm = layers.Bidirectional(layers.LSTM(40,return_sequences=True,dropout=0.1))(R_input) # R_lstm = layers.LSTM(32, return_sequences=True, dropout=0.2)(R_input) avg_r = layers.GlobalAveragePooling1D()(R_lstm) concat = layers.merge([avg_l, avg_r], mode='concat') # lstm = layers.LSTM(24,return_sequences=False,dropout=0.2)(concat) output = layers.Dense(output_shape[0],activation="softmax")(concat) model = models.Model(inputs=[L_input,R_input], outputs=output) learn_rate = 0.0005 model.compile(optimizer=optimizers.Adam(lr=learn_rate), loss=losses.binary_crossentropy, metrics=[precision,recall,f1_score]) model.summary() return model def getModel2(): ''' @summary: 时间分类模型 ''' L_input = layers.Input(shape=input_shape2[1:], dtype='float32') L_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(L_input) R_input = layers.Input(shape=input_shape2[1:], dtype='float32') R_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(R_input) L_input_drop = Dropout(0.3)(L_input) R_input_drop = Dropout(0.3)(R_input) # L_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(L_input) L_lstm = OurBidirectional(GRU(64, return_sequences=True))([L_input_drop,L_mask]) L_att = Attention02()(L_lstm,mask=K.squeeze(L_mask,axis=-1)) # R_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(R_input) R_lstm = OurBidirectional(GRU(64, return_sequences=True))([R_input_drop,R_mask]) R_att = Attention02()(R_lstm,mask=K.squeeze(R_mask,axis=-1)) L_R = layers.merge([L_lstm, R_lstm],concat_axis=1, mode='concat') L_R_mask = layers.merge([L_mask, R_mask],concat_axis=1, mode='concat') L_R_att = Attention02()(L_R,mask=K.squeeze(L_R_mask,axis=-1)) L_att = layers.add([L_att,L_R_att]) R_att = layers.add([R_att,L_R_att]) concat = layers.merge([L_att, R_att], mode='concat') concat = Dropout(0.2)(concat) output = layers.Dense(output_shape[0],activation="softmax")(concat) model = models.Model(inputs=[L_input,R_input], outputs=output) learn_rate = 0.00005 model.compile(optimizer=optimizers.Adam(lr=learn_rate), loss=losses.binary_crossentropy, metrics=[precision,recall,f1_score]) model.summary() return model # def getModel2(): # ''' # @summary: 时间分类模型 # ''' # L_input = layers.Input(shape=input_shape2[1:], dtype='float32') # L_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(L_input) # R_input = layers.Input(shape=input_shape2[1:], dtype='float32') # R_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(R_input) # # L_input_drop = Dropout(0.3)(L_input) # R_input_drop = Dropout(0.3)(R_input) # # L_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(L_input) # L_lstm = OurBidirectional(GRU(64, return_sequences=True))([L_input_drop,L_mask]) # L_att = Attention02()(L_lstm,mask=K.squeeze(L_mask,axis=-1)) # # R_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(R_input) # R_lstm = OurBidirectional(GRU(64, return_sequences=True))([R_input_drop,R_mask]) # R_att = Attention02()(R_lstm,mask=K.squeeze(R_mask,axis=-1)) # concat = layers.merge([L_att, R_att], mode='concat') # # concat = Dropout(0.2)(concat) # output = layers.Dense(output_shape[0],activation="softmax")(concat) # # model = models.Model(inputs=[L_input,R_input], outputs=output) # # learn_rate = 0.00005 # model.compile(optimizer=optimizers.Adam(lr=learn_rate), # loss=losses.binary_crossentropy, # metrics=[precision,recall,f1_score]) # model.summary() # return model def getModel3(): ''' @summary: 时间分类模型 ''' L_input = layers.Input(shape=input_shape2[1:], dtype='float32') L_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(L_input) R_input = layers.Input(shape=input_shape2[1:], dtype='float32') R_mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(R_input) L_input_drop = Dropout(0.3)(L_input) R_input_drop = Dropout(0.3)(R_input) # L_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(L_input) L_lstm = OurBidirectional(GRU(64, return_sequences=True))([L_input_drop,L_mask]) # L_att = Attention02()(L_lstm,mask=K.squeeze(L_mask,axis=-1)) # R_lstm = layers.Bidirectional(layers.GRU(40,return_sequences=True,dropout=0.1))(R_input) R_lstm = OurBidirectional(GRU(64, return_sequences=True))([R_input_drop,R_mask]) concat = layers.merge([L_lstm,R_lstm], mode='concat',concat_axis=1) concat_mask = layers.merge([L_mask,R_mask], mode='concat',concat_axis=1) att = Attention02()(concat,mask=K.squeeze(concat_mask,axis=-1)) # R_att = Attention02()(R_lstm,mask=K.squeeze(R_mask,axis=-1)) # concat = layers.merge([L_att, R_att], mode='concat') att = Dropout(0.2)(att) output = layers.Dense(output_shape[0],activation="softmax")(att) model = models.Model(inputs=[L_input,R_input], outputs=output) learn_rate = 0.0001 model.compile(optimizer=optimizers.Adam(lr=learn_rate), loss=losses.binary_crossentropy, metrics=[precision,recall,f1_score]) model.summary() return model class Attention(Layer): """多头注意力机制 """ def __init__(self, nb_head, size_per_head, **kwargs): self.nb_head = nb_head self.size_per_head = size_per_head self.out_dim = nb_head * size_per_head super(Attention, self).__init__(**kwargs) def build(self, input_shape): super(Attention, self).build(input_shape) q_in_dim = input_shape[0][-1] k_in_dim = input_shape[1][-1] v_in_dim = input_shape[2][-1] self.q_kernel = self.add_weight(name='q_kernel', shape=(q_in_dim, self.out_dim), initializer='glorot_normal') self.k_kernel = self.add_weight(name='k_kernel', shape=(k_in_dim, self.out_dim), initializer='glorot_normal') self.v_kernel = self.add_weight(name='w_kernel', shape=(v_in_dim, self.out_dim), initializer='glorot_normal') def mask(self, x, mask, mode='mul'): if mask is None: return x else: for _ in range(K.ndim(x) - K.ndim(mask)): mask = K.expand_dims(mask, K.ndim(mask)) if mode == 'mul': return x * mask else: return x - (1 - mask) * 1e10 def call(self, inputs): q, k, v = inputs[:3] v_mask, q_mask = None, None if len(inputs) > 3: v_mask = inputs[3] if len(inputs) > 4: q_mask = inputs[4] # 线性变换 qw = K.dot(q, self.q_kernel) kw = K.dot(k, self.k_kernel) vw = K.dot(v, self.v_kernel) # 形状变换 qw = K.reshape(qw, (-1, K.shape(qw)[1], self.nb_head, self.size_per_head)) kw = K.reshape(kw, (-1, K.shape(kw)[1], self.nb_head, self.size_per_head)) vw = K.reshape(vw, (-1, K.shape(vw)[1], self.nb_head, self.size_per_head)) # 维度置换 qw = K.permute_dimensions(qw, (0, 2, 1, 3)) kw = K.permute_dimensions(kw, (0, 2, 1, 3)) vw = K.permute_dimensions(vw, (0, 2, 1, 3)) # Attention a = K.batch_dot(qw, kw, [3, 3]) / self.size_per_head**0.5 a = K.permute_dimensions(a, (0, 3, 2, 1)) a = self.mask(a, v_mask, 'add') a = K.permute_dimensions(a, (0, 3, 2, 1)) a = K.softmax(a) # 完成输出 o = K.batch_dot(a, vw, [3, 2]) o = K.permute_dimensions(o, (0, 2, 1, 3)) o = K.reshape(o, (-1, K.shape(o)[1], self.out_dim)) o = self.mask(o, q_mask, 'mul') return o def compute_output_shape(self, input_shape): return (input_shape[0][0], input_shape[0][1], self.out_dim) class Attention02(Layer): def __init__(self, **kwargs): self.init = initializers.get('normal') self.supports_masking = True self.attention_dim = 50 super(Attention02, self).__init__(**kwargs) def build(self, input_shape): assert len(input_shape) == 3 self.W = K.variable(self.init((input_shape[-1], 1))) self.b = K.variable(self.init((self.attention_dim,))) self.u = K.variable(self.init((self.attention_dim, 1))) self.trainable_weights = [self.W, self.b, self.u] super(Attention02, self).build(input_shape) def compute_mask(self, inputs, mask=None): return mask def call(self, x, mask=None): uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b)) ait = K.dot(uit, self.u) ait = K.squeeze(ait, -1) ait = K.exp(ait) if mask is not None: ait = ait * K.cast(mask, K.floatx()) # ait = ait * mask ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx()) ait = K.expand_dims(ait) weighted_input = x * ait output = K.sum(weighted_input, axis=1) return output def compute_output_shape(self, input_shape): return (input_shape[0], input_shape[-1]) class OurLayer(Layer): """定义新的Layer，增加reuse方法，允许在定义Layer时调用现成的层 """ def reuse(self, layer, *args, **kwargs): if not layer.built: if len(args) > 0: inputs = args[0] else: inputs = kwargs['inputs'] if isinstance(inputs, list): input_shape = [K.int_shape(x) for x in inputs] else: input_shape = K.int_shape(inputs) layer.build(input_shape) outputs = layer.call(*args, **kwargs) for w in layer.trainable_weights: if w not in self._trainable_weights: self._trainable_weights.append(w) for w in layer.non_trainable_weights: if w not in self._non_trainable_weights: self._non_trainable_weights.append(w) for u in layer.updates: if not hasattr(self, '_updates'): self._updates = [] if u not in self._updates: self._updates.append(u) return outputs class OurBidirectional(OurLayer): """自己封装双向RNN，允许传入mask，保证对齐 """ def __init__(self, layer, **args): super(OurBidirectional, self).__init__(**args) self.forward_layer = layer.__class__.from_config(layer.get_config()) self.backward_layer = layer.__class__.from_config(layer.get_config()) self.forward_layer.name = 'forward_' + self.forward_layer.name self.backward_layer.name = 'backward_' + self.backward_layer.name def reverse_sequence(self, x, mask): """这里的mask.shape是[batch_size, seq_len, 1] """ seq_len = K.round(K.sum(mask, 1)[:, 0]) seq_len = K.cast(seq_len, 'int32') return tf.reverse_sequence(x, seq_len, seq_dim=1) def call(self, inputs): x, mask = inputs x_forward = self.reuse(self.forward_layer, x) x_backward = self.reverse_sequence(x, mask) x_backward = self.reuse(self.backward_layer, x_backward) x_backward = self.reverse_sequence(x_backward, mask) x = K.concatenate([x_forward, x_backward], -1) if K.ndim(x) == 3: return x * mask else: return x def compute_output_shape(self, input_shape): return input_shape[0][:-1] + (self.forward_layer.units * 2,) def training(): data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\newdata_30_prc.csv", index_col=0) data_load = data_load.reset_index(drop=True) test_data = data_load.sample(frac=0.2, random_state=8) train_data = data_load.drop(test_data.index, axis=0) train_data =train_data.reset_index(drop=True) train_x = [] train_y = [] for left, right, label in zip(train_data['context_left'], train_data['context_right'], train_data['re_label']): y = np.zeros(output_shape) y[label] = 1 left = str(left) right = str(right) if left=='nan': left = '' if right=='nan': right = '' left = list(left) right = list(right) context = [left, right] x = embedding_word(context, shape=input_shape) train_x.append(x) train_y.append(y) test_x = [] test_y = [] for left, right, label in zip(test_data['context_left'], test_data['context_right'], test_data['re_label']): y = np.zeros(output_shape) y[label] = 1 left = str(left) right = str(right) if left == 'nan': left = '' if right == 'nan': right = '' left = list(left) right = list(right) context = [left, right] x = embedding_word(context, shape=input_shape) test_x.append(x) test_y.append(y) train_y, test_y = (np.array(train_y), np.array(test_y)) train_x, test_x = (np.array(train_x), np.array(test_x)) train_x, test_x = (np.transpose(train_x, (1, 0, 2, 3)), np.transpose(test_x, (1, 0, 2, 3))) model = getModel() epochs = 150 batch_size = 256 checkpoint = ModelCheckpoint("model_label_time_classify.model.hdf5", monitor="val_loss", verbose=1, save_best_only=True, mode='min') # cw = class_weight.compute_class_weight('auto',np.unique(np.argmax(train_y,axis=1)),np.argmax(train_y,axis=1)) # cw = dict(enumerate(cw)) history = model.fit( x=[train_x[0], train_x[1]], y=train_y, validation_data=([test_x[0], test_x[1]], test_y), epochs=epochs, batch_size=batch_size, shuffle=True, callbacks=[checkpoint], class_weight='auto' ) # plot_loss(history=history) load_model = models.load_model("model_label_time_classify.model.hdf5", custom_objects={'precision': precision, 'recall': recall, 'f1_score': f1_score}) y_pre = load_model.predict([test_x[0], test_x[1]]) # y_pre = load_model.predict(test_x[0]) # 各类别预测评估 res1 = classification_report(np.argmax(test_y, axis=1), np.argmax(y_pre, axis=1)) print(res1) y_pre2 = load_model.predict([train_x[0], train_x[1]]) # y_pre2 = load_model.predict(train_x[0]) res2 = classification_report(np.argmax(train_y, axis=1), np.argmax(y_pre2, axis=1)) print(res2) def train2(): data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\tokens_data.csv", index_col=0) data_load = data_load.reset_index(drop=True) data_load['context_left'] = [left[2:-2].split("', '") for left in data_load['context_left']] data_load['context_right'] = [right[2:-2].split("', '") for right in data_load['context_right']] test_data = data_load.sample(frac=0.2, random_state=8) train_data = data_load.drop(test_data.index, axis=0) train_data =train_data.reset_index(drop=True) train_x = [] train_y = [] for left, right, label in zip(train_data['context_left'], train_data['context_right'], train_data['label']): y = np.zeros(output_shape) y[label] = 1 context = [left, right] x = embedding(context, shape=input_shape2) train_x.append(x) train_y.append(y) test_x = [] test_y = [] for left, right, label in zip(test_data['context_left'], test_data['context_right'], test_data['label']): y = np.zeros(output_shape) y[label] = 1 context = [left, right] x = embedding(context, shape=input_shape2) test_x.append(x) test_y.append(y) train_y, test_y = (np.array(train_y), np.array(test_y)) train_x, test_x = (np.array(train_x), np.array(test_x)) train_x, test_x = (np.transpose(train_x, (1, 0, 2, 3)), np.transpose(test_x, (1, 0, 2, 3))) model = getModel() epochs = 150 batch_size = 256 checkpoint = ModelCheckpoint("model_label_time_classify.model.hdf5", monitor="val_loss", verbose=1, save_best_only=True, mode='min') # cw = class_weight.compute_class_weight('auto',np.unique(np.argmax(train_y,axis=1)),np.argmax(train_y,axis=1)) # cw = dict(enumerate(cw)) history = model.fit( x=[train_x[0], train_x[1]], y=train_y, validation_data=([test_x[0], test_x[1]], test_y), epochs=epochs, batch_size=batch_size, shuffle=True, callbacks=[checkpoint], class_weight='auto' ) # plot_loss(history=history) load_model = models.load_model("model_label_time_classify.model.hdf5", custom_objects={'precision': precision, 'recall': recall, 'f1_score': f1_score}) y_pre = load_model.predict([test_x[0], test_x[1]]) # y_pre = load_model.predict(test_x[0]) # 各类别预测评估 res1 = classification_report(np.argmax(test_y, axis=1), np.argmax(y_pre, axis=1)) print(res1) y_pre2 = load_model.predict([train_x[0], train_x[1]]) # y_pre2 = load_model.predict(train_x[0]) res2 = classification_report(np.argmax(train_y, axis=1), np.argmax(y_pre2, axis=1)) print(res2) def train3(): # data_load = pd.read_excel("tokens_tolabel_data1.xlsx", index_col=0) data_load = pd.read_excel("tokens_tolabel_data1_res12.xlsx", index_col=0) # data_load = pd.concat([data_load[data_load['re_label']==0],data_load]) # data_load = data_load[data_load['pre_label_prob']>0.97] # data_load = data_load[data_load['is_same']==1] data_zero = pd.read_excel("tokens_label0_data1.xlsx") # data_old = pd.read_excel("tokens_data_02.xlsx") data_old = pd.read_excel("tokens_data_02_res6.xlsx") data_zero = data_zero[(data_zero['label']!=0)|(data_zero['is_same']==2)] # data_zero = pd.concat([data_zero,data_zero]) # data_zero = pd.concat([data_zero[(data_zero['label']!=0)|(data_zero['is_same']==2)],data_zero.sample(n=3000)]) # data_zero = data_zero.sample(n=80000) print("输入shape：",input_shape2) data_x = [] data_y = [] for left, right, label,_label in zip(data_load['context_left'], data_load['context_right'], data_load['re_label'], data_load['label']): if label==_label: y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) data_load2 = data_load[data_load['re_label']==0] for left, right, label,_label in zip(data_load2['context_left'], data_load2['context_right'], data_load2['re_label'], data_load2['label']): if label==_label: y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] if len(left)>30: left = left[2:] elif len(left)>15: left = left[1:] right = eval(right) right = right[:40] if len(right)>15: right = right[:-1] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) for left, right, label in zip(data_zero['context_left'], data_zero['context_right'], data_zero['label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) for left, right, label in zip(data_zero['context_left'], data_zero['context_right'], data_zero['label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] if len(left) > 30: left = left[2:] elif len(left) > 15: left = left[1:] right = eval(right) right = right[:40] if len(right) > 15: right = right[:-1] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) # for left, right, label in zip(data_old['context_left'], data_old['context_right'], data_old['label']): # y = np.zeros(output_shape) # y[label] = 1 # left = eval(left) # left = left[-40:] # right = eval(right) # right = right[:40] # context = [left, right] # # x = embedding(context, shape=input_shape2) # data_x.append(context) # data_y.append(y) _data = [d for d in zip(data_x,data_y)] import random random.shuffle(_data) data_x = [i[0] for i in _data] data_y = [i[1] for i in _data] test_len = int(len(data_x) * 0.13) test_x = data_x[:test_len] test_y = data_y[:test_len] print("测试数据量：", len(test_x)) train_x = data_x[test_len:] train_y = data_y[test_len:] for left, right, label in zip(data_old['context_left'], data_old['context_right'], data_old['label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) train_x.append(context) train_y.append(y) print("训练数据量：", len(train_x)) # train_y, test_y = np.array(train_y), np.array(test_y) # train_x = np.array(train_x) # test_x = np.array(test_x) # test_x = np.transpose(test_x, (1, 0, 2, 3)) # train_x, test_x = (np.transpose(train_x, (1, 0, 2, 3)), np.transpose(test_x, (1, 0, 2, 3))) training_generator = DataGenerator(train_x, train_y) # training_generator = DataGenerator(data_x, data_y) validation_generator = DataGenerator(test_x, test_y) # model = getModel3() model = getModel2() epochs = 100 # batch_size = 256 checkpoint = ModelCheckpoint("model_time_classify.weights",save_weights_only=True, monitor="val_loss", verbose=1, save_best_only=True, mode='min') # checkpoint = ModelCheckpoint("model_time_classify2.weights",save_weights_only=True, monitor="loss", verbose=1, # save_best_only=True, mode='min') history = model.fit_generator( generator=training_generator, validation_data=validation_generator, use_multiprocessing=True, workers=2, epochs=epochs, shuffle=True, callbacks=[checkpoint], class_weight='auto' ) # plot_loss(history=history) # load_model = models.load_model("model_label_time_classify.model.hdf5", # custom_objects={'precision': precision, 'recall': recall, 'f1_score': f1_score}) # y_pre = load_model.predict([test_x[0], test_x[1]]) # # y_pre = load_model.predict(test_x[0]) # # 各类别预测评估 # res1 = classification_report(np.argmax(test_y, axis=1), np.argmax(y_pre, axis=1)) # print(res1) # y_pre2 = load_model.predict([train_x[0], train_x[1]]) # # y_pre2 = load_model.predict(train_x[0]) # res2 = classification_report(np.argmax(train_y, axis=1), np.argmax(y_pre2, axis=1)) # print(res2) def train4(): # data_load = pd.read_excel("tokens_tolabel_data1.xlsx", index_col=0) data_load = pd.read_excel("tokens_tolabel_data1_res13New.xlsx", index_col=0) # data_load = pd.concat([data_load[data_load['re_label']==0],data_load]) # data_load = data_load[data_load['pre_label_prob']>0.97] # data_load = data_load[data_load['is_same']==1] data_zero = pd.read_excel("time_entity5.xlsx") data_zero = data_zero[(data_zero['viewed']==1)|(data_zero['is_same']==2)] # data_old = pd.read_excel("tokens_data_02.xlsx") data_old = pd.read_excel("tokens_data_02_res7New.xlsx") data_delay1 = pd.read_excel("delayTime_entity1.xlsx") data_delay1 = data_delay1[data_delay1['label']!=0] data_delay2 = pd.read_excel("delayTime_entity2.xlsx") # data_zero = pd.concat([data_zero,data_zero]) # data_zero = pd.concat([data_zero[(data_zero['label']!=0)|(data_zero['is_same']==2)],data_zero.sample(n=3000)]) # data_zero = data_zero.sample(n=80000) print("输入shape：",input_shape2) data_x = [] data_y = [] import random for left, right, label,_label in zip(data_load['context_left'], data_load['context_right'], data_load['re_label'], data_load['label']): # if label==_label: y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) # data_load2 = data_load[data_load['re_label']==0] # for left, right, label,_label in zip(data_load2['context_left'], data_load2['context_right'], data_load2['re_label'], data_load2['label']): # if label==_label: # y = np.zeros(output_shape) # y[label] = 1 # left = eval(left) # left = left[-40:] # if len(left)>30: # left = left[2:] # elif len(left)>15: # left = left[1:] # right = eval(right) # right = right[:40] # if len(right)>15: # right = right[:-1] # context = [left, right] # # x = embedding(context, shape=input_shape2) # data_x.append(context) # data_y.append(y) for left, right, label in zip(data_zero['context_left'], data_zero['context_right'], data_zero['re_label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) for left, right, label in zip(data_delay1['context_left'], data_delay1['context_right'], data_delay1['label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) for left, right, label in zip(data_delay2['context_left'], data_delay2['context_right'], data_delay2['re_label']): y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) # for left, right, label in zip(data_zero['context_left'], data_zero['context_right'], data_zero['label']): # y = np.zeros(output_shape) # y[label] = 1 # left = eval(left) # left = left[-40:] # if len(left) > 30: # left = left[2:] # elif len(left) > 15: # left = left[1:] # right = eval(right) # right = right[:40] # if len(right) > 15: # right = right[:-1] # context = [left, right] # # x = embedding(context, shape=input_shape2) # data_x.append(context) # data_y.append(y) # for left, right, label in zip(data_old['context_left'], data_old['context_right'], data_old['label']): # y = np.zeros(output_shape) # y[label] = 1 # left = eval(left) # left = left[-40:] # right = eval(right) # right = right[:40] # context = [left, right] # # x = embedding(context, shape=input_shape2) # data_x.append(context) # data_y.append(y) for left, right, label,pre_label,is_same in zip(data_old['context_left'], data_old['context_right'], data_old['label'], data_old['pre_label'],data_old['is_same']): if label==0: if is_same==1: pass else: if pre_label>3: label = pre_label else: continue y = np.zeros(output_shape) y[label] = 1 left = eval(left) left = left[-40:] right = eval(right) right = right[:40] context = [left, right] # x = embedding(context, shape=input_shape2) data_x.append(context) data_y.append(y) _data = [d for d in zip(data_x,data_y)] random.shuffle(_data) data_x = [i[0] for i in _data] data_y = [i[1] for i in _data] test_len = int(len(data_x) * 0.11) test_x = data_x[:test_len] test_y = data_y[:test_len] print("测试数据量：", len(test_x)) train_x = data_x[test_len:] train_y = data_y[test_len:] # for left, right, label,pre_label,is_same in zip(data_old['context_left'], data_old['context_right'], data_old['label'], # data_old['pre_label'],data_old['is_same']): # # if label==0: # # if random.random()>0.25: # # continue # if label==0: # if is_same==1: # pass # else: # if pre_label>3: # label = pre_label # else: # continue # y = np.zeros(output_shape) # y[label] = 1 # left = eval(left) # left = left[-40:] # right = eval(right) # right = right[:40] # context = [left, right] # # x = embedding(context, shape=input_shape2) # train_x.append(context) # train_y.append(y) print("训练数据量：", len(train_x)) # train_y, test_y = np.array(train_y), np.array(test_y) # train_x = np.array(train_x) # test_x = np.array(test_x) # test_x = np.transpose(test_x, (1, 0, 2, 3)) # train_x, test_x = (np.transpose(train_x, (1, 0, 2, 3)), np.transpose(test_x, (1, 0, 2, 3))) training_generator = DataGenerator(train_x, train_y,is_train=True) # training_generator = DataGenerator(data_x, data_y) validation_generator = DataGenerator(test_x, test_y,is_train=False,shuffle=False) # model = getModel3() model = getModel2() epochs = 100 # batch_size = 256 checkpoint = ModelCheckpoint("model_time_classify.weights",save_weights_only=True, monitor="val_loss", verbose=1, save_best_only=True, mode='min') # checkpoint = ModelCheckpoint("model_time_classify2.weights",save_weights_only=True, monitor="loss", verbose=1, # save_best_only=True, mode='min') history = model.fit_generator( generator=training_generator, validation_data=validation_generator, use_multiprocessing=True, workers=2, epochs=epochs, shuffle=True, callbacks=[checkpoint], class_weight='auto' ) from keras.utils import Sequence,to_categorical class DataGenerator(Sequence): 'Generates data for Keras' def __init__(self, texts, labels, is_train=True,batch_size=256, n_classes=len(time_label_dict), shuffle=True): 'Initialization' # self.dim = dim self.batch_size = batch_size self.labels = labels self.texts = texts self.n_classes = n_classes self.shuffle = shuffle self.is_train = is_train self.on_epoch_end() def __len__(self): 'Denotes the number of batches per epoch' _len = len(self.texts) // self.batch_size if len(self.texts) % self.batch_size != 0: _len += 1 return _len def __getitem__(self, index): 'Generate one batch of data' # Generate indexes of the batch indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size] # Find list of IDs list_texts = [self.texts[k] for k in indexes] _label = [self.labels[k] for k in indexes] # Generate data X, y = self.__data_generation(list_texts,_label) return X, y def on_epoch_end(self): 'Updates indexes after each epoch' self.indexes = np.arange(len(self.texts)) if self.shuffle == True: np.random.shuffle(self.indexes) def __data_generation(self, list_texts,_label): 'Generates data containing batch_size samples' # Initialization # X = np.empty((self.batch_size, *self.dim)) # y = np.empty((self.batch_size), dtype=int) # batch_len = len(list_texts) # x = np.empty((batch_len, *self.dim)) x = [] # y = np.empty((batch_len), dtype=int) # Generate data for i, context in enumerate(list_texts): # Store sample if self.is_train: left = context[0] if len(left) > 30: if random.random() > 0.5: left = left[2:] elif len(left) > 15: if random.random() > 0.5: left = left[1:] right = context[1] if len(right) > 30: if random.random() > 0.5: right = right[:-2] elif len(right) > 15: if random.random() > 0.5: right = right[:-1] context = [left, right] words_matrix = embedding_mywords(context, shape=input_shape2) # Store class # y[i] = _label[i] x.append(words_matrix) x = np.array(x) x = np.transpose(x, (1, 0, 2, 3)) return [x[0],x[1]], np.array(_label) def predict2(): model1 = models.load_model("model_label_time_classify.model.hdf5",custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score}) data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\tokens_data.csv", index_col=0) data_load['context_left'] = [left[2:-2].split("', '") for left in data_load['context_left']] data_load['context_right'] = [right[2:-2].split("', '") for right in data_load['context_right']] test_x = [] test_y = [] for left, right, label in zip(data_load['context_left'], data_load['context_right'], data_load['label']): y = np.zeros(output_shape) y[label] = 1 context = [left, right] x = embedding(context, shape=input_shape2) test_x.append(x) test_y.append(y) test_x = np.transpose(np.array(test_x), (1, 0, 2, 3)) pre_y = model1.predict([test_x[0],test_x[1]]) data_load['pre'] = [np.argmax(item) for item in pre_y] error_data = data_load[data_load['label']!=data_load['pre']] # print(error_data.info()) error_data.to_csv("C:\\Users\\admin\\Desktop\\error4-30.csv") def predict3(): data = pd.read_csv("new_tokens_data1.csv", chunksize=5000) model1 = models.load_model("model_label_time_classify.model.hdf5",custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score}) new_data = pd.DataFrame() idx = 0 for _data in data: test_x = [] test_y = [] for left, right, label in zip(_data['context_left'], _data['context_right'], _data['label']): left = eval(left) left = left[-10:] right = eval(right) right = right[:10] label = int(label) y = np.zeros(output_shape) y[label] = 1 context = [left, right] x = embedding(context, shape=input_shape2) test_x.append(x) test_y.append(y) test_x = np.transpose(np.array(test_x), (1, 0, 2, 3)) pre_y = model1.predict([test_x[0], test_x[1]]) _data['pre'] = [np.argmax(item) for item in pre_y] _data['is_same'] = [1 if int(_label)==_pre else 0 for _label,_pre in zip(_data['label'],_data['pre'])] # data['label'] = label new_data = pd.concat([new_data, _data]) idx += 5000 print(idx) # data.to_csv("new_tokens_data1.csv") new_data.to_excel("new_tokens_data1_res.xlsx") def predict4(): data = pd.read_csv("tokens_data_02_res6New.csv", chunksize=3000) # data = pd.read_excel("C:\\Users\\Administrator\\Desktop\\time_entity4.xlsx") # data.to_csv("C:\\Users\\Administrator\\Desktop\\time_entity4.csv") # data = pd.read_csv("C:\\Users\\Administrator\\Desktop\\time_entity4.csv", chunksize=3000) model1 = getModel2() model1.load_weights("model_time_classify.weights") new_data = pd.DataFrame() idx = 0 for _data in data: test_x = [] test_y = [] for left, right, label in zip(_data['context_left'], _data['context_right'], _data['re_label']): left = eval(left) left = left[-40:] right = eval(right) right = right[:40] label = int(label) y = np.zeros(output_shape) y[label] = 1 context = [left, right] x = embedding_mywords(context, shape=input_shape2) test_x.append(x) test_y.append(y) test_x = np.transpose(np.array(test_x), (1, 0, 2, 3)) pre_y = model1.predict([test_x[0], test_x[1]]) _data['pre_label'] = [np.argmax(item) for item in pre_y] _data['pre_label_prob'] = [max(item) for item in pre_y] _data['is_same'] = [1 if int(_label)==_pre else 0 for _label,_pre in zip(_data['label'],_data['pre_label'])] # _data['is_same'] = [1 if int(_re)==int(_pre) and int(_re)==int(_label) else 0 for _label,_re,_pre in zip(_data['label'],_data['re_label'],_data['pre_label'])] # data['label'] = label new_data = pd.concat([new_data, _data]) idx += 3000 print(idx) # new_data.to_csv("tokens_data_02_res7New.csv") new_data.to_excel("tokens_data_02_res7New.xlsx") # new_data.to_excel("C:\\Users\\Administrator\\Desktop\\tokens_data_02_res7New.xlsx") def predict(): model1 = models.load_model("model_label_time_classify.model.hdf5",custom_objects={'precision':precision,'recall':recall,'f1_score':f1_score}) data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\newdata_30_prc.csv", index_col=0) test_x = [] test_y = [] for left, right, label in zip(data_load['context_left'], data_load['context_right'], data_load['re_label']): y = np.zeros(output_shape) y[label] = 1 left = str(left) right = str(right) if left == 'nan': left = '' if right == 'nan': right = '' left = list(left) right = list(right) context = [left, right] x = embedding_word(context, shape=input_shape) test_x.append(x) test_y.append(y) test_x = np.transpose(np.array(test_x), (1, 0, 2, 3)) pre_y = model1.predict([test_x[0],test_x[1]]) data_load['pre'] = [np.argmax(item) for item in pre_y] error_data = data_load[data_load['re_label']!=data_load['pre']] # print(error_data.info()) error_data.to_csv("C:\\Users\\admin\\Desktop\\error4-30.csv") def data_process(): data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\newdata_30.csv", index_col=0) re_left = re.compile("。[^。]*?$") re_right = re.compile("^[^。]*?。") left_list = [] right_list = [] for left, right in zip(data_load['context_left'], data_load['context_right']): left = str(left) right = str(right) if right=='nan': right = '' # print(1) if re.search("。",left): left = re_left.search(left) left = left.group()[1:] if re.search("。",right): right = re_right.search(right) right = right.group() left_list.append(left) right_list.append(right) data_load['context_left'] = left_list data_load['context_right'] = right_list data_load.to_csv("C:\\Users\\admin\\Desktop\\newdata_30_prc.csv") def data_process2(): data_load = pd.read_csv("C:\\Users\\admin\\Desktop\\newdata_30_prc.csv", index_col=0) left_list = [] right_list = [] for left, right in zip(data_load['context_left'], data_load['context_right']): left = str(left) right = str(right) if right=='nan': right = '' if left=='nan': left = '' left = left[max(len(left)-20,0):] right = right[:20] left_list.append(left) right_list.append(right) data_load['context_left'] = left_list data_load['context_right'] = right_list data_load.to_csv("C:\\Users\\admin\\Desktop\\newdata_20_prc.csv") def data_process3(): data = load('db_time_data.pk') data = data.drop('value', axis=1) token_begin = [] token_end = [] context_left = [] context_right = [] data2 = pd.read_csv("newdata_30_prc2.csv") label = [] # data=data[:20] for id,sentences,tokens,offset,begin,end,entity_text in zip(data['document_id'],data['sentences'],data['tokens'],data['offsets_to_text'], data['begin_index'],data['end_index'],data['entity_text']): _label = data2[(data2['document_id']==int(id)) & (data2['begin_index']==int(begin))][:1] if not _label.empty: _label = int(_label['re_label']) else: _label=0 label.append(_label) begin = int(begin) end = int(end) entity_tbegin = 0 entity_tend = 0 find_begin = False for t in range(len(offset)): if not find_begin: if offset[t]==begin: entity_tbegin = t find_begin = True if offset[t]>begin: entity_tbegin = t-1 find_begin = True if offset[t] >= end: entity_tend = t break token_begin.append(entity_tbegin) token_end.append(entity_tend) s = spanWindow(tokens=tokens,begin_index=entity_tbegin,end_index=entity_tend-1,size=40) s1 = s[0] _temp1 = [] for i in range(len(s1)): if s1[i]=="。": _temp1.append(i) if _temp1: s1 = s1[_temp1[-1]+1:] s2 = s[1] _temp2 = [] for i in range(len(s2)): if s2[i] == "。": _temp2.append(i) break if _temp2: s2 = s2[:_temp2[0]+1] # print(s2) context_left.append(s1) context_right.append(s2) print(id) # print(_label) # print(entity_text) # print(tokens[entity_tbegin:entity_tend]) data['token_begin'] = token_begin data['token_end'] = token_end data['context_left'] = context_left data['context_right'] = context_right data['label'] = label data = data.drop(['tokens','offsets_to_text','sentences'],axis=1) # data.to_csv("tokens_data_02.csv") data.to_excel("tokens_data_02.xlsx") def plot_loss(history): plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('Model loss') plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Train', 'Test'], loc='upper left') plt.show() def embedding_mywords(datas,shape): ''' @summary:查找词汇对应的词向量 @param: datas:词汇的list shape:结果的shape @return: array,返回对应shape的词嵌入 ''' model_w2v = getModel_w2v() embed = np.zeros(shape) length = shape[1] out_index = 0 #print(datas) for data in datas: index = 0 for item in data: item_not_space = re.sub("\s*","",item) if index>=length: break if item_not_space in model_w2v.vocab: embed[out_index][index] = model_w2v[item_not_space] index += 1 else: embed[out_index][index] = model_w2v['unk'] index += 1 out_index += 1 return embed def save_model(): graph = tf.Graph() with graph.as_default() as graph: with tf.Session(graph=graph).as_default() as sess: test_model = getModel2() test_model.load_weights("model_time_classify.weights") tf.saved_model.simple_save(sess, "models/timesplit_model2/", inputs={"input0": test_model.input[0], "input1":test_model.input[1] }, outputs={"outputs": test_model.output}) if __name__ == '__main__': # get_data() # getModel() # getModel2() # getModel3() # training() # train2() # train3() # train4() # data_process() # data_process2() # data_process3() # predict() # predict2() # predict3() # predict4() save_model() pass