''' Created on 2019年12月31日 @author: User ''' # from tensorflow.contrib import rnn # from tensorflow.contrib.crf import crf_log_likelihood # from tensorflow.contrib.layers.python.layers import initializers from zipfile import ZipFile from BiddingKG.dl_dev.BertNer.BertModel import * from BiddingKG.dl.common.Utils import * def _load_map_file(path, char_map_name, id_map_name): with ZipFile(path) as myzip: with myzip.open('all_map.json') as myfile: content = myfile.readline() content = content.decode() data = json.loads(content) return data.get(char_map_name), data.get(id_map_name) def shape_list(x): """Return list of dims, statically where possible.""" x = tf.convert_to_tensor(x) # If unknown rank, return dynamic shape if x.get_shape().dims is None: return tf.shape(x) static = x.get_shape().as_list() shape = tf.shape(x) ret = [] for i in range(len(static)): dim = static[i] if dim is None: dim = shape[i] ret.append(dim) return ret class BertCRF(object): def __init__(self): config = {'lstm_dim':100, 'num_chars':6591, 'num_tags':25, 'char_dim':100, 'lr':0.00002, 'input_dropout_keep':1.0, 'optimizer':'adam', 'clip':5, 'bert_hidden':100} self.config = config self.lstm_dim = config["lstm_dim"] self.num_chars = config["num_chars"] self.num_tags = config["num_tags"] self.char_dim = config["char_dim"] self.lr = config["lr"] self.bert_hidden = config["bert_hidden"] self.graph = tf.Graph() with self.graph.as_default(): self.char_to_id, self.id_to_seg = _load_map_file(os.path.dirname(__file__)+"/data/map.zip", "char_map", "ner_map") self.id_to_tag = {int(k):v for k,v in self.id_to_seg.items()} self.tag_to_id = {v:int(k) for k,v in self.id_to_seg.items()} #self.char_embeding = tf.get_variable(name="char_embeding", initializer=embeddings) self.char_embeding = tf.get_variable(name="char_embeding",shape=(self.num_chars,self.char_dim)) #添加一串全0的坑,fool发行版和源代码不一样 self.const = tf.constant(value=0,dtype=tf.float32,shape=[1,100]) self.char_embeding = tf.concat([self.const,self.char_embeding],0) self.global_step = tf.Variable(0, trainable=False) self.initializer = initializers.xavier_initializer() self.char_inputs = tf.placeholder(dtype=tf.int32, shape=[None, None], name="char_inputs") self.targets = tf.placeholder(dtype=tf.int32, shape=[None, None], name="targets") self.dropout = tf.placeholder(dtype=tf.float32, name="dropout") self.lengths = tf.placeholder(dtype=tf.int32, shape=[None, ], name="lengths") # self.middle_dropout_keep_prob = tf.placeholder_with_default(1.0, [], name="middle_dropout_keep_prob") # self.hidden_dropout_keep_prob = tf.placeholder_with_default(1.0, [], name="hidden_dropout_keep_prob") self.input_dropout_keep_prob = tf.placeholder_with_default(config["input_dropout_keep"], [], name="input_dropout_keep_prob") self.batch_size = tf.shape(self.char_inputs)[0] self.num_steps = tf.shape(self.char_inputs)[-1] # forward embedding = self.embedding_layer(self.char_inputs) bert_outputs = self.bert_layer(embedding) #lstm_inputs = tf.nn.dropout(bert_outputs, self.input_dropout_keep_prob) ## bi-directional lstm layer #lstm_outputs = self.bilstm_layer(lstm_inputs) ## logits for tags self.project_layer(bert_outputs) ## loss of the model self.loss = self.loss_layer(self.logits, self.lengths) with tf.variable_scope("optimizer"): optimizer = self.config["optimizer"] if optimizer == "sgd": self.opt = tf.train.GradientDescentOptimizer(self.lr) elif optimizer == "adam": self.opt = tf.train.AdamOptimizer(self.lr) elif optimizer == "adgrad": self.opt = tf.train.AdagradOptimizer(self.lr) else: raise KeyError grads_vars = self.opt.compute_gradients(self.loss) capped_grads_vars = [[tf.clip_by_value(g, -self.config["clip"], self.config["clip"]), v] for g, v in grads_vars] self.train_op = self.opt.apply_gradients(capped_grads_vars, self.global_step) config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.7 config.gpu_options.allow_growth = True self.sess = tf.Session(graph=self.graph,config=config) def embedding_layer(self, char_inputs): with tf.variable_scope("char_embedding"), tf.device('/cpu:0'): embed = tf.nn.embedding_lookup(self.char_embeding, char_inputs) return embed def bilstm_layer(self, lstm_inputs, name=None): with tf.variable_scope("char_bilstm" if not name else name): lstm_fw_cell = rnn.BasicLSTMCell(self.lstm_dim, state_is_tuple=True) lstm_bw_cell = rnn.BasicLSTMCell(self.lstm_dim, state_is_tuple=True) outputs, _ = tf.nn.bidirectional_dynamic_rnn(lstm_fw_cell, lstm_bw_cell, lstm_inputs, dtype=tf.float32, sequence_length=self.lengths) return tf.concat(outputs, axis=2) def get_timing_signal_1d(self,length, channels, min_timescale=1.0, max_timescale=1.0e4, start_index=0): """Gets a bunch of sinusoids of different frequencies. Each channel of the input Tensor is incremented by a sinusoid of a different frequency and phase. This allows attention to learn to use absolute and relative positions. Timing signals should be added to some precursors of both the query and the memory inputs to attention. The use of relative position is possible because sin(x+y) and cos(x+y) can be expressed in terms of y, sin(x) and cos(x). In particular, we use a geometric sequence of timescales starting with min_timescale and ending with max_timescale. The number of different timescales is equal to channels / 2. For each timescale, we generate the two sinusoidal signals sin(timestep/timescale) and cos(timestep/timescale). All of these sinusoids are concatenated in the channels dimension. Args: length: scalar, length of timing signal sequence. channels: scalar, size of timing embeddings to create. The number of different timescales is equal to channels / 2. min_timescale: a float max_timescale: a float start_index: index of first position Returns: a Tensor of timing signals [1, length, channels] """ position = tf.to_float(tf.range(length) + start_index) num_timescales = channels // 2 log_timescale_increment = ( math.log(float(max_timescale) / float(min_timescale)) / (tf.to_float(num_timescales) - 1)) inv_timescales = min_timescale * tf.exp( tf.to_float(tf.range(num_timescales)) * -log_timescale_increment) scaled_time = tf.expand_dims(position, 1) * tf.expand_dims(inv_timescales, 0) signal = tf.concat([tf.sin(scaled_time), tf.cos(scaled_time)], axis=1) signal = tf.pad(signal, [[0, 0], [0, tf.mod(channels, 2)]]) signal = tf.reshape(signal, [1, length, channels]) return signal def add_timing_signal_1d(self,x, min_timescale=1.0, max_timescale=1.0e4, start_index=0): """Adds a bunch of sinusoids of different frequencies to a Tensor. Each channel of the input Tensor is incremented by a sinusoid of a different frequency and phase. This allows attention to learn to use absolute and relative positions. Timing signals should be added to some precursors of both the query and the memory inputs to attention. The use of relative position is possible because sin(x+y) and cos(x+y) can be experessed in terms of y, sin(x) and cos(x). In particular, we use a geometric sequence of timescales starting with min_timescale and ending with max_timescale. The number of different timescales is equal to channels / 2. For each timescale, we generate the two sinusoidal signals sin(timestep/timescale) and cos(timestep/timescale). All of these sinusoids are concatenated in the channels dimension. Args: x: a Tensor with shape [batch, length, channels] min_timescale: a float max_timescale: a float start_index: index of first position Returns: a Tensor the same shape as x. """ length = shape_list(x)[1] channels = shape_list(x)[2] signal = self.get_timing_signal_1d(length, channels, min_timescale, max_timescale, start_index) return x + signal def bert_layer(self,tensor_embedding): #增加位置向量信息 tensor_after_position = self.add_timing_signal_1d(tensor_embedding) with tf.variable_scope("encoder"): # This converts a 2D mask of shape [batch_size, seq_length] to a 3D # mask of shape [batch_size, seq_length, seq_length] which is used # for the attention scores. attention_mask = tf.tile(tf.expand_dims(tf.cast(tf.sequence_mask(self.lengths,tf.reduce_max(self.lengths)),"int32"),1),[1,tf.reduce_max(self.lengths),1])#create_attention_mask_from_input_mask(input_ids, input_mask) tf.Print(attention_mask,[],"attention_mask") # Run the stacked transformer. # `sequence_output` shape = [batch_size, seq_length, hidden_size]. self.all_encoder_layers = transformer_model( input_tensor=tensor_after_position, attention_mask=attention_mask, hidden_size=self.bert_hidden, num_hidden_layers=4, num_attention_heads=10, intermediate_size=128, intermediate_act_fn=get_activation("gelu"), hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, initializer_range=0.02, do_return_all_layers=False) print(self.all_encoder_layers) return self.all_encoder_layers def project_layer(self, lstm_outputs, name=None): """ """ with tf.variable_scope("project" if not name else name): with tf.variable_scope("hidden"): w_tanh = tf.get_variable("w_tanh", shape=[self.bert_hidden, self.lstm_dim], dtype=tf.float32, initializer=self.initializer, regularizer=tf.contrib.layers.l2_regularizer(0.001)) b_tanh = tf.get_variable("b_tanh", shape=[self.lstm_dim], dtype=tf.float32, initializer=tf.zeros_initializer()) output = tf.reshape(lstm_outputs, shape=[-1, self.bert_hidden]) hidden = tf.tanh(tf.nn.xw_plus_b(output, w_tanh, b_tanh)) drop_hidden = tf.nn.dropout(hidden, self.dropout) # project to score of tags with tf.variable_scope("output"): w_out = tf.get_variable("w_out", shape=[self.bert_hidden, self.num_tags], dtype=tf.float32, initializer=self.initializer, regularizer=tf.contrib.layers.l2_regularizer(0.001)) b_out = tf.get_variable("b_out", shape=[self.num_tags], dtype=tf.float32, initializer=tf.zeros_initializer()) pred = tf.nn.xw_plus_b(drop_hidden, w_out, b_out, name="pred") self.logits = tf.reshape(pred, [-1, self.num_steps, self.num_tags], name="logits") def loss_layer(self, project_logits, lengths, name=None): with tf.variable_scope("crf_loss" if not name else name): small = -1000.0 start_logits = tf.concat( [small * tf.ones(shape=[self.batch_size, 1, self.num_tags]), tf.zeros(shape=[self.batch_size, 1, 1])], axis=-1) pad_logits = tf.cast(small * tf.ones([self.batch_size, self.num_steps, 1]), tf.float32) logits = tf.concat([project_logits, pad_logits], axis=-1) logits = tf.concat([start_logits, logits], axis=1) targets = tf.concat( [tf.cast(self.num_tags * tf.ones([self.batch_size, 1]), tf.int32), self.targets], axis=-1) self.trans = tf.get_variable( "transitions", shape=[self.num_tags + 1, self.num_tags + 1], initializer=self.initializer) log_likelihood, self.trans = crf_log_likelihood( inputs=logits, tag_indices=targets, transition_params=self.trans, sequence_lengths=lengths + 1) return tf.reduce_mean(-log_likelihood) def initVariables(self): dict_tensor_values = load(os.path.dirname(__file__)+"/dict_tensor_values.pk") with self.graph.as_default(): init_op = tf.global_variables_initializer() self.sess.run(init_op) ''' trainable_variables = tf.trainable_variables() for item in trainable_variables: print(item.name,"prefix/"+item.name in dict_tensor_values.keys()) self.sess.run(tf.assign(item,dict_tensor_values["prefix/"+item.name])) print((self.sess.run(item)==dict_tensor_values["prefix/"+item.name]).all()) ''' '''''' for _key in dict_tensor_values.keys(): print("init variable %s"%(_key)) self.sess.run(tf.assign(self.graph.get_tensor_by_name(_key[7:]),dict_tensor_values[_key])) #print(self.sess.run(tf.nn.embedding_lookup(self.char_embeding, np.array([[1]], dtype=np.int32)))) #print(self.sess.run(self.char_embeding)) return self def restore(self,path=None): print("restore weights") with self.graph.as_default(): saver = tf.train.Saver() if path is None: path_add = "0-12/" saver.restore(self.sess, os.path.dirname(__file__)+'/model/'+path_add+'model.ckpt') ''' path_add = "0-4/" saver.restore(self.sess, os.path.dirname(__file__)+'/model-server/'+path_add+'model.ckpt') ''' else: saver.restore(self.sess,path) list_v = [v.name for v in tf.trainable_variables()] print(list_v) list_value = bert.sess.run(list_v) dict_key_value = dict() for k,v in zip(list_v,list_value): if re.search("encoder",k) is not None or re.search("char_embeding",k) is not None: dict_key_value[k] = v print(k,v.shape) save(self.char_to_id, "fool_char_to_id.pk") save(dict_key_value, "dict_key_value.pk") return self def getNodes(self): return self.char_inputs,self.targets,self.lengths,self.dropout,self.logits,self.trans,self.loss,self.train_op def predict(self,sess,sents): inputs = [] lengths = [len(text) for text in sents] max_len = max(lengths) for sent in sents: sent_ids = [self.char_to_id.get(w) if w in self.char_to_id else self.char_to_id.get("") for w in sent] padding = [0] * (max_len - len(sent_ids)) sent_ids += padding inputs.append(sent_ids) inputs = np.array(inputs, dtype=np.int32) if USE_PAI_EAS: request = tf_predict_pb2.PredictRequest() request.inputs["char_inputs"].dtype = tf_predict_pb2.DT_INT32 request.inputs["char_inputs"].array_shape.dim.extend(np.shape(inputs)) request.inputs["char_inputs"].int_val.extend(np.array(inputs,dtype=np.int32).reshape(-1)) request.inputs["lengths"].dtype = tf_predict_pb2.DT_INT32 request.inputs["lengths"].array_shape.dim.extend(np.shape(lengths)) request.inputs["lengths"].int_val.extend(np.array(lengths,dtype=np.int32).reshape(-1)) request.inputs["dropout"].dtype = tf_predict_pb2.DT_FLOAT request.inputs["dropout"].float_val.extend([1.0]) request_data = request.SerializeToString() list_outputs = ["logits","trans"] result = vpc_requests(selffool_url, selffool_authorization, request_data, list_outputs) if result is not None: logits = result["logits"] trans = result["trans"] else: feed_dict = { self.char_inputs: inputs, self.lengths: lengths, self.dropout: 1.0 } logits, trans = sess.run([self.logits, self.trans], feed_dict=feed_dict) else: feed_dict = { self.char_inputs: inputs, self.lengths: lengths, self.dropout: 1.0 } logits, trans = sess.run([self.logits, self.trans], feed_dict=feed_dict) path = decode(logits, trans, lengths, self.num_tags) labels = [[self.id_to_tag.get(l) for l in p] for p in path] return labels def ner(self, text_list): text_list = _check_input(text_list) ner_labels = self.predict(self.sess,text_list) #print(ner_labels) all_entitys = [] for ti, text in enumerate(text_list): ens = [] entity = "" i = 0 ner_label = ner_labels[ti] chars = list(text) for label, word in zip(ner_label, chars): i += 1 if label == "O": continue lt = label.split("_")[1] lb = label.split("_")[0] if lb == "S": ens.append((i, i + 1, lt, word)) elif lb == "B": entity = "" entity += word elif lb == "M": entity += word elif lb == "E": entity += word ens.append((i - len(entity), i + 1, lt, entity)) entity = "" if entity: ens.append((i - len(entity), i + 1, lt, entity)) all_entitys.append(ens) return all_entitys def decode(logits, trans, sequence_lengths, tag_num): viterbi_sequences = [] small = -1000.0 start = np.asarray([[small] * tag_num + [0]]) for logit, length in zip(logits, sequence_lengths): score = logit[:length] pad = small * np.ones([length, 1]) score = np.concatenate([score, pad], axis=1) score = np.concatenate([start, score], axis=0) viterbi_seq, viterbi_score = viterbi_decode(score, trans) viterbi_sequences.append(viterbi_seq[1:]) return viterbi_sequences def _check_input(text, ignore=False): if not text: return [] if not isinstance(text, list): text = [text] null_index = [i for i, t in enumerate(text) if not t] if null_index and not ignore: raise Exception("null text in input ") return text if __name__=="__main__": bert = BertCRF() bert.restore()