|
|
@@ -0,0 +1,796 @@
|
|
|
+import tensorflow as tf
|
|
|
+from tensorflow.contrib.crf import crf_log_likelihood
|
|
|
+from tensorflow.contrib.layers.python.layers import initializers
|
|
|
+import numpy as np
|
|
|
+import pandas as pd
|
|
|
+import os
|
|
|
+import psycopg2
|
|
|
+import re
|
|
|
+import pickle
|
|
|
+from BiddingKG.dl.common.Utils import *
|
|
|
+from keras.preprocessing.sequence import pad_sequences
|
|
|
+
|
|
|
+def get_data():
|
|
|
+ # with open("viewTrain.txt",'r',encoding='utf-8') as f1,open("viewTest.txt",'r',encoding='utf-8') as f2:
|
|
|
+ # rows1 = f1.readlines()
|
|
|
+ # rows2 = f2.readlines()
|
|
|
+ # rows = rows1 + rows2
|
|
|
+ # sentence = []
|
|
|
+ # sentence_label = []
|
|
|
+ # sentences_and_labels = []
|
|
|
+ # for row in rows:
|
|
|
+ # if row[1]!='#':
|
|
|
+ # sentence.append(row[0])
|
|
|
+ # sentence_label.append(row[2:-1])
|
|
|
+ # else:
|
|
|
+ # sentences_and_labels.append((sentence,sentence_label))
|
|
|
+ # sentence = []
|
|
|
+ # sentence_label = []
|
|
|
+ # print(sentences_and_labels)
|
|
|
+ # save(sentences_and_labels,"data/old_datas.pk")
|
|
|
+ conn = psycopg2.connect(dbname="iepy",user="postgres",password="postgres",host="192.168.2.101")
|
|
|
+ user_list = [
|
|
|
+ ["test1","2020-08-01","2020-11-25"],
|
|
|
+ ["test11","2020-08-01","2020-11-25"],
|
|
|
+ ["test12","2020-08-01","2020-11-25"],
|
|
|
+ ["test17","2020-08-01","2020-10-31"],
|
|
|
+ ["test19","2020-08-01","2020-11-25"],
|
|
|
+ ["test2","2020-08-01","2020-11-25"],
|
|
|
+ ["test3","2020-08-01","2020-11-25"],
|
|
|
+ ["test7","2020-08-01","2020-11-25"],
|
|
|
+ ["test8","2020-08-01","2020-11-25"],
|
|
|
+ ["test9","2020-08-01","2020-11-25"],
|
|
|
+ ]
|
|
|
+ db_data = []
|
|
|
+ for u in user_list:
|
|
|
+ cur1 = conn.cursor()
|
|
|
+ sql = "SELECT B.document_id,A.text,A.sentences,B.value " \
|
|
|
+ "FROM corpus_iedocument A,brat_bratannotation B " \
|
|
|
+ "WHERE A.human_identifier = B.document_id " \
|
|
|
+ "AND A.edituser = '%s' " \
|
|
|
+ "AND A.edittime >= '%s':: date " \
|
|
|
+ "AND A.edittime <= '%s':: date "
|
|
|
+ # "ORDER BY B.document_id"
|
|
|
+ cur1.execute(sql % (u[0], u[1], u[2]))
|
|
|
+ db_data.extend(cur1.fetchall())
|
|
|
+ cur1.close()
|
|
|
+ # print(len(db_data))
|
|
|
+ # print(db_data[0])
|
|
|
+ columns = ['document_id','text', 'sentences', 'value']
|
|
|
+ df = pd.DataFrame(db_data, columns=columns)
|
|
|
+ df = df[df['value'].str.contains('^T')]
|
|
|
+ # df = df[df['value'].str.contains('code|name|org|company')]
|
|
|
+ df = df[df['value'].str.contains('code|name')]
|
|
|
+ df = df.reset_index(drop=True)
|
|
|
+ value_split = df['value'].str.split(expand=True)
|
|
|
+ value_split.columns = ['_', 'entity_type', 'begin', 'end', 'entity_text']
|
|
|
+ value_split = value_split.drop('_', axis=1)
|
|
|
+ df = pd.concat([df, value_split], axis=1)
|
|
|
+ df = df.drop('value', axis=1)
|
|
|
+ df['begin'] = [int(_) for _ in df['begin']]
|
|
|
+ df['end'] = [int(_) for _ in df['end']]
|
|
|
+ code_left_list = []
|
|
|
+ for begin,text,entity_type in zip(df['begin'],df['text'],df['entity_type']):
|
|
|
+ code_left = ''
|
|
|
+ if entity_type == 'code':
|
|
|
+ code_left = text[max(0,begin-8):begin]
|
|
|
+ code_left_list.append(code_left)
|
|
|
+ df['code_left'] = code_left_list
|
|
|
+ df.to_excel("C:\\Users\\admin\\Desktop\\项目编号和名称\\Code&Name_dbData.xlsx")
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+def data_process():
|
|
|
+ data = pd.read_excel("C:\\Users\\admin\\Desktop\\项目编号和名称\\Code&Name_dbData.xlsx",index_col=0)
|
|
|
+ data['sentences'] = [sentences[1:-1].split(',') for sentences in data['sentences']]
|
|
|
+ data['sentences'] = [[int(s) for s in sentences] for sentences in data['sentences']]
|
|
|
+ memory_set = set()
|
|
|
+ id_list = []
|
|
|
+ text_list = []
|
|
|
+ text_tagLabels = dict()
|
|
|
+ for _id, _text, _sentences in zip(data['document_id'], data['text'], data['sentences']):
|
|
|
+ if _id not in memory_set:
|
|
|
+ memory_set.add(_id)
|
|
|
+ text_list.append(_text)
|
|
|
+ id_list.append(_id)
|
|
|
+ text_tagLabels[_id] = [[],[]]
|
|
|
+ re_drop = re.compile("((?:公开)?招标?|中标(?:结果)?|结果|公[告示]?|招标公告?|中标公[告示]?|候选人公[告示]|终止|"
|
|
|
+ "[流废]标|资格预审|预审|成交(?:结果)?|交易|交易信息|入围|合同|通知书)$")
|
|
|
+ re_errorCode = re.compile("账号|身份证|机构编号|代理机构|品目|单位编[号码]|索引号|标准[^项目]*$|资产编号|型号|序列号"
|
|
|
+ "|宗地编号|地块编号|监测编号|不动产证")
|
|
|
+ # |备案[^,.;，。；]*[号码]
|
|
|
+ for id, text, sentences, entity_type, begin, end, entity_text, code_left in zip(data['document_id'], data['text'],
|
|
|
+ data['sentences'], data['entity_type'],
|
|
|
+ data['begin'], data['end'],
|
|
|
+ data['entity_text'], data['code_left']):
|
|
|
+ if entity_type == 'name':
|
|
|
+ if re_drop.search(entity_text):
|
|
|
+ name_2 = re_drop.sub('', re_drop.sub('', entity_text))
|
|
|
+ entity_text = name_2
|
|
|
+ text_tagLabels[id][0].append(entity_text)
|
|
|
+ if entity_type == 'code':
|
|
|
+ if not re_errorCode.search(str(code_left)):
|
|
|
+ text_tagLabels[id][1].append(entity_text)
|
|
|
+ train_data = []
|
|
|
+ max_len = 400
|
|
|
+ def hasNotBeenLabeled(items,code_begin,code):
|
|
|
+ for i in range(code_begin,code_begin+len(code)):
|
|
|
+ if items[i]!="O":
|
|
|
+ return False
|
|
|
+ return True
|
|
|
+ count = 0
|
|
|
+ for id,text in zip(id_list,text_list):
|
|
|
+ count += 1
|
|
|
+ print(count)
|
|
|
+ names = text_tagLabels[id][0]
|
|
|
+ names = list(set(names))
|
|
|
+ names.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ codes = text_tagLabels[id][1]
|
|
|
+ codes = list(set(codes))
|
|
|
+ codes.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ sentences = text.split('。')
|
|
|
+ for sentence in sentences:
|
|
|
+ l = len(sentence)
|
|
|
+ if l==0:
|
|
|
+ continue
|
|
|
+ elif l > max_len:
|
|
|
+ l = max_len
|
|
|
+ sentence = sentence[:400]
|
|
|
+ sentence_label = ['O']*l
|
|
|
+ code_find_flag = False
|
|
|
+ name_find_flag = False
|
|
|
+ if names:
|
|
|
+ for name in names:
|
|
|
+ name_begins = findAllIndex(name,sentence)
|
|
|
+ for name_begin in name_begins:
|
|
|
+ if hasNotBeenLabeled(sentence_label,name_begin,name):
|
|
|
+ for j in range(name_begin,name_begin+len(name)):
|
|
|
+ if j==name_begin:
|
|
|
+ sentence_label[j] = "PN_B"
|
|
|
+ elif j==name_begin+len(name)-1:
|
|
|
+ sentence_label[j] = "PN_E"
|
|
|
+ else:
|
|
|
+ sentence_label[j] = "PN_M"
|
|
|
+ name_find_flag = True
|
|
|
+ if codes:
|
|
|
+ for code in codes:
|
|
|
+ code_begins = findAllIndex(code,sentence)
|
|
|
+ for code_begin in code_begins:
|
|
|
+ if hasNotBeenLabeled(sentence_label,code_begin,code):
|
|
|
+ for j in range(code_begin,code_begin+len(code)):
|
|
|
+ if j==code_begin:
|
|
|
+ sentence_label[j] = "PC_B"
|
|
|
+ elif j==code_begin+len(code)-1:
|
|
|
+ sentence_label[j] = "PC_E"
|
|
|
+ else:
|
|
|
+ sentence_label[j] = "PC_M"
|
|
|
+ code_find_flag = True
|
|
|
+ if code_find_flag or name_find_flag:
|
|
|
+ train_data.append([sentence,sentence_label])
|
|
|
+ else:
|
|
|
+ if np.random.random() <= 0.75:
|
|
|
+ train_data.append([sentence,sentence_label])
|
|
|
+ print(len(train_data))
|
|
|
+ save(train_data,'train_data_new.pk')
|
|
|
+
|
|
|
+def add_data_process():
|
|
|
+ def hasNotBeenLabeled(items, code_begin, code):
|
|
|
+ for i in range(code_begin, code_begin + len(code)):
|
|
|
+ if items[i] != "O":
|
|
|
+ return False
|
|
|
+ return True
|
|
|
+ train_data = []
|
|
|
+ max_len = 400
|
|
|
+ data_path = "C:\\Users\\admin\\Desktop\\项目编号和名称\\补充数据\\data_"
|
|
|
+ data_names = ["合同编号","出让公告","询价编号","询价单编号","出让成交公示",
|
|
|
+ "的通知","公告编号","交易编号","询价单号","房产英文类项目名称",
|
|
|
+ "挂牌编号","申购单号","订单编号","询价书编号"]
|
|
|
+ for data_name in data_names:
|
|
|
+ data = pd.read_csv(data_path+data_name+"_process.csv",index_col=0,encoding='utf-8')
|
|
|
+ count = 0
|
|
|
+ for text,_name,_code in zip(data['text'],data['pj_name'],data['pj_code']):
|
|
|
+ count += 1
|
|
|
+ print(count)
|
|
|
+ names = str(_name).split('+')
|
|
|
+ names.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ codes = str(_code).split('+')
|
|
|
+ codes.sort(key=lambda x: len(x), reverse=True)
|
|
|
+ sentences = text.split('。')
|
|
|
+ for sentence in sentences:
|
|
|
+ l = len(sentence)
|
|
|
+ if l == 0:
|
|
|
+ continue
|
|
|
+ elif l > max_len:
|
|
|
+ l = max_len
|
|
|
+ sentence = sentence[:400]
|
|
|
+ sentence_label = ['O'] * l
|
|
|
+ if names:
|
|
|
+ for name in names:
|
|
|
+ name_begins = findAllIndex(name, sentence)
|
|
|
+ for name_begin in name_begins:
|
|
|
+ if hasNotBeenLabeled(sentence_label, name_begin, name):
|
|
|
+ for j in range(name_begin, name_begin + len(name)):
|
|
|
+ if j == name_begin:
|
|
|
+ sentence_label[j] = "PN_B"
|
|
|
+ elif j == name_begin + len(name) - 1:
|
|
|
+ sentence_label[j] = "PN_E"
|
|
|
+ else:
|
|
|
+ sentence_label[j] = "PN_M"
|
|
|
+ if codes:
|
|
|
+ for code in codes:
|
|
|
+ code_begins = findAllIndex(code, sentence)
|
|
|
+ for code_begin in code_begins:
|
|
|
+ if hasNotBeenLabeled(sentence_label, code_begin, code):
|
|
|
+ for j in range(code_begin, code_begin + len(code)):
|
|
|
+ if j == code_begin:
|
|
|
+ sentence_label[j] = "PC_B"
|
|
|
+ elif j == code_begin + len(code) - 1:
|
|
|
+ sentence_label[j] = "PC_E"
|
|
|
+ else:
|
|
|
+ sentence_label[j] = "PC_M"
|
|
|
+ train_data.append([sentence, sentence_label])
|
|
|
+ d = load('train_data_new.pk')
|
|
|
+ print(len(d))
|
|
|
+ train_data = d + train_data
|
|
|
+ print(len(train_data))
|
|
|
+ print('ok')
|
|
|
+ save(train_data, 'train_data_new2.pk')
|
|
|
+
|
|
|
+def train2():
|
|
|
+ chunk_tags = {
|
|
|
+ 'O':0,
|
|
|
+ 'PN_B':1,
|
|
|
+ 'PN_M':2,
|
|
|
+ 'PN_E':3,
|
|
|
+ 'PC_B':4,
|
|
|
+ 'PC_M':5,
|
|
|
+ 'PC_E':6,
|
|
|
+ }
|
|
|
+ # 获取预训练的字向量矩阵
|
|
|
+ w2v_matrix = load('w2v_matrix.pk')
|
|
|
+ # print(w2v_matrix[:3])
|
|
|
+ vocab = load('codename_vocab.pk')
|
|
|
+ word2index = dict((w, i) for i, w in enumerate(np.array(vocab)))
|
|
|
+ print(vocab[:2])
|
|
|
+ MAXLEN = 400
|
|
|
+
|
|
|
+ data_x = []
|
|
|
+ data_y = []
|
|
|
+ data1 = load('train_data_new2.pk')
|
|
|
+ for _data in data1:
|
|
|
+ _x = list(_data[0])
|
|
|
+ _x = [word2index.get(_,word2index.get('<unk>')) for _ in _x]
|
|
|
+ _y = _data[1]
|
|
|
+ data_x.append(_x)
|
|
|
+ data_y.append(_y)
|
|
|
+ # 旧的标注数据加入
|
|
|
+ old_datas = load("data/old_datas2.pk")
|
|
|
+ for old_data in old_datas:
|
|
|
+ data_x.append([word2index.get(word,word2index.get('<unk>')) for word in old_data[0]])
|
|
|
+ data_y.append(old_data[1])
|
|
|
+ print("数据量：",len(data_x))
|
|
|
+ data_x = np.array([np.array(x) for x in data_x])
|
|
|
+ x_len = [MAXLEN if len(x) > MAXLEN else len(x) for x in data_x]
|
|
|
+ data_y = np.array([np.array([chunk_tags[_] for _ in y]) for y in data_y])
|
|
|
+ data_x = pad_sequences(data_x, maxlen=MAXLEN, padding="post", truncating="post")
|
|
|
+ data_y = pad_sequences(data_y, maxlen=MAXLEN, padding="post", truncating="post")
|
|
|
+ indices = np.random.permutation(data_x.shape[0])
|
|
|
+ count = len(data_x)
|
|
|
+ test_count = int(0.2 * count)
|
|
|
+ test_idx, train_idx = indices[:test_count], indices[test_count:]
|
|
|
+ train_x, test_x = data_x[train_idx, :], data_x[test_idx, :]
|
|
|
+ train_y, test_y = data_y[train_idx, :], data_y[test_idx, :]
|
|
|
+ train_x_len = np.array([x_len[idx] for idx in train_idx])
|
|
|
+ test_x_len = np.array([x_len[idx] for idx in test_idx])
|
|
|
+ print("训练数据量：",len(train_x))
|
|
|
+ print("训练数据量：",len(test_x))
|
|
|
+ # save([test_x,test_y,test_x_len],'my_test_data.pk')
|
|
|
+ with tf.Session(graph=tf.Graph()) as sess:
|
|
|
+ char_input,logits,target,keepprob,length,crf_loss,trans,train_op = BiLSTM_CRF_tfmodel(sess,embedding_weights=w2v_matrix)
|
|
|
+ sess.run(tf.global_variables_initializer())
|
|
|
+ epochs = 150
|
|
|
+ saver = tf.train.Saver(max_to_keep=max(epochs,10))
|
|
|
+ batch_size = 1024
|
|
|
+ _test_loss = 10000.
|
|
|
+ _test_f1 = 0.
|
|
|
+ for epoch in range(epochs):
|
|
|
+ batch_nums = 0
|
|
|
+ for x_batch,y_batch,x_len_batch in batch_iter(train_x,train_y,train_x_len,batch_size=batch_size):
|
|
|
+ train_loss,_ = sess.run([crf_loss,train_op],feed_dict={char_input:x_batch,target:y_batch,length:x_len_batch,keepprob:0.7})
|
|
|
+ batch_nums += 1
|
|
|
+ print("--epoch:" + str(epoch))
|
|
|
+ print("--"+str(batch_nums)+"batch_train--", "loss:", train_loss)
|
|
|
+ test_loss_sum = 0.
|
|
|
+ test_sum = 0
|
|
|
+ acc_sum = 0.
|
|
|
+ precision_1 = 0
|
|
|
+ precision_2 = 0
|
|
|
+ recall_1 = 0
|
|
|
+ recall_2 = 0
|
|
|
+ for test_xbatch,test_ybatch,test_xlen in batch_iter(test_x,test_y,test_x_len,batch_size=batch_size):
|
|
|
+ test_loss,_logits,_trans = sess.run([crf_loss,logits,trans],feed_dict={char_input:test_xbatch,target:test_ybatch,length:test_xlen,keepprob:1.0})
|
|
|
+ acc,_precision,_recall = getAcc(test_ybatch, _logits, _trans, test_xlen)
|
|
|
+ batch_len = len(test_xbatch)
|
|
|
+ test_sum += batch_len
|
|
|
+ acc_sum += acc*batch_len
|
|
|
+ precision_1 += _precision[0]
|
|
|
+ precision_2 += _precision[1]
|
|
|
+ recall_1 += _recall[0]
|
|
|
+ recall_2 += _recall[1]
|
|
|
+ test_loss_sum += test_loss*batch_len
|
|
|
+ print("==>epoch:" + str(epoch)+"have_done")
|
|
|
+ epoch_test_loss = test_loss_sum/test_sum
|
|
|
+ epoch_test_acc = acc_sum/test_sum
|
|
|
+ test_precision = precision_1/precision_2
|
|
|
+ test_recall = recall_1/recall_2
|
|
|
+ test_f1 = ner_f1_score(test_precision,test_recall)
|
|
|
+ print("--test --"," acc:",epoch_test_acc,'test_loss:',epoch_test_loss)
|
|
|
+ print('test_precision:',test_precision,'test_recall',test_recall,'test_f1',test_f1)
|
|
|
+ # if test_f1 > _test_f1:
|
|
|
+ # _test_f1 = test_f1
|
|
|
+ print("Saving-"+str(epoch)+"-model,test_loss:"+str(epoch_test_loss),'test_f1',test_f1)
|
|
|
+ saver.save(sess,"models_tf/"+str(epoch)+"-L"+str(epoch_test_loss)+"-F"+str(test_f1)+"-P"+str(test_precision)+"-R"+str(test_recall)+"/model.ckpt")
|
|
|
+
|
|
|
+def BiLSTM_CRF_tfmodel(sess,embedding_weights):
|
|
|
+ '''
|
|
|
+ :param embedding_weights: 预训练的字向量矩阵
|
|
|
+
|
|
|
+ '''
|
|
|
+ BiRNN_Unit = 100
|
|
|
+ chunk_tags = {
|
|
|
+ 'O': 0,
|
|
|
+ 'PN_B': 1,
|
|
|
+ 'PN_M': 2,
|
|
|
+ 'PN_E': 3,
|
|
|
+ 'PC_B': 4,
|
|
|
+ 'PC_M': 5,
|
|
|
+ 'PC_E': 6,
|
|
|
+ }
|
|
|
+
|
|
|
+ def embedding_layer(input,keepprob):
|
|
|
+ # 加载预训练的字向量矩阵
|
|
|
+ embedding = tf.get_variable(name="embedding",initializer=np.array(embedding_weights, dtype=np.float32),dtype=tf.float32)
|
|
|
+ embedding = tf.nn.embedding_lookup(params=embedding,ids=input)
|
|
|
+ embedding_drop = tf.nn.dropout(embedding,keepprob)
|
|
|
+ return embedding_drop
|
|
|
+
|
|
|
+ def BiLSTM_Layer(input,length):
|
|
|
+ with tf.variable_scope("BiLSTM"):
|
|
|
+ forward_cell = tf.contrib.rnn.BasicLSTMCell(BiRNN_Unit,state_is_tuple=True)
|
|
|
+ backward_cell = tf.contrib.rnn.BasicLSTMCell(BiRNN_Unit,state_is_tuple=True)
|
|
|
+ output, _ = tf.nn.bidirectional_dynamic_rnn(forward_cell,backward_cell,input,dtype=tf.float32,sequence_length=length)
|
|
|
+ output = tf.concat(output,2)
|
|
|
+ return output
|
|
|
+
|
|
|
+ def CRF_layer(input,num_tags,BiRNN_Unit,time_step,keepprob):
|
|
|
+ with tf.variable_scope("CRF"):
|
|
|
+ with tf.variable_scope("hidden"):
|
|
|
+ w_hidden = tf.get_variable(name='w_hidden',shape=(BiRNN_Unit*2,BiRNN_Unit),dtype=tf.float32,
|
|
|
+ initializer=initializers.xavier_initializer(),regularizer=tf.contrib.layers.l2_regularizer(0.001))
|
|
|
+ b_hidden = tf.get_variable(name='b_hidden',shape=(BiRNN_Unit),dtype=tf.float32,initializer=tf.zeros_initializer())
|
|
|
+ # print(input)
|
|
|
+ input_reshape = tf.reshape(input,shape=(-1,BiRNN_Unit*2))
|
|
|
+ hidden = tf.tanh(tf.nn.xw_plus_b(input_reshape,w_hidden,b_hidden))
|
|
|
+ hidden = tf.nn.dropout(hidden,keepprob)
|
|
|
+ with tf.variable_scope("output"):
|
|
|
+ w_output = tf.get_variable(name='w_output',shape=(BiRNN_Unit,num_tags),dtype=tf.float32,initializer=initializers.xavier_initializer(),regularizer=tf.contrib.layers.l2_regularizer(0.001))
|
|
|
+ b_output = tf.get_variable(name='b_output',shape=(num_tags),dtype=tf.float32,initializer=tf.zeros_initializer())
|
|
|
+ pred = tf.nn.xw_plus_b(hidden,w_output,b_output)
|
|
|
+ logits_ = tf.reshape(pred,shape=(-1,time_step,num_tags),name='logits')
|
|
|
+ return logits_
|
|
|
+
|
|
|
+ def layer_loss(input,true_target,num_tags,length):
|
|
|
+ with tf.variable_scope("crf_loss"):
|
|
|
+ trans = tf.get_variable(name='transitons',shape=(num_tags,num_tags),dtype=tf.float32,initializer=initializers.xavier_initializer())
|
|
|
+ log_likelihood,trans = crf_log_likelihood(inputs=input,tag_indices=true_target,transition_params=trans,sequence_lengths=length)
|
|
|
+ return tf.reduce_mean(-log_likelihood),trans
|
|
|
+
|
|
|
+ with sess.graph.as_default():
|
|
|
+ char_input = tf.placeholder(name='char_input',shape=(None,None),dtype=tf.int32)
|
|
|
+ target = tf.placeholder(name='target',shape=(None,None),dtype=tf.int32)
|
|
|
+ length = tf.placeholder(name='length',shape=(None,),dtype=tf.int32)
|
|
|
+ keepprob = tf.placeholder(name='keepprob',dtype=tf.float32)
|
|
|
+
|
|
|
+ _embedding = embedding_layer(char_input,keepprob)
|
|
|
+ _shape = tf.shape(char_input)
|
|
|
+ batch_size = _shape[0]
|
|
|
+ step_size = _shape[-1]
|
|
|
+ bilstm = BiLSTM_Layer(_embedding,length)
|
|
|
+ _logits = CRF_layer(bilstm,num_tags=len(chunk_tags),BiRNN_Unit=BiRNN_Unit,time_step=step_size,keepprob=keepprob)
|
|
|
+ crf_loss,trans = layer_loss(_logits,true_target=target,num_tags=len(chunk_tags),length=length)
|
|
|
+ global_step = tf.Variable(0,trainable=False)
|
|
|
+ with tf.variable_scope("optimizer"):
|
|
|
+ opt = tf.train.AdamOptimizer(0.002)
|
|
|
+ grads_vars = opt.compute_gradients(crf_loss)
|
|
|
+ capped_grads_vars = [[tf.clip_by_value(g,-5,5),v] for g,v in grads_vars]
|
|
|
+ train_op = opt.apply_gradients(capped_grads_vars,global_step)
|
|
|
+ return char_input,_logits,target,keepprob,length,crf_loss,trans,train_op
|
|
|
+
|
|
|
+
|
|
|
+def batch_iter(x, y,x_len, batch_size=256):
|
|
|
+ '''
|
|
|
+ :param x: content2id
|
|
|
+ :param y: label2id
|
|
|
+ :param batch_size: 每批参与训练的句子数量
|
|
|
+ :return:
|
|
|
+ '''
|
|
|
+ data_len = len(x)
|
|
|
+ num_batch = int((data_len - 1) / batch_size) + 1 #计算一个epoch,需要多少次batch
|
|
|
+ indices = np.random.permutation(data_len) #生成随机数列
|
|
|
+ x = x[indices]
|
|
|
+ y = y[indices]
|
|
|
+ x_len = x_len[indices]
|
|
|
+ for i in range(num_batch):
|
|
|
+ start_id = batch_size * i
|
|
|
+ end_id = min(batch_size*(i+1), data_len)
|
|
|
+ yield x[start_id:end_id], y[start_id:end_id],x_len[start_id:end_id]
|
|
|
+from sklearn.metrics import accuracy_score
|
|
|
+def getAcc(y_batch,logits,trans,lengths):
|
|
|
+ index = 0
|
|
|
+ small = -1000.0
|
|
|
+
|
|
|
+ preds = []
|
|
|
+ true_tags = []
|
|
|
+ for score, length in zip(logits, lengths):
|
|
|
+ score = score[:length]
|
|
|
+ path, _ = tf.contrib.crf.viterbi_decode(score, trans)
|
|
|
+ preds += path[0:]
|
|
|
+ index += 1
|
|
|
+
|
|
|
+ for y, length in zip(y_batch, lengths):
|
|
|
+ y = y.tolist()
|
|
|
+ true_tags += y[: length]
|
|
|
+ _preds = list(preds)
|
|
|
+ _true_tags = list(true_tags)
|
|
|
+ acc = accuracy_score(np.reshape(true_tags,(-1)), np.reshape(preds,(-1)))
|
|
|
+ precision_1,precision_2,_ = ner_precision(_preds,_true_tags)
|
|
|
+ recall_1,recall_2,_ = ner_recall(_preds,_true_tags)
|
|
|
+
|
|
|
+ return acc,[precision_1,precision_2],[recall_1,recall_2]
|
|
|
+
|
|
|
+
|
|
|
+def decode(logits, trans, sequence_lengths, tag_num):
|
|
|
+ viterbi_sequences = []
|
|
|
+ for logit, length in zip(logits, sequence_lengths):
|
|
|
+ score = logit[:length]
|
|
|
+ viterbi_seq, viterbi_score = viterbi_decode(score, trans)
|
|
|
+ viterbi_sequences.append(viterbi_seq)
|
|
|
+ return viterbi_sequences
|
|
|
+
|
|
|
+
|
|
|
+def new_process():
|
|
|
+ data = pd.read_csv("C:\\Users\\admin\\Desktop\\项目编号和名称\\data_询价书编号.csv",index_col=0,encoding='utf-8')
|
|
|
+ text_list = []
|
|
|
+ for id,text in zip(data['id'],data['text']):
|
|
|
+ # id_list.append(id)
|
|
|
+ text_list.append(text)
|
|
|
+ page_content = get_article1(text_list)
|
|
|
+ data['text'] = page_content
|
|
|
+ data.to_csv('C:\\Users\\admin\\Desktop\\项目编号和名称\\data_询价书编号_process.csv')
|
|
|
+
|
|
|
+def new_test_code():
|
|
|
+ data = pd.read_csv("C:\\Users\\admin\\Desktop\\code_test_process2.csv",index_col=0)
|
|
|
+ sentences_list = []
|
|
|
+ for text in data['text']:
|
|
|
+ sentences = text.split("。")
|
|
|
+ sentences_list.append(sentences)
|
|
|
+ model_path = "models_tf/27-0.984184712668-0.598231307426/model.ckpt"
|
|
|
+ name_list,code_list = predict_CodeName(sentences_list,model_path)
|
|
|
+ data['code'] = code_list
|
|
|
+ data['name'] = name_list
|
|
|
+ data.to_csv("C:\\Users\\admin\\Desktop\\code_test结果2-3.csv")
|
|
|
+
|
|
|
+
|
|
|
+def predict_CodeName(articles,model_path):
|
|
|
+
|
|
|
+ w2v_matrix = load('w2v_matrix.pk')
|
|
|
+ vocab = load('codename_vocab.pk')
|
|
|
+ word2index = dict((w, i) for i, w in enumerate(np.array(vocab)))
|
|
|
+
|
|
|
+ model_path = model_path
|
|
|
+ sess = tf.Session(graph=tf.Graph())
|
|
|
+ with sess:
|
|
|
+ char_input, logits, target, keepprob,length, crf_loss, trans, train_op = BiLSTM_CRF_tfmodel(sess, w2v_matrix)
|
|
|
+ sess.run(tf.global_variables_initializer())
|
|
|
+ saver = tf.train.Saver()
|
|
|
+ saver.restore(sess, model_path)
|
|
|
+ re_name = re.compile("12*3")
|
|
|
+ re_code = re.compile("45*6")
|
|
|
+ article_name_list = []
|
|
|
+ article_code_list = []
|
|
|
+ count = 0
|
|
|
+ for sentences in articles:
|
|
|
+ if len(sentences)>500:
|
|
|
+ sentences = sentences[:500]
|
|
|
+ # print(len(sentences))
|
|
|
+ count += 1
|
|
|
+ print(count)
|
|
|
+ sentence_len = [ min(len(sentence),2000) for sentence in sentences]
|
|
|
+ # maxlen = max(sentence_len)
|
|
|
+ maxlen = max(sentence_len)
|
|
|
+ sentences_x = []
|
|
|
+ for sentence in sentences:
|
|
|
+ sentence = list(sentence)
|
|
|
+ sentence2id = [word2index.get(word,word2index.get('<unk>')) for word in sentence]
|
|
|
+ sentences_x.append(sentence2id)
|
|
|
+ sentences_x = pad_sequences(sentences_x,maxlen=maxlen,padding="post", truncating="post")
|
|
|
+ sentences_x = [np.array(x) for x in sentences_x]
|
|
|
+
|
|
|
+ _logits,_trans = sess.run([logits,trans],feed_dict={char_input:np.array(sentences_x),length:sentence_len,keepprob:1.0})
|
|
|
+
|
|
|
+ viterbi_sequence = decode(logits=_logits,trans=_trans,sequence_lengths=sentence_len,tag_num=7)
|
|
|
+ # print("==",_logits)
|
|
|
+ name_list = []
|
|
|
+ code_list = []
|
|
|
+ sentence_index = 0
|
|
|
+ for _seq,sentence in zip(viterbi_sequence,sentences):
|
|
|
+ seq_id = ''.join([str(s) for s in _seq])
|
|
|
+ if re_name.search(seq_id):
|
|
|
+ for _name in re_name.finditer(seq_id):
|
|
|
+ start = _name.start()
|
|
|
+ end = _name.end()
|
|
|
+ n = sentence[start:end]
|
|
|
+ name_list.append((n,start + sentence_index,end + sentence_index))
|
|
|
+ if re_code.search(seq_id):
|
|
|
+ for _code in re_code.finditer(seq_id):
|
|
|
+ start = _code.start()
|
|
|
+ end = _code.end()
|
|
|
+ c = sentence[start:end]
|
|
|
+ # print(n,'<==>',start,end)
|
|
|
+ code_list.append((c,start + sentence_index,end + sentence_index))
|
|
|
+ sentence_index += len(sentence)
|
|
|
+ article_name_list.append(name_list)
|
|
|
+ article_code_list.append(code_list)
|
|
|
+ return article_name_list,article_code_list
|
|
|
+from BiddingKG.dl.interface.Preprocessing import *
|
|
|
+# 网页公告处理
|
|
|
+def get_article1(articles,cost_time = dict(),useselffool=True):
|
|
|
+ '''
|
|
|
+ :param articles: 待处理的article source html
|
|
|
+ :param useselffool: 是否使用selffool
|
|
|
+ :return: list_articles
|
|
|
+ '''
|
|
|
+
|
|
|
+ list_articles = []
|
|
|
+ for article in articles:
|
|
|
+ a_time = time.time()
|
|
|
+ sourceContent = article
|
|
|
+ #表格处理
|
|
|
+ key_preprocess = "tableToText"
|
|
|
+ start_time = time.time()
|
|
|
+ article_processed = segment(tableToText(BeautifulSoup(sourceContent,"lxml")))
|
|
|
+
|
|
|
+ # log(article_processed)
|
|
|
+
|
|
|
+ if key_preprocess not in cost_time:
|
|
|
+ cost_time[key_preprocess] = 0
|
|
|
+ cost_time[key_preprocess] += time.time()-start_time
|
|
|
+
|
|
|
+ #article_processed = article[1]
|
|
|
+ list_articles.append(article_processed)
|
|
|
+ print(time.time()-a_time)
|
|
|
+ return list_articles
|
|
|
+# 分句处理
|
|
|
+def get_sentences1(list_articles,useselffool=True,cost_time=dict()):
|
|
|
+ '''
|
|
|
+
|
|
|
+ :param list_articles: 经过预处理的article text
|
|
|
+ :return: list_sentences
|
|
|
+ '''
|
|
|
+
|
|
|
+ list_sentences = []
|
|
|
+ for article in list_articles:
|
|
|
+ a_time = time.time()
|
|
|
+ list_sentences_temp = []
|
|
|
+ #表格处理
|
|
|
+ key_preprocess = "tableToText"
|
|
|
+ start_time = time.time()
|
|
|
+ article_processed = article
|
|
|
+
|
|
|
+
|
|
|
+ if key_preprocess not in cost_time:
|
|
|
+ cost_time[key_preprocess] = 0
|
|
|
+ cost_time[key_preprocess] += time.time()-start_time
|
|
|
+
|
|
|
+ #nlp处理
|
|
|
+ if article_processed is not None and len(article_processed)!=0:
|
|
|
+ split_patten = "。"
|
|
|
+ sentences = []
|
|
|
+ _begin = 0
|
|
|
+ sentences_set = set()
|
|
|
+ for _iter in re.finditer(split_patten,article_processed):
|
|
|
+ _sen = article_processed[_begin:_iter.span()[1]]
|
|
|
+ if len(_sen)>0 and _sen not in sentences_set:
|
|
|
+ sentences.append(_sen)
|
|
|
+ sentences_set.add(_sen)
|
|
|
+ _begin = _iter.span()[1]
|
|
|
+ _sen = article_processed[_begin:]
|
|
|
+ if len(_sen)>0 and _sen not in sentences_set:
|
|
|
+ sentences.append(_sen)
|
|
|
+ sentences_set.add(_sen)
|
|
|
+
|
|
|
+
|
|
|
+ '''
|
|
|
+ tokens_all = fool.cut(sentences)
|
|
|
+ #pos_all = fool.LEXICAL_ANALYSER.pos(tokens_all)
|
|
|
+ #ner_tag_all = fool.LEXICAL_ANALYSER.ner_labels(sentences,tokens_all)
|
|
|
+ ner_entitys_all = fool.ner(sentences)
|
|
|
+ '''
|
|
|
+ #限流执行
|
|
|
+ key_nerToken = "nerToken"
|
|
|
+ start_time = time.time()
|
|
|
+ # tokens_all = getTokens(sentences,useselffool=useselffool)
|
|
|
+ if key_nerToken not in cost_time:
|
|
|
+ cost_time[key_nerToken] = 0
|
|
|
+ cost_time[key_nerToken] += time.time()-start_time
|
|
|
+
|
|
|
+
|
|
|
+ for sentence_index in range(len(sentences)):
|
|
|
+
|
|
|
+ sentence_text = sentences[sentence_index]
|
|
|
+
|
|
|
+
|
|
|
+ list_sentences_temp.append(sentence_text)
|
|
|
+
|
|
|
+ if len(list_sentences_temp)==0:
|
|
|
+ list_sentences_temp.append(sentence_text)
|
|
|
+ list_sentences.append(list_sentences_temp)
|
|
|
+ print('2:',time.time()-a_time)
|
|
|
+ return list_sentences
|
|
|
+
|
|
|
+def _find_tag(labels,B_label,M_label,E_label):
|
|
|
+ result = []
|
|
|
+ ner_begin = 0
|
|
|
+ ner_end = 0
|
|
|
+ for num in range(len(labels)):
|
|
|
+ if labels[num] == B_label:
|
|
|
+ ner_begin = num
|
|
|
+ continue
|
|
|
+ if labels[num] == M_label and labels[num-1] == B_label:
|
|
|
+ continue
|
|
|
+ if labels[num] == M_label and labels[num-1] == M_label:
|
|
|
+ continue
|
|
|
+ if labels[num] == E_label:
|
|
|
+ if labels[num-1] == M_label or labels[num-1] == B_label:
|
|
|
+ ner_end = num+1
|
|
|
+ result.append((ner_begin,ner_end))
|
|
|
+ ner_begin = 0
|
|
|
+ ner_end = 0
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def find_all_tag(labels):
|
|
|
+ # tags = [("PN_B","PN_M","PN_E"),("PC_B","PC_M","PC_E")]
|
|
|
+ tags = [(1,2,3),(4,5,6)]
|
|
|
+ result = []
|
|
|
+ for tag in tags:
|
|
|
+ res = _find_tag(labels,B_label=tag[0],M_label=tag[1],E_label=tag[2])
|
|
|
+ result.append(res)
|
|
|
+ return result
|
|
|
+
|
|
|
+
|
|
|
+def ner_precision(pre_labels,true_labels):
|
|
|
+ '''
|
|
|
+ :param pre_tags: list
|
|
|
+ :param true_tags: list
|
|
|
+ :return:
|
|
|
+ '''
|
|
|
+ pre = []
|
|
|
+
|
|
|
+ pre_result = find_all_tag(pre_labels)
|
|
|
+ for item in pre_result:
|
|
|
+ for _item in item:
|
|
|
+ if pre_labels[_item[0]:_item[1]] == true_labels[_item[0]:_item[1]]:
|
|
|
+ pre.append(1)
|
|
|
+ else:
|
|
|
+ pre.append(0)
|
|
|
+ _sum = sum(pre)
|
|
|
+ _l = len(pre)
|
|
|
+ if not _l:
|
|
|
+ _l = 0.0001
|
|
|
+ return _sum,_l,_sum/_l
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+def ner_recall(pre_labels,true_labels):
|
|
|
+ '''
|
|
|
+ :param pre_tags: list
|
|
|
+ :param true_tags: list
|
|
|
+ :return:
|
|
|
+ '''
|
|
|
+ recall = []
|
|
|
+
|
|
|
+ true_result = find_all_tag(true_labels)
|
|
|
+ for item in true_result:
|
|
|
+ for _item in item:
|
|
|
+ if pre_labels[_item[0]:_item[1]] == true_labels[_item[0]:_item[1]]:
|
|
|
+ recall.append(1)
|
|
|
+ else:
|
|
|
+ recall.append(0)
|
|
|
+ _sum = sum(recall)
|
|
|
+ _l = len(recall)
|
|
|
+ if not _l:
|
|
|
+ _l = 0.0001
|
|
|
+ return _sum, _l, _sum/_l
|
|
|
+
|
|
|
+
|
|
|
+def ner_f1_score(precision,recall):
|
|
|
+ _temp = precision+recall
|
|
|
+ if not _temp:
|
|
|
+ _temp = 0.0001
|
|
|
+ return (2*precision*recall)/(_temp)
|
|
|
+
|
|
|
+def old_data_update():
|
|
|
+ data = load('data/old_datas.pk')
|
|
|
+ # print(len(data))
|
|
|
+ re_code = re.compile("(?:(?:公告|合同)[^，,。：;]{,3}编号[:：]*|寻源单据?号|计划[编文]?号|交易编[号码]|询价单编?[码号]|采购项目编号)([\-\d\w\(\)\（\）\[\]\【\】号]{3,})",re.A)
|
|
|
+ index = 0
|
|
|
+ updat_list = []
|
|
|
+ for d in data:
|
|
|
+ sentence = ''.join(d[0])
|
|
|
+ label = d[1]
|
|
|
+ if re_code.search(sentence):
|
|
|
+ for item in re_code.finditer(sentence):
|
|
|
+ begin,end = item.span()
|
|
|
+ # print(sentence[max(0,begin-8):end])
|
|
|
+ # print(sentence[begin:end])
|
|
|
+ la = label[begin:end]
|
|
|
+ if 'PC_B' not in la:
|
|
|
+ updat_list.append(index)
|
|
|
+ index += 1
|
|
|
+ updat_list = list(set(updat_list))
|
|
|
+ print(len(updat_list))
|
|
|
+ for u in updat_list:
|
|
|
+ item = data[u]
|
|
|
+ sentence = ''.join(item[0])
|
|
|
+ label = item[1]
|
|
|
+ re_res = re_code.findall(sentence)
|
|
|
+ for res in re_res:
|
|
|
+ begin = findAllIndex(res,sentence)
|
|
|
+ for b in begin:
|
|
|
+ e = b + len(res)
|
|
|
+ label[b] = 'PC_B'
|
|
|
+ label[e-1] = 'PC_E'
|
|
|
+ for i in range(b+1,e-1):
|
|
|
+ label[i] = 'PC_M'
|
|
|
+ data[u] = (item[0],label)
|
|
|
+ # print(sentence)
|
|
|
+ # print('---')
|
|
|
+ # print(label)
|
|
|
+ save(data,'data/old_datas2.pk')
|
|
|
+
|
|
|
+def get_word_matrix():
|
|
|
+ # 获取预训练的字向量
|
|
|
+ vocab_model = getModel_word()
|
|
|
+ _, w2v_matrix = getVocabAndMatrix(vocab_model, Embedding_size=60)
|
|
|
+ # 去除第一行<pad>全0.行
|
|
|
+ w2v_matrix = w2v_matrix[1:]
|
|
|
+ # <pad>
|
|
|
+ pad_0 = np.zeros((1, w2v_matrix.shape[1]), dtype=float)
|
|
|
+ # <unk>
|
|
|
+ unk_1 = np.random.normal(-0.25, 0.25, (1, w2v_matrix.shape[1]))
|
|
|
+ w2v_matrix = np.concatenate((pad_0, unk_1, w2v_matrix), axis=0)
|
|
|
+ print(w2v_matrix[:3])
|
|
|
+ save(w2v_matrix,"w2v_matrix.pk")
|
|
|
+
|
|
|
+if __name__ == '__main__':
|
|
|
+ # get_data()
|
|
|
+ # data_process()
|
|
|
+ # add_data_process()
|
|
|
+ # train2()
|
|
|
+ # test2()
|
|
|
+ # new_test()
|
|
|
+ # new_process()
|
|
|
+ # new_test_code()
|
|
|
+ # get_word_matrix()
|
|
|
+ # old_data_update()
|
|
|
+
|
|
|
+ # model_path = "models_tf/76-L0.472526232355-F0.8848208266348597-P0.8845455959355073-R0.8850962286662862/model.ckpt"
|
|
|
+ model_path = "models_tf/59-L0.471516189943-F0.8802154826344823-P0.8789179683459191-R0.8815168335321886/model.ckpt"
|
|
|
+ text = '''[X2002185]2020年11月麻城市生活垃圾焚烧发电项目厂前区零星计划
|
|
|
+ '''
|
|
|
+ name_list, code_list = predict_CodeName([text.split('。')], model_path)
|
|
|
+ print(name_list)
|
|
|
+ print(code_list)
|
|
|
+
|
|
|
+ pass
|
Jiasheng