import os,sys # sys.path.append('/data/python/znj/BIDI_ML_INFO_EXTRACTION/') import numpy as np from BiddingKG.dl.common.models import * # import tensorflow as tf from keras.layers import * from keras.models import Model from keras.utils import Sequence,to_categorical import keras.backend as K # from keras.callbacks import Callback from keras.optimizers import Adam import pandas as pd from sklearn.metrics import classification_report maxlen = 512 words_size = 128 # batch_size = 64 class Attention(Layer): """多头注意力机制 """ def __init__(self, nb_head, size_per_head, **kwargs): self.nb_head = nb_head self.size_per_head = size_per_head self.out_dim = nb_head * size_per_head super(Attention, self).__init__(**kwargs) def build(self, input_shape): super(Attention, self).build(input_shape) q_in_dim = input_shape[0][-1] k_in_dim = input_shape[1][-1] v_in_dim = input_shape[2][-1] self.q_kernel = self.add_weight(name='q_kernel', shape=(q_in_dim, self.out_dim), initializer='glorot_normal') self.k_kernel = self.add_weight(name='k_kernel', shape=(k_in_dim, self.out_dim), initializer='glorot_normal') self.v_kernel = self.add_weight(name='w_kernel', shape=(v_in_dim, self.out_dim), initializer='glorot_normal') def mask(self, x, mask, mode='mul'): if mask is None: return x else: for _ in range(K.ndim(x) - K.ndim(mask)): mask = K.expand_dims(mask, K.ndim(mask)) if mode == 'mul': return x * mask else: return x - (1 - mask) * 1e10 def call(self, inputs): q, k, v = inputs[:3] v_mask, q_mask = None, None if len(inputs) > 3: v_mask = inputs[3] if len(inputs) > 4: q_mask = inputs[4] # 线性变换 qw = K.dot(q, self.q_kernel) kw = K.dot(k, self.k_kernel) vw = K.dot(v, self.v_kernel) # 形状变换 qw = K.reshape(qw, (-1, K.shape(qw)[1], self.nb_head, self.size_per_head)) kw = K.reshape(kw, (-1, K.shape(kw)[1], self.nb_head, self.size_per_head)) vw = K.reshape(vw, (-1, K.shape(vw)[1], self.nb_head, self.size_per_head)) # 维度置换 qw = K.permute_dimensions(qw, (0, 2, 1, 3)) kw = K.permute_dimensions(kw, (0, 2, 1, 3)) vw = K.permute_dimensions(vw, (0, 2, 1, 3)) # Attention a = K.batch_dot(qw, kw, [3, 3]) / self.size_per_head**0.5 a = K.permute_dimensions(a, (0, 3, 2, 1)) a = self.mask(a, v_mask, 'add') a = K.permute_dimensions(a, (0, 3, 2, 1)) a = K.softmax(a) # 完成输出 o = K.batch_dot(a, vw, [3, 2]) o = K.permute_dimensions(o, (0, 2, 1, 3)) o = K.reshape(o, (-1, K.shape(o)[1], self.out_dim)) o = self.mask(o, q_mask, 'mul') return o def compute_output_shape(self, input_shape): return (input_shape[0][0], input_shape[0][1], self.out_dim) class OurLayer(Layer): """定义新的Layer，增加reuse方法，允许在定义Layer时调用现成的层 """ def reuse(self, layer, *args, **kwargs): if not layer.built: if len(args) > 0: inputs = args[0] else: inputs = kwargs['inputs'] if isinstance(inputs, list): input_shape = [K.int_shape(x) for x in inputs] else: input_shape = K.int_shape(inputs) layer.build(input_shape) outputs = layer.call(*args, **kwargs) for w in layer.trainable_weights: if w not in self._trainable_weights: self._trainable_weights.append(w) for w in layer.non_trainable_weights: if w not in self._non_trainable_weights: self._non_trainable_weights.append(w) for u in layer.updates: if not hasattr(self, '_updates'): self._updates = [] if u not in self._updates: self._updates.append(u) return outputs class OurBidirectional(OurLayer): """自己封装双向RNN，允许传入mask，保证对齐 """ def __init__(self, layer, **args): super(OurBidirectional, self).__init__(**args) self.forward_layer = layer.__class__.from_config(layer.get_config()) self.backward_layer = layer.__class__.from_config(layer.get_config()) self.forward_layer.name = 'forward_' + self.forward_layer.name self.backward_layer.name = 'backward_' + self.backward_layer.name def reverse_sequence(self, x, mask): """这里的mask.shape是[batch_size, seq_len, 1] """ seq_len = K.round(K.sum(mask, 1)[:, 0]) seq_len = K.cast(seq_len, 'int32') return tf.reverse_sequence(x, seq_len, seq_dim=1) def call(self, inputs): x, mask = inputs x_forward = self.reuse(self.forward_layer, x) x_backward = self.reverse_sequence(x, mask) x_backward = self.reuse(self.backward_layer, x_backward) x_backward = self.reverse_sequence(x_backward, mask) x = K.concatenate([x_forward, x_backward], -1) if K.ndim(x) == 3: return x * mask else: return x def compute_output_shape(self, input_shape): return input_shape[0][:-1] + (self.forward_layer.units * 2,) def classify_model(): num_classes = 6 embed_input = Input(shape=(None,words_size)) mask = Lambda(lambda x: K.cast(K.not_equal(K.sum(x,axis=-1,keepdims=True), 0), 'float32'))(embed_input) # mask = Lambda(lambda x: K.cast(K.not_equal(x, np.zeros(words_size,dtype=float)), 'float32'))(embed_input) # test_model = Model([embed_input],mask) input_drop = Dropout(0.25)(embed_input) t = OurBidirectional(GRU(64, return_sequences=True))([input_drop,mask]) h = Attention(8, 16)([t, t, t, mask]) h = Concatenate()([t, h]) # avg = layers.GlobalAveragePooling1D()(h) # output = Dense(num_classes, activation='softmax')(avg) h = Lambda(lambda x: x[0] * x[1])([h, mask]) h_dim = K.int_shape(h)[-1] h = Masking(mask_value=np.zeros(h_dim), input_shape=(maxlen, h_dim))(h) h = Dropout(0.25)(h) gru_output = Bidirectional(GRU(128))(h) output = Dense(num_classes, activation='softmax')(gru_output) # h = Dropout(0.25)(h) # atten = Attention02()(h,mask=K.squeeze(mask,axis=-1)) # output = Dense(num_classes, activation='softmax')(atten) model = Model([embed_input],output) model.summary() learn_rate = 0.0002 model.compile(optimizer=optimizers.Adam(lr=learn_rate), loss=losses.binary_crossentropy, metrics=[precision, recall, f1_score]) return model from BiddingKG.dl.common.nerUtils import getTokens import jieba def preprocess(text): text = re.sub("\n+",'，',text) text = re.sub("\s+|？+",'',text) text = re.sub("[\.·_]{2,}", '，', text) text = re.sub("_", '', text) text = text[:2500] sentences = text.split("。") sentences = [s+"。" for s in sentences if s] if not sentences: return [] tokens = getTokens(sentences) new_tokens = [] for t in tokens: new_tokens.extend(t) return new_tokens def preprocess2(text): text = re.sub("\n+",'，',text) text = re.sub("\s+|？+",'',text) text = re.sub("[\.·_]{2,}", '，', text) text = re.sub("_", '', text) text = text[:2500] tokens = list(jieba.cut(text)) return tokens from BiddingKG.dl.common.Utils import getModel_w2v model_w2v = getModel_w2v() def get_words_matrix(words): if words in model_w2v.vocab: return model_w2v[words] else: return model_w2v['unk'] def data_generate(): train_x = [] train_y = [] train_text = [] # # 数据1 # attachmentcon_list = [] # re_label_list = [] # data = pd.read_excel("attachment_data_relabel01.xlsx") # data = data[data['re_label'] != 6] # attachmentcon_list.extend([i for i in data['attachmentcon']]) # re_label_list.extend([i for i in data['re_label']]) # # filetitle_工程量清单标注.xlsx # data2 = pd.read_excel("filetitle_3.xlsx") # data2 = data2[:1887] # attachmentcon_list.extend([i for i in data2['attachmentcon']]) # re_label_list.extend([i for i in data2['re_label']]) # # filetitle_评标办法.xlsx # data3 = pd.read_excel("filetitle_5.xlsx") # attachmentcon_list.extend([i for i in data3['attachmentcon']]) # re_label_list.extend([i for i in data3['re_label']]) # # filetitle_限价（控制价）.xlsx # data7 = pd.read_excel("filetitle_2.xlsx") # attachmentcon_list.extend([i for i in data7['attachmentcon']]) # re_label_list.extend([i for i in data7['re_label']]) # # data4 = pd.read_excel("attachment_data_pred_label2.xlsx") # data4 = data4[(data4['pred_label'] == 2)|(data4['pred_label'] == 5)|(data4['pred_label'] == 6)] # attachmentcon_list.extend([i for i in data4['attachmentcon']]) # re_label_list.extend([i for i in data4['re_label']]) # # data5 = pd.read_excel("attachment_data_nolabel01_test_pred.xlsx") # data5 = data5[(data5['pred_label'] == 5)|(data5['pred_label'] == 6)] # attachmentcon_list.extend([i for i in data5['attachmentcon']]) # re_label_list.extend([i for i in data5['re_label']]) # # filetitle_采购清单.xlsx # data6 = pd.read_excel("filetitle_4.xlsx") # data6 = data6[:900] # for filetitle,attachmentcon,re_label in zip(data6['filetitle'],data6['attachmentcon'],data6['re_label']): # if re_label==6: # re_label = 4 # attachmentcon = filetitle + attachmentcon # attachmentcon_list.append(attachmentcon) # re_label_list.append(re_label) # # data8 = pd.read_excel("attachment_data_relabel01_test_pred2.xlsx") # data8 = data8[(data8['pred_label'] == 5) | (data8['pred_label'] == 6)] # attachmentcon_list.extend([i for i in data8['attachmentcon']]) # re_label_list.extend([i for i in data8['re_label']]) # # for text, label in zip(attachmentcon_list, re_label_list): # text = str(text) # tokens = preprocess2(text) # tokens = tokens[:maxlen] # train_text.append("".join(tokens)) # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # train_x.append(words_matrix) # y = np.zeros(6) # y[int(label)] = 1 # train_y.append(y) # 修正后数据 # data = pd.read_excel("test_pre_result4.xlsx") # 修正后数据(新家“评标结果”) data = pd.read_excel("test_pre_result5.xlsx") for text, label in zip(data['text'], data['re_label']): text = str(text) # tokens = preprocess2(text) # tokens = tokens[:maxlen] # train_text.append("".join(tokens)) # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # train_x.append(words_matrix) # y = np.zeros(6) # y[int(label)] = 1 # train_y.append(y) train_y.append(label) train_x.append(text) # 'filetitle_评标办法222.xlsx' # data2 = pd.read_excel("filetitle_5222.xlsx") # # data2 = data[(data['filetype']!='zip')&(data['filetype']!='rar')] # for text, label in zip(data2['attachmentcon'], data2['re_label']): # text = str(text) # tokens = preprocess2(text) # tokens = tokens[:maxlen] # train_text.append("".join(tokens)) # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # train_x.append(words_matrix) # y = np.zeros(6) # y[int(label)] = 1 # train_y.append(y) # train_y.append(label) # train_x.append(text) # filetitle_pingbiaojieguo0_pred2 评标结果类 # data3 = pd.read_excel("filetitle_pingbiaojieguo0_pred2.xlsx") # data3 = data3[data3['re_label']!=6] # for text, label in zip(data3['attachmentcon'], data3['re_label']): # text = str(text) # tokens = preprocess2(text) # tokens = tokens[:maxlen] # train_text.append("".join(tokens)) # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # train_x.append(words_matrix) # y = np.zeros(6) # y[int(label)] = 1 # train_y.append(y) # train_y.append(label) # train_x.append(text) print("数据总量：",len(train_x)) # train_x = np.array(train_x) # train_y = np.array(train_y) # data_len = len(train_x) # indices = np.random.permutation(data_len) # train_x = train_x[indices] # train_y = train_y[indices] # test_len = int(data_len*0.1) # # test_idx = indices[:test_len] # # train_idx = indices[test_len:] # test_x = train_x[:test_len] # test_y = train_y[:test_len] # print("测试数据量：", len(test_x)) # train_x = train_x[test_len:] # train_y = train_y[test_len:] # print("训练数据量：",len(train_x)) return train_x,train_y,train_text def add_data(): train_x = [] train_y = [] # train_text = [] # add数据 data = pd.read_excel("time_202196_pred3.xlsx") data = data[(data['filetype']!='zip')&(data['filetype']!='rar')] data = data[(data['label_prob']>0.965)|(data['pred_label']==0)] for text, label in zip(data['attachmentcon'], data['pred_label']): text = str(text) # tokens = preprocess2(text) # tokens = tokens[:maxlen] # train_text.append("".join(tokens)) # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # train_x.append(words_matrix) # y = np.zeros(6) # y[int(label)] = 1 # train_y.append(y) train_y.append(label) train_x.append(text) print("add数据量：",len(train_x)) # return train_x,train_y,train_text return train_x,train_y def train_1(): model = classify_model() # 载入数据 train_x, train_y,train_text = data_generate() data_len = len(train_x) # np.random.seed(7) # indices = np.random.permutation(data_len) import random random.seed(7) train_data = [d for d in zip(train_x,train_y,train_text)] random.shuffle(train_data) train_x = np.array([i[0] for i in train_data]) train_y = np.array([i[1] for i in train_data]) train_text = [i[2] for i in train_data] # train_x = train_x[indices] # train_y = train_y[indices] # train_text = [train_text[i] for i in indices.tolist()] test_len = int(data_len * 0.1) test_x = train_x[:test_len] test_y = train_y[:test_len] test_text = train_text[:test_len] print("测试数据量：", len(test_x)) train_x = train_x[test_len:] train_y = train_y[test_len:] train_text = train_text[test_len:] print("训练数据量：", len(train_x)) epochs = 45 batch_size = 256 checkpoint = ModelCheckpoint("model_label_classify3.weights",save_weights_only=True, monitor="val_loss", verbose=1, save_best_only=True, mode='min') model.fit(x=[train_x],y=train_y,validation_data=([test_x],test_y), epochs=epochs,batch_size=batch_size,shuffle=True,class_weight='auto',callbacks=[checkpoint]) model.load_weights("model_label_classify3.weights") y_pre = model.predict([test_x]) # 各类别预测评估 res1 = classification_report(np.argmax(test_y, axis=1), np.argmax(y_pre, axis=1)) print(res1) y_pre2 = model.predict([train_x]) res2 = classification_report(np.argmax(train_y, axis=1), np.argmax(y_pre2, axis=1)) print(res2) result_df = pd.DataFrame({"text":test_text+train_text,'y_label':[np.argmax(i) for i in test_y.tolist()+train_y.tolist()],'pre_label':[np.argmax(i) for i in y_pre.tolist()+y_pre2.tolist()]}) result_df['is_same'] = [1 if i==j else 0 for i,j in zip(result_df['y_label'],result_df['pre_label'])] result_df.to_excel("test_pre_result5.xlsx") return model class DataGenerator(Sequence): 'Generates data for Keras' def __init__(self, texts, labels, batch_size=256, dim=(maxlen,words_size), n_classes=6, shuffle=True): 'Initialization' self.dim = dim self.batch_size = batch_size self.labels = labels self.texts = texts self.n_classes = n_classes self.shuffle = shuffle self.on_epoch_end() def __len__(self): 'Denotes the number of batches per epoch' _len = len(self.texts) // self.batch_size if len(self.texts) % self.batch_size != 0: _len += 1 return _len # return int(np.floor(len(self.texts) / self.batch_size)) def __getitem__(self, index): 'Generate one batch of data' # Generate indexes of the batch indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size] # Find list of IDs list_texts = [self.texts[k] for k in indexes] _label = [self.labels[k] for k in indexes] # Generate data X, y = self.__data_generation(list_texts,_label) return X, y def on_epoch_end(self): 'Updates indexes after each epoch' self.indexes = np.arange(len(self.texts)) if self.shuffle == True: np.random.shuffle(self.indexes) def __data_generation(self, list_texts,_label): 'Generates data containing batch_size samples' # Initialization # X = np.empty((self.batch_size, *self.dim)) # y = np.empty((self.batch_size), dtype=int) batch_len = len(list_texts) x = np.empty((batch_len, *self.dim)) y = np.empty((batch_len), dtype=int) # Generate data for i, text in enumerate(list_texts): # Store sample tokens = preprocess2(text) tokens = tokens[:maxlen] words_matrix = np.zeros((maxlen, words_size)) for j in range(len(tokens)): words_matrix[j] = np.array(get_words_matrix(tokens[j])) x[i,] = words_matrix # Store class y[i] = _label[i] return x, to_categorical(y, num_classes=self.n_classes) def train_2(): model = classify_model() # 载入数据 train_x, train_y = add_data() train_x2, train_y2,train_text = data_generate() data_len = len(train_x) import random random.seed(7) # train_data = [d for d in zip(train_x,train_y,train_text)] train_data = [d for d in zip(train_x,train_y)] random.shuffle(train_data) train_x = [i[0] for i in train_data] train_y = [i[1] for i in train_data] # train_text = [i[2] for i in train_data] test_len = int(data_len * 0.1) test_x = train_x[:test_len] test_y = train_y[:test_len] # test_text = train_text[:test_len] print("测试数据量：", len(test_x)) train_x = train_x[test_len:] train_y = train_y[test_len:] # 加上 train_x2, train_y2 train_x.extend(train_x2) train_y.extend(train_y2) # train_x = np.array(train_x) # train_y = np.array(train_y) # train_text = train_text[test_len:] print("训练数据量：", len(train_x)) epochs = 30 batch_size = 256 checkpoint = ModelCheckpoint("model_label_classify6.weights",save_weights_only=True, monitor="val_loss", verbose=1, save_best_only=True, mode='min') # model.fit(x=[train_x],y=train_y,validation_data=([test_x],test_y), # epochs=epochs,batch_size=batch_size,shuffle=True,class_weight='auto',callbacks=[checkpoint]) training_generator = DataGenerator(train_x, train_y) validation_generator = DataGenerator(test_x, test_y) model.fit_generator(generator=training_generator,validation_data=validation_generator, use_multiprocessing=True, workers=3, epochs=epochs,shuffle=True,class_weight='auto',callbacks=[checkpoint]) # model.load_weights("model_label_classify4.weights") # y_pre = model.predict([test_x]) # # 各类别预测评估 # res1 = classification_report(np.argmax(test_y, axis=1), np.argmax(y_pre, axis=1)) # print(res1) # y_pre2 = model.predict([train_x]) # res2 = classification_report(np.argmax(train_y, axis=1), np.argmax(y_pre2, axis=1)) # print(res2) # result_df = pd.DataFrame({"text":test_text+train_text,'y_label':[np.argmax(i) for i in test_y.tolist()+train_y.tolist()],'pre_label':[np.argmax(i) for i in y_pre.tolist()+y_pre2.tolist()]}) # result_df['is_same'] = [1 if i==j else 0 for i,j in zip(result_df['y_label'],result_df['pre_label'])] # result_df.to_excel("test_pre_result5.xlsx") return model class Attention02(Layer): def __init__(self, **kwargs): self.init = initializers.get('normal') self.supports_masking = True self.attention_dim = 50 super(Attention02, self).__init__(**kwargs) def build(self, input_shape): assert len(input_shape) == 3 self.W = K.variable(self.init((input_shape[-1], 1))) self.b = K.variable(self.init((self.attention_dim,))) self.u = K.variable(self.init((self.attention_dim, 1))) self.trainable_weights = [self.W, self.b, self.u] super(Attention02, self).build(input_shape) def compute_mask(self, inputs, mask=None): return mask def call(self, x, mask=None): uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b)) ait = K.dot(uit, self.u) ait = K.squeeze(ait, -1) ait = K.exp(ait) if mask is not None: ait = ait * K.cast(mask, K.floatx()) # ait = ait * mask ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx()) ait = K.expand_dims(ait) weighted_input = x * ait output = K.sum(weighted_input, axis=1) return output def compute_output_shape(self, input_shape): return (input_shape[0], input_shape[-1]) def predict_one(text,model): text = str(text) tokens = preprocess2(text) tokens = tokens[:maxlen] words_matrix = np.zeros((maxlen, words_size)) for i in range(len(tokens)): words_matrix[i] = np.array(get_words_matrix(tokens[i])) y = model.predict([np.array([words_matrix])]) y_label = np.argmax(y[0]) prob = y[0][y_label] return y_label,prob def test01(): # test & predict model = classify_model() model.load_weights("model_label_classify6.weights") test_batch_size = 2000 # data = pd.read_csv("attachment_data_relabel01.csv") # data = pd.read_csv("time_202196.csv",chunksize=test_batch_size ) data = pd.read_csv("time_20210923.csv",chunksize=test_batch_size ) # data = pd.read_csv("filetitle_pingbiaojieguo0.csv",chunksize=test_batch_size ) classes_dict = { 0: '其他', 1: '招标文件', 2: '限价（控制价）', 3: '工程量清单', 4: '采购清单', 5: '评标办法' } # data = data[data['new_label'] == 6] # print("test_nums",len(data)) idx = 0 new_df = pd.DataFrame() for df in data: train_x = [] train_text = [] for text in df['attachmentcon']: text = str(text) tokens = preprocess2(text) tokens = tokens[:maxlen] train_text.append("".join(tokens)) words_matrix = np.zeros((maxlen, words_size)) for i in range(len(tokens)): words_matrix[i] = np.array(get_words_matrix(tokens[i])) train_x.append(words_matrix) train_x = np.array(train_x) y_pre = model.predict([train_x]) pred_label = [np.argmax(i) for i in y_pre.tolist()] label_prob = [y[y_label] for y,y_label in zip(y_pre.tolist(),pred_label)] classes = [classes_dict[label] for label in pred_label] text_len = [len(text) for text in train_text] df['pred_label'] = pred_label df['类别'] = classes df['label_prob'] = label_prob df['attachmentcon'] = train_text df['text_len'] = text_len new_df = pd.concat([new_df,df]) idx += test_batch_size print(idx) new_df.to_excel("time_20210923_pred.xlsx") # 大量标注 def data_process4(): model, test_model = classify_model() model.load_weights("model_label_classify6.weights") # data = pd.read_csv("C:/Users/Administrator/Desktop/attachment_data/attachment_data02.csv") data = pd.read_excel("C:/Users/Administrator/Desktop/attachment_data/time_202196.xlsx") print(data.info()) pred_label = [] label_prob = [] new_text = [] idx = 0 for text in data['attachmentcon']: print(idx) idx += 1 y_label, prob = predict_one(text, model) pred_label.append(y_label) label_prob.append(prob) data['pred_label'] = pred_label data['label_prob'] = label_prob # data['attachmenthtml'] = [re.sub('\n{2,}','',i.replace("
",'',i))[:4000] for i in data['attachmenthtml']] data['attachmenthtml'] = [i[:4500] for i in data['attachmenthtml']] data.to_excel("attachment_data_pred_label3.xlsx") def save_model(): graph = tf.Graph() with graph.as_default() as graph: with tf.Session(graph=graph).as_default() as sess: test_model = classify_model() test_model.load_weights("model_label_classify6.weights") tf.saved_model.simple_save(sess, "models2/model_attachment_classify/", inputs={"input0": test_model.input}, outputs={"outputs": test_model.output}) if __name__ == '__main__': # model = classify_model() test_text = '''招标文件项目编号：SDGP370302202102000110项目名称:淄川经济开发区中心小学校园智能化采购项目采购人:山东淄川经 济开发区管理委员会采购代理机构：淄博正益招标有限公司发出日期：2021年8月目录第一章投标邀请7一、项目基本情况7二、申请人的资格要 求8三、获取招标文件8四、提交投标文件截止时间、开标时间和地点8五、公告期限9六、其他补充事宜9第二章投标人须知11一、总则161.采 购人、采购代理机构及投标人162.资金来源183.投标费用184.适用法律18二、招标文件185.招标文件构成186.招标文件的澄清与修改207.投 标截止时间的顺延20三、投标文件的编制208.编制要求209.投标范围及投标文件中标准和计量单位的使用2110.投标文件构成2211.投标报价241 2.电子版投标文件2513.投标保证金2614.投标有效期2615.投标文件的签署及规定26四、投标文件的递交2616.投标文件的递交2617.递交 投标文件的截止时间2718.投标文件的接收、修改与撤回27五、开标及评标2719.开标2720.资格审查2821.组建评标委员会2922.投标文件符 合性审查与澄清3023.投标偏离3224.投标无效3225.比较和评价3326.废标3527.保密要求36六、确定中标3628.中标候选人的确定原则及标 准3629.确定中标候选人和中标人3630.采购任务取消3631.中标通知书3632.签订合同3633.履约保证金3734.政府采购融资担保3735.预付 款3736.廉洁自律规定3737.人员回避3738.质疑与接收3739.项目其他相关费用3940.合同公示3941.验收4042.履约验收公示4043.招标文 件解释权40第三章货物需求41一、项目概述41 ''' # test_text = re.sub('\n','',test_text) # print(preprocess(test_text)) # train # model = train_1() # model = train_2() # tokens = preprocess(test_text) # tokens = tokens[:maxlen] # words_matrix = np.zeros((maxlen, words_size)) # for i in range(len(tokens)): # words_matrix[i] = np.array(get_words_matrix(tokens[i])) # y = model.predict([np.array([words_matrix])]) # print('y:',y) # y_label = np.argmax(y[0]) # print('y_label:',y_label,y[0][y_label]) # test_mask = test_model.predict([np.array([words_matrix])]) # print('test_mask:',test_mask) test01() # save_model() # print(jieba.lcut("他来到上海交通大学")) # data_process4() # d1 = pd.read_excel("C:/Users/Administrator/Desktop/attachment_data/attachment_data_pred_label3.xlsx") # d2 = pd.read_excel("C:/Users/Administrator/Desktop/attachment_data/attachment_data_pred_label2.xlsx") # d1 = pd.concat([d1,d2]) # print(len(d1)) # d1 = d1[d1['pred_label']!=0] # print(len(d1)) # d1 = d1[d1['pred_label']!=1] # print(len(d1)) # data = pd.read_excel("C:/Users/Administrator/Desktop/attachment_data/time_202196_pred2.xlsx") # d0 = data[data['pred_label']==0][:2000] # d1 = data[data['pred_label']==1][:2000] # d2 = data[data['pred_label']==2][:2000] # d3 = data[data['pred_label']==3][:2000] # d4 = data[data['pred_label']==4][:2000] # d5 = data[data['pred_label']==5][:2000] # d = pd.concat([d0,d1,d2,d3,d4,d5]) # d.to_excel("C:/Users/Administrator/Desktop/attachment_data/test_pred.xlsx") # d1 = pd.read_csv("C:/Users/Administrator/Desktop/attachment_data/attachment_data_relabel01_test_pred2.csv") # d1.to_excel("C:/Users/Administrator/Desktop/attachment_data/attachment_data_relabel01_test_pred2.xlsx") # d1 = pd.read_excel("C:/Users/Administrator/Desktop/attachment_data/filetitle_评标结果0.xlsx") # d1.to_csv("C:/Users/Administrator/Desktop/attachment_data/filetitle_评标结果0.csv") # model_w2v = getModel_w2v() # i = 0 # print('unk',model_w2v.vocab['unk']) # print('unk',model_w2v.similar_by_word('unk')) # print('unk',model_w2v.vocab['pad']) # # print('unk',model_w2v.similar_by_word('pad')) # print(model_w2v.vocab['unk']) pass