BIDI_ML_INFO_EXTRACTION/BiddingKG/dl_dev/industry/data_process.py

from collections import Counter
import pandas as pd
import fool
import pickle
import json
import random
import time
import re
import numpy as np
import gensim

w2v = r"D:\bidi\BIDI_ML_INFO_EXTRACTION\BiddingKG\dl\wiki_128_word_embedding_new.vector"
# w2v = "/data/python/lishimin/BiddingKG/dl/wiki_128_word_embedding_new.vector"
model_w2v = gensim.models.KeyedVectors.load_word2vec_format(w2v, binary=True)

def cut_words(text_list):
    return fool.cut(text_list)

def words_docs_list2array(words_docs_list, maxSententLen=20):
    array = np.zeros((len(words_docs_list), maxSententLen, 128))
    for i in range(len(words_docs_list)):
        words = words_docs_list[i][-maxSententLen:]
        for j in range(len(words)):
            if words[j] in model_w2v:
                array[i][j] = model_w2v[words[j]]
            else:
                print('word not in w2v: ', words[j])
    return array

def dump_segwords():
    # df = pd.read_excel('data/新标准所有规则标注数据帅选数据.xlsx')[:]
    # df = pd.read_excel('data/215类数据太少类别补充数据.xlsx')[:]
    df = pd.read_excel('data/新标准所有规则标注数据去除产品长度大于100帅选并补充后165595条215类.xlsx')[:]
    df.dropna(subset=['industry_type'], inplace=True)
    print(df.head(5))
    df.fillna(' ', inplace=True)
    print('len_df', len(df))
    df = df[df['industry_type']!=' ']
    print('len_df', len(df))

    from collections import Counter
    c = Counter(df['industry_type'])
    print('类别数：', len(c))

    def del_words(tenderee, text):
        tenderee = tenderee.replace('(', '（').replace(')', '）')
        text = text.replace('(', '（').replace(')', '）')
        text = re.sub(
            '(废标|终止|综?合?评审|评标|开标|资审|履约|验收|成交|中标人?|中选人?|单一来源|合同|候选人|结果|变更|更正|答疑|澄清|意向|需求|采购|招标|询比?价|磋商|谈判|比选|比价|竞价|议价)的?(公告|预告|公示)?|关于为?|选取|定点|直接|邀请函?|通知书?|备案|公开|公示|公告|记录|竞争性',
            '', text)
        text = text.replace(tenderee, '')
        text = ' ' if text=="" else text
        return text

    def get_winer(sub_docs_json, winner):
        # winner = ' '
        if winner == ' ' and 'win_tenderer' in sub_docs_json:
            l = json.loads(sub_docs_json)
            for d in l:
                if d.get('win_tenderer', '')!="":
                    winner = d.get('win_tenderer', '')
                    return winner
        return winner
    df['doctitle'] = df.apply(lambda x:del_words(x['tenderee'], x['doctitle']), axis=1)
    df['project_name'] = df.apply(lambda x:del_words(x['tenderee'], x['project_name']), axis=1)
    # df['win_tenderer'] = df['sub_docs_json'].apply(lambda x:get_winer(x))
    df['win_tenderer'] = df.apply(lambda x:get_winer(x['sub_docs_json'], x['win_tenderer']), axis=1)
    labels = sorted(set(df['industry_type']))
    lb2id = {k:v for v, k in enumerate(labels)}
    print(df.head(5))
    print(labels, len(labels))
    print(lb2id[labels[0]], labels[0])

    t1 = time.time()

    f = open('data/all_segwords.txt', 'w', encoding='utf-8')
    # f = open('data/all_segwords.txt', 'a', encoding='utf-8')
    n = 0
    for docid, title, project, product, win_tenderer, lb in \
     zip(df['docid'], df['doctitle'], df['project_name'], df['product'], df['win_tenderer'], df['industry_type']):
        try:
            title_, project_, product_, win_tenderer_  = cut_words([title, project, product, win_tenderer])
            segwords = ' '.join(title_)+'#split#'+' '.join(project_)+'#split#'+' '.join(product_)+'#split#'+' '.join(win_tenderer_)
            f.write('%s\t%s\t%s\n'%(lb, segwords, docid))
            n += 1
            if n %1000==0:
                print('已完成%d篇'%n)
        except Exception as e:
            print('error : ', e)

    f.close()

        # all_data = []
    # all_data = [(docid,cut_words([title, project, product, win_tenderer]), lb) for docid,title,project,product,win_tenderer,lb in zip(df['docid'],df['doctitle'],df['project_name'],df['product'],df['win_tenderer'],df['industry_type'])]
    # print('all_data_len:', len(all_data))
    # with open('data/all_data.pkl', 'wb') as f:
    #     pickle.dump(all_data, f)
    # with open('data/lb2id.pkl', 'wb') as f:
    #     pickle.dump(lb2id, f)
    t2 = time.time()
    print('总耗时：', t2-t1)

# dump_segwords()

def get_array_data():
    with open('E:/行业分类/all_data.pkl', 'rb') as f:
        all_data = pickle.load(f)
    with open('E:/行业分类/lb2id.pkl', 'rb') as f:
        lb2id = pickle.load( f)
    print('all_data_len', len(all_data))
    print(lb2id)
    title = [it[0][0] for it in all_data]
    project = [it[0][1] for it in all_data]
    product = [it[0][2] for it in all_data]
    labels_type = [it[1] for it in all_data]

    title_array = words_docs_list2array(title)
    project_array = words_docs_list2array(project)
    product_array = words_docs_list2array(product)
    label_ids = [lb2id[it] for it in labels_type]
    label_array = np.zeros(shape=((len(label_ids), len(lb2id))))
    for i in range(len(label_ids)):
        label_array[i][label_ids[i]] = 1
    print(title_array.shape, label_array.shape)
    return title_array, project_array, product_array, label_array, all_data, lb2id

def split_train_test():
    # with open('E:/行业分类/all_data.pkl', 'rb') as f:
    #     all_data = pickle.load(f)
    # df = pd.DataFrame(all_data, columns=['segword_list', 'label'])
    # df['segword_list'] = df['segword_list'].apply(lambda x: json.dumps(x, ensure_ascii=False))
    # df.drop_duplicates(subset=['segword_list', 'label'], inplace=True)
    # df.drop_duplicates(subset=['segword_list'], inplace=True)

    with open('data/all_segwords.txt', 'r', encoding='utf-8') as f:
        lines = f.read().split('\n')
    lines = [it.strip().split('\t') for it in lines if '#split#' in it]
    print(lines[:3])
    df = pd.DataFrame(lines, columns=['label', 'segwords', 'docid'])
    print(len(df), df.head(3))
    df.drop_duplicates(subset=['segwords'], inplace=True)
    print(len(df))

    c = Counter(df['label'])
    less_10 = []
    for k, v in c.items():
        if v < 10:
            less_10.append(k)
    df = df[~df['label'].isin(less_10)]
    df_test = pd.DataFrame()
    for lb in set(df['label']):
        df_tmp = df[df['label']==lb]
        n = int(len(df_tmp)*0.2)
        df_test = df_test.append(df_tmp.sample(n=n), ignore_index=True)
        df_test = df_test.sample(frac=1)
    # df_train = df[~df['segword_list'].isin(df_test['segword_list'])]
    df_train = df[~df['segwords'].isin(df_test['segwords'])]
    df_train = df_train.sample(frac=1)
    print(len(df), len(df_train), len(df_test))
    df_train.to_excel('data/新标准215类train.xlsx')
    df_test.to_excel('data/新标准215类test.xlsx')
    return df_train, df_test

def df_to_array(df, seq_len=20):
    # df['segword_list'] = df['segword_list'].apply(lambda x:json.loads(x))
    # with open('E:/行业分类/lb2id.pkl', 'rb') as f:
    #     lb2id = pickle.load( f)
    labels = sorted(set(df['label']))
    lb2id = {k:v for v, k in enumerate(labels)}
    # all_data = list(df['segword_list'])
    all_data = df['segwords'].apply(lambda x:x.split('#split#'))
    for l in all_data:
        assert len(l) == 4
    print('all_data_len', len(all_data))
    print(lb2id)
    title = [it[0].split() for it in all_data]
    project = [it[1].split() for it in all_data]
    product = [it[2].split() for it in all_data]
    labels_type = [it for it in df['label']]

    title = [[it for it in l if re.search('^[\u4e00-\u9fa5]+$', it)] for l in title]
    project = [[it for it in l if re.search('^[\u4e00-\u9fa5]+$', it)] for l in project]
    product = [[it for it in l if re.search('^[\u4e00-\u9fa5]+$', it)] for l in product]

    title_array = words_docs_list2array(title,seq_len)
    project_array = words_docs_list2array(project,seq_len)
    product_array = words_docs_list2array(product,seq_len)
    label_ids = [lb2id[it] for it in labels_type]
    label_array = np.zeros(shape=((len(label_ids), len(lb2id))))
    for i in range(len(label_ids)):
        label_array[i][label_ids[i]] = 1
    print(title_array.shape, label_array.shape)
    return title_array, project_array, product_array, label_array, all_data, lb2id

def getVocabAndMatrix(model):
    '''
    加载词向量或字向量，返回词、字列表及向量矩阵,过滤掉非纯中文词、字
    :param model: 加载的词或字向量模型  gensim.models.keyedvectors.KeyedVectors
    :return:
    '''
    vocab = ['<pad>'] + [it for it in model.index2word if re.search('^[\u4e00-\u9fa5]+$', it)]
    word2index = {k:v for k,v in enumerate(vocab)}
    Embedding_size = model[model.index2word[0]].shape[0]
    embedding_matrix = np.zeros((len(vocab), Embedding_size))
    for i in range(1, len(vocab)):
        embedding_matrix[i] = model[vocab[i]]
    return vocab, embedding_matrix, word2index

def df_to_ids(df, word2index):
    def words2ids(words_docs_list, maxSententLen=20):
        array = np.zeros((len(words_docs_list), maxSententLen))
        for i in range(len(words_docs_list)):
            words = words_docs_list[i][-maxSententLen:]
            for j in range(len(words)):
                if words[j] in word2index:
                    array[i][j] = word2index[words[j]]
        return array
    df['segword_list'] = df['segword_list'].apply(lambda x:json.loads(x))
    # with open('E:/行业分类/lb2id.pkl', 'rb') as f:
    #     lb2id = pickle.load( f)
    labels = sorted(set(df['label']))
    lb2id = {k:v for v, k in enumerate(labels)}
    all_data = list(df['segword_list'])
    print('all_data_len', len(all_data))
    print(lb2id)
    title = [it[0] for it in all_data]
    project = [it[1] for it in all_data]
    product = [it[2] for it in all_data]
    labels_type = [it for it in df['label']]

    title_ids = words2ids(title)
    project_ids = words2ids(project)
    product_ids = words2ids(product)
    label_ids = [lb2id[it] for it in labels_type]
    label_array = np.zeros(shape=((len(label_ids), len(lb2id))))
    for i in range(len(label_ids)):
        label_array[i][label_ids[i]] = 1
    return title_ids, project_ids, product_ids, label_array, all_data, lb2id
if __name__ == "__main__":
    # dump_segwords()
    # split_train_test()
    # df = pd.read_excel('E:/行业分类/新标准所有规则标注数据过滤掉标题、项目、产品三要素重复的剩余498976条215类.xlsx')[:]
    # df.fillna(' ', inplace=True)
    # c = Counter(df['industry_type'])
    import copy
    # df_fill = pd.DataFrame()
    # for k, v in c.items():
    #     if v > 1000:
    #         print(k, v)
    #         df2 = copy.deepcopy(df[df['industry_type']==k])
    #         print(len(df2))
    #         if len(df2)>1000:
    #             df2.drop_duplicates(subset=['doctitle'], inplace=True)
    #             print(len(df2))
    #             if len(df2) > 1000:
    #                 df2.drop_duplicates(subset=['project_name'], inplace=True)
    #                 print(len(df2))
    #                 if len(df2) > 1000:
    #                     df2.drop_duplicates(subset=['product'], inplace=True)
    #                     print(len(df2))
    #                     if len(df2) > 1000:
    #                         df2.drop_duplicates(subset=['tenderee'], inplace=True)
    #                         print(len(df2))
    #
    #         df_fill = df_fill.append(df2, ignore_index=True)
    #     else:
    #         df2 = copy.deepcopy(df[df['industry_type'] == k])
    #         df_fill = df_fill.append(df2, ignore_index=True)
    # print('len_df_fill', len(df_fill))
    # df_fill = df_fill.sample(frac=1)
    # df_fill.to_excel('E:/行业分类/新标准所有规则标注数据帅选数据.xlsx')
    # import re
    # df = pd.read_excel('E:/行业分类/新标准所有规则标注数据帅选数据.xlsx')
    # c = Counter(df['industry_type'])
    # print(c.most_common())
    # l = []
    # pos = neg = 0
    # for i in df.index:
    #     title = df.loc[i, 'doctitle']
    #     name = df.loc[i, 'project_name']
    #     ree = df.loc[i, 'tenderee']
    #     name = re.sub(
    #         '(废标|终止|综?合?评审|评标|开标|资审|履约|验收|成交|中标人?|中选人?|单一来源|合同|候选人|结果|变更|更正|答疑|澄清|意向|需求|采购|招标|询比?价|磋商|谈判|比选|比价|竞价|议价)的?(公告|预告|公示)?|关于为?|选取|定点|直接|邀请函?|通知书?|备案|公开|公示|公告|记录|竞争性',
    #         '', name)
    #     if name in title:
    #         pos += 1
    #     else:
    #         neg += 1
    #         print(name,   title)
        # text = title.replace(name, '##').replace(ree, '')
        # text_l = text.split('##')
        # for w in text_l:
        #     l.append(w)

    # c = Counter(l)
    # print(c.most_common(100))
    # print('pos:%d, neg:%d'%(pos,neg))
    # with open('E:/行业分类/过滤词.txt', 'w', encoding='utf-8') as f:
    #     for it in c.most_common(200):
    #         f.write(it[0]+'\n')