ContentExtract/module/detail/content/1.py

'''
Created on 2019年3月7日

@author: User
'''

from Utils import *
from featureEngine import *
import math

data_file = "temp_data.pk"

data = load(data_file)

print(len(data))
print(data[0])
print(len(data[0][0]))
print(len(data[0][0][0]))


new_data = []
''''''
for i in range(len(data)):
    if len(data[i][0])<=100:
        new_data.append(data[i])

print("new data",len(new_data))
        
for i in range(len(new_data)):
    for k in range(len(new_data[i][0])):
        for h in range(len(new_data[i][0][k])):
            if math.isinf(new_data[i][0][k][h]) or math.isnan(new_data[i][0][k][h]):
                new_data[i][0][k][h] = -1
                print(new_data[i][0][k][h])
        


        
#print(count)
new_data.sort(key=lambda x:x[2])
save(new_data,"sourceData_36Input_"+str(len(new_data))+"_sort.pk")

#new_data = load("sourceData_27Input_25547.pk")

pd_data = paddinig(new_data)

save(pd_data, "data_"+str(len(pd_data[1]))+".pk")