<jupyter_start><jupyter_text>Mini Project Part 3: Classification using adjacent dataset and attribute datasetAuthor: <NAME>  SID: 12032471  Date: 2021.5.6  1 Read dataset and assign feature names to dataset<jupyter_code># import basic module
import numpy as np
import pandas as pd
import csv
import string
# read data from txt file
attr_data = pd.read_csv("attr.csv",header = None,usecols = [1,2,3,4,5,6])
train_label = pd.read_csv("label_train.csv",header=None,usecols = [1])
# assign a feature name to each column of data set, test set and label
feature_names = []
for i in range(attr_data.shape[1]):
    feature_names.append("feature" + str(i+1))
attr_data = pd.read_csv("attr.csv",header = None,usecols = [1,2,3,4,5,6],names=feature_names)
train_data = pd.read_csv("attr.csv",header = None,usecols = [1,2,3,4,5,6],nrows=4000,names=feature_names)
train_label = pd.read_csv("label_train.csv",header=None,usecols = [1],names=["label"])
test_data = pd.read_csv("attr.csv",header = None,usecols = [1,2,3,4,5,6],skiprows = 4000,nrows=1298,names=feature_names)
test_label = pd.read_csv("label_test.csv",header=None,usecols = [1],names=["label"])


print("feature names: ",feature_names,"\n")
print("\ndata.head:\n",attr_data.head(),"\n")
print("train data: \n",train_data.head(),"\n")
print("shape of train data: ",train_data.shape,"\n")
print("train label:\n ",train_label.head(),"\n")
print("shape of train label: ",train_label.shape,"\n")
print("test data:\n ",test_data.head(),"\n")
print("shape of test data: ",test_data.shape,"\n")
print("test label: \n",test_label.head(),"\n")
print("shape of test label: ",test_label.shape,"\n")<jupyter_output>feature names:  ['feature1', 'feature2', 'feature3', 'feature4', 'feature5', 'feature6'] 


data.head:
    feature1  feature2  feature3  feature4  feature5  feature6
0         1         2         8        22         0     23574
1         2         2        15        14       229         0
2         1         1         3         0       237      9976
3         2         2         3         0       230     19776
4         1         2         6         0       229     17097 

train data: 
    feature1  feature2  feature3  feature4  feature5  feature6
0         1         2         8        22         0     23574
1         2         2        15        14       229         0
2         1         1         3         0       237      9976
3         2         2         3         0       230     19776
4         1         2         6         0       229     17097 

shape of train data:  (4000, 6) 

train label:
     label
0   2009
1   2004
2   2008
3   2002
4   2007 

shape of train label:  (4000,[...]<jupyter_text>2 Read the  adjacency matrix and obtain accessible matrix<jupyter_code># get the adj matrix
tep_adjdata = []
with open('adjlist.csv','r') as csvfile:
    reader = csv.reader(csvfile)
    for row in reader:
        tep_adjdata.append(row)
adjlist_data = []
for i in range(len(tep_adjdata)):
    if len(tep_adjdata[i]) == 2 and tep_adjdata[i][1] == '':
        tep_adjdata[i].pop()
    tmp = []
    for j in range(len(tep_adjdata[i])):
        tmp.append(int(tep_adjdata[i][j]))
    adjlist_data.append(tmp)
adjmatrix = np.zeros((5298,5298),dtype = int)
for row in range(len(adjlist_data)):
    for column in range(len(adjlist_data[row])):
        if column != 0:
            adjmatrix[row][adjlist_data[row][column]] = 1
adjmatrix = pd.DataFrame(adjmatrix)


print("attr_data.head:\n",attr_data.head(),"\n")
print("Shape of attr_data: ",attr_data.shape,"\n")
print("train_label.head:\n",train_label.head(),"\n")
print("Shape of train_label: ",train_label.shape) 
print("adj_matrix:\n",adjmatrix)<jupyter_output>attr_data.head:
    feature1  feature2  feature3  feature4  feature5  feature6
0         1         2         8        22         0     23574
1         2         2        15        14       229         0
2         1         1         3         0       237      9976
3         2         2         3         0       230     19776
4         1         2         6         0       229     17097 

Shape of attr_data:  (5298, 6) 

train_label.head:
    label
0   2009
1   2004
2   2008
3   2002
4   2007 

Shape of train_label:  (4000, 1)
adj_matrix:
       0     1     2     3     4     5     6     7     8     9     ...  5288  \
0        0     0     0     0     0     0     0     0     0     0  ...     0   
1        0     0     0     0     0     1     0     0     0     0  ...     0   
2        0     0     0     0     0     0     0     0     0     0  ...     0   
3        0     0     0     0     0     0     0     0     0     0  ...     0   
4        0     0     0     0     0     0     0     0     0  [...]<jupyter_text>3 Divide features into categorical value and continous value<jupyter_code># divide features into categorical value and continous value
categorical_val = []
continous_val = []
for column in train_data.columns:
    print('---------------------------')
    print(f"{column} : {train_data[column].unique()}")
    if len(train_data[column].unique()) <= 10:
        categorical_val.append(column)
    else:
        continous_val.append(column)
print("\ncategorical features:",categorical_val)
print("\ncontinous_val features:",continous_val)<jupyter_output>---------------------------
feature1 : [1 2 5 3 6 4]
---------------------------
feature2 : [2 1 0]
---------------------------
feature3 : [ 8 15  3  6  7 10  0 14  2 18 16  1  9 13 12 29 39  5 30 27  4 17 33 24
 25 31 44 22 11 28 34 32 26 41 20 43 21 23 37 35 42 19 40]
---------------------------
feature4 : [22 14  0 10  6  7 15 24 28  9 30  3 25 23 18 29  4 16 36 41 26  8 13  5
 17 12  1  2 31 27 34 11 19 33 37 21 43 35 20 39 44 32 40]
---------------------------
feature5 : [  0 229 237 230 286 268 283 236 227 228 274 275 256 231 255 258 278 241
 251 261 240 233 276 254 246 265 247 232 285 244 264 253 242 272 250 234
 270 257 260 259 245 249 267 248 235 262 280 284 266 243 263 288 282 269
 271 252 239 279 290 281 287 289 273 238]
---------------------------
feature6 : [23574     0  9976 ... 53700  9884 53767]

categorical features: ['feature1', 'feature2']

continous_val features: ['feature3', 'feature4', 'feature5', 'feature6']<jupyter_text>4 Dummy coding of categorical features<jupyter_code># dummy coding of categorical features
total_data = pd.concat([train_data,test_data],axis = 0)
total_dataset = pd.get_dummies(total_data, columns = categorical_val)
print(total_dataset.shape)
train_dataset = total_dataset[0:4000][:]
print(train_dataset.shape)
test_dataset = total_dataset[4000:][:]
print(test_dataset.shape)<jupyter_output>(5298, 13)
(4000, 13)
(1298, 13)<jupyter_text>5 Features scaling of continous features<jupyter_code># standardization
from sklearn.preprocessing import StandardScaler
from pandas import Series
standard_scaler = StandardScaler()
data_aft_stdsscsl = train_dataset
data_aft_stdsscsl[continous_val] = standard_scaler.fit_transform(train_dataset[continous_val])
data_aft_stdsscsl.head()
data_aft_stdsscsl = pd.concat([data_aft_stdsscsl,adjmatrix[0:4000]], axis=1)
data_aft_stdsscsl.head()
print(data_aft_stdsscsl.shape)

test_data_aft_stdsscsl = test_dataset
test_data_aft_stdsscsl[continous_val] = standard_scaler.fit_transform(test_dataset[continous_val])
tmp_adjmatrix = adjmatrix[4000:]
adjmatrix_column_id = [i for i in range(1298)]
tmp_adjmatrix.index = Series(adjmatrix_column_id)
test_data_aft_stdsscsl.head()
test_data_aft_stdsscsl = pd.concat([test_data_aft_stdsscsl,tmp_adjmatrix], axis=1)
test_data_aft_stdsscsl.head()
print(test_data_aft_stdsscsl.shape)<jupyter_output>(4000, 5311)
(1298, 5311)<jupyter_text>6 Neural network classifier and K-fold cross validation<jupyter_code>import torch
import torch.nn.functional as F
from sklearn.preprocessing import LabelEncoder
import torch.utils.data as Data
from sklearn.model_selection import StratifiedKFold,KFold

# get the number of features, it is also the number of input neurons
n_feature = data_aft_stdsscsl.shape[1]
# process training data, tensor
data_aft_stdsscsl = np.array(data_aft_stdsscsl)
data_aft_stdsscsl = torch.tensor(data_aft_stdsscsl)
data_aft_stdsscsl = data_aft_stdsscsl.to(torch.float32)
# process test data, tensor
test_data_aft_stdsscsl = np.array(test_data_aft_stdsscsl)
test_data_aft_stdsscsl = torch.tensor(test_data_aft_stdsscsl)
test_data_aft_stdsscsl = test_data_aft_stdsscsl.to(torch.float32)
# process train labels and test labels at the same time
train_label = np.array(train_label).flatten()
test_label = np.array(test_label).flatten()
le = LabelEncoder()
total_label = np.hstack((train_label, test_label))
le.fit(total_label)
label_cnt = len(le.classes_)
train_label = le.transform(train_label)
train_label = torch.tensor(train_label)
test_label = le.transform(test_label)
test_label = torch.tensor(test_label)

# K-fold cross validation
skf = StratifiedKFold(n_splits=3)
for fold,(train_id,valiation_id) in enumerate(skf.split(data_aft_stdsscsl,train_label)):
    print("fold:",fold,"------------")
    kfold_train_data = data_aft_stdsscsl[train_id]
    kfold_train_label = train_label[train_id]
    kfold_test_data = data_aft_stdsscsl[valiation_id]
    kfold_test_label = train_label[valiation_id]
    net = torch.nn.Sequential(
        torch.nn.Linear(n_feature, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, label_cnt),
    )

    # optimizer is a tool used for training
    optimizer  = torch.optim.Adam(net.parameters(), lr=0.01, betas=(0.9, 0.99))
    # when calculating errors, it is noted that the real value is not one-hot, but 1D tensor (batch,)
    # but the predicted value is 2D tensor (batch, n_ classes)
    loss_func = torch.nn.CrossEntropyLoss()

    torch.manual_seed(1)    # reproducible
    BATCH_SIZE = 64      # number of batch training data
    # first, convert it into a dataset recognized by torch
    torch_dataset = Data.TensorDataset(kfold_train_data,kfold_train_label)
    # put the dataset into the dataloader
    loader = Data.DataLoader(
        dataset=torch_dataset,      # torch TensorDataset format
        batch_size=BATCH_SIZE,      # mini batch size
        shuffle=True,               # do you want to scramble the data,yes
        num_workers=2,              # multithreading to read data
    )

    for epoch in range(8):
        for step ,(batch_x,batch_y) in enumerate(loader):
            out = net(batch_x)     # feed net training data x, output analysis value
            loss = loss_func(out, batch_y)     # calculate the error between the two
            optimizer.zero_grad()   # clear the residual update parameter value of the previous step
            loss.backward()         # the error is back propagated and the updated values of parameters are calculated
            optimizer.step()        # apply the parameter update value to the parameters of net
            if epoch % 2 == 0 and step == 0:
                prediction = torch.max(out, 1)[1]
                pred_y = prediction.data.numpy()
                target_y = batch_y.data.numpy()
                accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
                print('Epoch:',epoch,'| train accuracy: %.2f' % accuracy)

    torch.save(net.state_dict(), 'net_params.pkl')   # only save the parameters in the network (fast, less memory)

    # build a new net for predicting
    predict_net = torch.nn.Sequential(
        torch.nn.Linear(n_feature, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, 100),
        torch.nn.ReLU(),
        torch.nn.Linear(100, label_cnt),
    )
    # copy saved parameters to predict_net
    predict_net.load_state_dict(torch.load('net_params.pkl'),False)
    prediction = predict_net(kfold_test_data)
    prediction = torch.max(prediction, 1)[1]
    pred_y = prediction.data.numpy()
    target_y = kfold_test_label.data.numpy()
    accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
    print(accuracy)<jupyter_output>fold: 0 ------------
Epoch: 0 | train accuracy: 0.00
Epoch: 2 | train accuracy: 0.80
Epoch: 4 | train accuracy: 0.92
Epoch: 6 | train accuracy: 0.95
0.7923538230884558
fold: 1 ------------
Epoch: 0 | train accuracy: 0.02
Epoch: 2 | train accuracy: 0.75
Epoch: 4 | train accuracy: 0.92
Epoch: 6 | train accuracy: 0.94
0.8124531132783196
fold: 2 ------------
Epoch: 0 | train accuracy: 0.00
Epoch: 2 | train accuracy: 0.72
Epoch: 4 | train accuracy: 0.89
Epoch: 6 | train accuracy: 0.92
0.7561890472618155<jupyter_text>7 Final training and prediction  *****Note:**Now all the train data can be used to train the ensembel model, and the trained model will be used to predict the test data.***<jupyter_code>net = torch.nn.Sequential(
    torch.nn.Linear(n_feature, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, label_cnt),
)

# optimizer is a tool used for training
optimizer  = torch.optim.Adam(net.parameters(), lr=0.01, betas=(0.9, 0.99))
# when calculating errors, it is noted that the real value is not one-hot, but 1D tensor (batch,)
# but the predicted value is 2D tensor (batch, n_ classes)
loss_func = torch.nn.CrossEntropyLoss()

torch.manual_seed(1)    # reproducible
BATCH_SIZE = 64      # number of batch training data
# first, convert it into a dataset recognized by torch
torch_dataset = Data.TensorDataset(kfold_train_data,kfold_train_label)
# put the dataset into the dataloader
loader = Data.DataLoader(
    dataset=torch_dataset,      # torch TensorDataset format
    batch_size=BATCH_SIZE,      # mini batch size
    shuffle=True,               # do you want to scramble the data,yes
    num_workers=2,              # multithreading to read data
)

for epoch in range(8):
    for step ,(batch_x,batch_y) in enumerate(loader):
        out = net(batch_x)     # feed net training data x, output analysis value
        loss = loss_func(out, batch_y)     # calculate the error between the two
        optimizer.zero_grad()   # clear the residual update parameter value of the previous step
        loss.backward()         # the error is back propagated and the updated values of parameters are calculated
        optimizer.step()        # apply the parameter update value to the parameters of net
        if epoch % 2 == 0 and step == 0:
            prediction = torch.max(out, 1)[1]
            pred_y = prediction.data.numpy()
            target_y = batch_y.data.numpy()
            accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
            print('Epoch:',epoch,'| train accuracy: %.2f' % accuracy)

torch.save(net.state_dict(), 'net_params.pkl')   # only save the parameters in the network (fast, less memory)

# build a new net for predicting
predict_net = torch.nn.Sequential(
    torch.nn.Linear(n_feature, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, label_cnt),
)
# copy saved parameters to predict_net
predict_net.load_state_dict(torch.load('final_net_params.pkl'),False)
prediction = predict_net(test_data_aft_stdsscsl)
prediction = torch.max(prediction, 1)[1]
pred_y = prediction.data.numpy()
target_y = test_label.data.numpy()
real_pred_y = le.inverse_transform(pred_y)
print("predicted label: ",real_pred_y)
accuracy = float((pred_y == target_y).astype(int).sum()) / float(target_y.size)
print("Accuracy on test set: ",accuracy*100,"%")
np.savetxt("predict_testlabel_with_adjlist.txt",real_pred_y)<jupyter_output>Epoch: 0 | train accuracy: 0.00
Epoch: 2 | train accuracy: 0.72
Epoch: 4 | train accuracy: 0.89
Epoch: 6 | train accuracy: 0.92
predicted label:  [2005 2008 2006 ... 2007 2008 2005]
Accuracy on test set:  80.12326656394453 %<jupyter_text>8 Disscuss the performance of this classifier<jupyter_code>from sklearn.metrics import precision_score,f1_score
micro_precision = precision_score(target_y, pred_y, average="micro")
macro_precision = precision_score(target_y, pred_y, average="macro")
weighted_precision = precision_score(target_y, pred_y, average="weighted")
micro_f1_score = f1_score(target_y, pred_y, average='micro')
print("micro_precision:", micro_precision)
print("macro_precision:", macro_precision)
print("weighted_precision:", weighted_precision)
print("micro_f1_score:", micro_f1_score)<jupyter_output>micro_precision: 0.8012326656394453
macro_precision: 0.299221153063926
weighted_precision: 0.7862371149684687
micro_f1_score: 0.8012326656394453
{'fraction_non_alphanumeric': 0.11991122013901057, 'fraction_numerical': 0.0746451725950587, 'mean_word_length': 2.8993395581872012, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 43, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '549757', 'license': 'Apache-2.0', 'n_tokens_mistral': 6585, 'n_tokens_neox': 5711, 'n_words': 1535, 'path': 'clf_with_adjlist.ipynb', 'repo_name': 'RadarSun/SUSTech-CSE5002-Project', 'size': '17126'}