<jupyter_start><jupyter_text>Activity 2.01<jupyter_code>import pandas as pd<jupyter_output><empty_output><jupyter_text>Dealing with Messy Data<jupyter_code>data = pd.read_csv("YearPredictionMSD.csv", nrows=50000)
data.head()
cols = data.columns

num_cols = data._get_numeric_data().columns

list(set(cols) - set(num_cols))
data.isnull().sum().sum()
outliers = {}
for i in range(data.shape[1]):
    min_t = data[data.columns[i]].mean() - (3 * data[data.columns[i]].std())
    max_t = data[data.columns[i]].mean() + (3 * data[data.columns[i]].std())
    count = 0
    for j in data[data.columns[i]]:
        if j < min_t or j > max_t:
            count += 1
    percentage = count/data.shape[0]
    outliers[data.columns[i]] = "%.3f" % percentage

print(outliers)<jupyter_output>{'0': '0.019', '1': '0.010', '2': '0.011', '3': '0.011', '4': '0.015', '5': '0.008', '6': '0.010', '7': '0.010', '8': '0.010', '9': '0.011', '10': '0.005', '11': '0.010', '12': '0.010', '13': '0.015', '14': '0.017', '15': '0.016', '16': '0.016', '17': '0.014', '18': '0.016', '19': '0.013', '20': '0.016', '21': '0.015', '22': '0.013', '23': '0.015', '24': '0.013', '25': '0.018', '26': '0.017', '27': '0.017', '28': '0.018', '29': '0.017', '30': '0.016', '31': '0.015', '32': '0.017', '33': '0.016', '34': '0.015', '35': '0.016', '36': '0.017', '37': '0.017', '38': '0.016', '39': '0.015', '40': '0.017', '41': '0.017', '42': '0.015', '43': '0.014', '44': '0.016', '45': '0.015', '46': '0.017', '47': '0.017', '48': '0.017', '49': '0.015', '50': '0.016', '51': '0.015', '52': '0.015', '53': '0.015', '54': '0.016', '55': '0.018', '56': '0.017', '57': '0.016', '58': '0.013', '59': '0.016', '60': '0.016', '61': '0.015', '62': '0.016', '63': '0.017', '64': '0.017', '65': '0.016', '66': '0.018', '67'[...]<jupyter_text>Data Rescaling<jupyter_code>X = data.iloc[:,1:]
Y = data.iloc[:,0]
X = (X - X.mean())/X.std()
X.head()<jupyter_output><empty_output><jupyter_text>Splitting the Dataset<jupyter_code>from sklearn.model_selection import train_test_split
X_shuffle = X.sample(frac=1, random_state=0)
Y_shuffle = Y.sample(frac=1, random_state=0)
x_new, x_test, y_new, y_test = train_test_split(X_shuffle, Y_shuffle, test_size=0.2, random_state=0)
dev_per = x_test.shape[0]/x_new.shape[0]
x_train, x_dev, y_train, y_dev = train_test_split(x_new, y_new, test_size=dev_per, random_state=0)
print(x_train.shape, y_train.shape)
print(x_dev.shape, y_dev.shape)
print(x_test.shape, y_test.shape)<jupyter_output>(30000, 90) (30000,)
(10000, 90) (10000,)
(10000, 90) (10000,)<jupyter_text>Activity 2.02<jupyter_code>import torch
import torch.nn as nn
torch.manual_seed(0)
x_train = torch.tensor(x_train.values).float()
y_train = torch.tensor(y_train.values).float()

x_dev = torch.tensor(x_dev.values).float()
y_dev = torch.tensor(y_dev.values).float()

x_test = torch.tensor(x_test.values).float()
y_test = torch.tensor(y_test.values).float()
model = nn.Sequential(nn.Linear(x_train.shape[1],10),
                      nn.ReLU(),
                      
                      nn.Linear(10,7),
                      nn.ReLU(),
                      
                      nn.Linear(7,5),
                      nn.ReLU(),
                     
                      nn.Linear(5,1))
loss_function = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
for i in range(3000):
    y_pred = model(x_train).squeeze()
    loss = loss_function(y_pred, y_train)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    if i%250 == 0:
        print(i, loss.item())
pred = model(x_test[0])
print("Ground truth:", y_test[0].item(), "Prediction:", pred.item())<jupyter_output>Ground truth: 1995.0 Prediction: 1998.0279541015625
{'fraction_non_alphanumeric': 0.24241610738255034, 'fraction_numerical': 0.13583892617449664, 'mean_word_length': 3.722433460076046, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '405456', 'license': 'MIT', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1584, 'n_words': 296, 'path': 'Chapter02/Activity2.02/Activity2.02.ipynb', 'repo_name': 'thoolihan/The-Deep-Learning-with-PyTorch-Workshop', 'size': '3725'}