<jupyter_start><jupyter_text>PyTorch Version<jupyter_code>import torch
print(torch.__version__)
# move to HPC, pytorch=1.6.0<jupyter_output>0.4.0<jupyter_text>Import data<jupyter_code>train_df = pd.read_csv('/Users/wh1165/Desktop/AIcrowd_Competition/NLP/input/train_set.csv', sep='\t')
test_df = pd.read_csv('/Users/wh1165/Desktop/AIcrowd_Competition/NLP/input/test_a.csv', sep='\t', nrows=None)<jupyter_output><empty_output><jupyter_text>Split Data  devided into train set and validation set<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_val, y_train, y_val = train_test_split(
    train_df['text'].iloc[:].values, train_df['label'].iloc[:].values,
    test_size=0.05, random_state=42
)<jupyter_output><empty_output><jupyter_text>one-hot encoding for labels （Not needed for MLP）<jupyter_code># from sklearn.preprocessing import OneHotEncoder

# n=y_train.shape[0]
# m=y_val.shape[0]

# train_label = y_train.reshape(n,1)
# val_label = y_val.reshape(m,1)

# ohe = OneHotEncoder()
# ohe.fit([[0],[1],[2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13]])

# train_label_oh = ohe.transform(train_label).toarray()
# val_label_oh = ohe.transform(val_label).toarray()
# one-hot, another method
# torch.nn.functional.one_hot(x)<jupyter_output><empty_output><jupyter_text>convert one-hot output to standard output<jupyter_code># 例如
# int(np.matrix([0,1,0,0])*np.matrix([1,2,3,4]).T)<jupyter_output><empty_output><jupyter_text>Featurization<jupyter_code>vectorizer = CountVectorizer(ngram_range=(1,3), max_df=0.9, max_features=20000)
vectorizer.fit(train_df['text'].iloc[:].values)

X_train_vec = vectorizer.transform(X_train)
X_val_vec = vectorizer.transform(X_val)
X_test_vec = vectorizer.transform(test_df['text'].iloc[:].values)
print(X_train_vec.shape)
print(X_val_vec.shape)
print(X_test_vec.shape)<jupyter_output>(190000, 20000)
(10000, 20000)
(50000, 20000)<jupyter_text>Shuffle, can be done by PyTorh DataLoader<jupyter_code># shuffle_index = np.random.permutation(trainset.shape[0])   #shuffle the data
# train_s, label_s = trainset[shuffle_index], train_df['label'].iloc[:].values[shuffle_index]<jupyter_output><empty_output><jupyter_text>CountVectorizer to Dense matrix.<jupyter_code># CountVectorizer has given sparse matrix,
X_train_tensor = torch.from_numpy(X_train_vec.todense()).float()
X_val_tensor = torch.from_numpy(X_val_vec.todense()).float()
X_test_tensor = torch.from_numpy(X_test_vec.todense()).float()

# y_train_tensor = train_label_oh
# y_val_tensor = val_label_oh

y_train_tensor = torch.from_numpy(np.array(y_train))
y_val_tensor = torch.from_numpy(np.array(y_val))

# X_train, y_train, X_val, y_val = map(
#     torch.tensor, (X_train, y_train, X_val, y_val)
# )<jupyter_output><empty_output><jupyter_text>Build deep MLP model<jupyter_code>from torch import nn
import torch.nn.functional as F

loss_func = F.cross_entropy ## specify loss function

class MLP(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(20000, 500) ## n_hidden1
        self.fc2 = nn.Linear(500, 100) ## n_hidden2
        self.fc3 = nn.Linear(100, 14)

    def forward(self, x):
        x = x.view(-1, 20000)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        return self.fc3(x)
def accuracy(out, y_real):
    y_pred = torch.argmax(out, dim=1)
    return (y_pred == y_real).float().mean()
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
from torch import optim

lr = 0.01  # learning rate
epochs = 20  # how many epochs to train for
bs = 50

# Use DataLoader class for batching but it requires TensorDataset
train_ds = TensorDataset(X_train_tensor, y_train_tensor)
#create batches
train_dl = DataLoader(train_ds, batch_size=bs, shuffle=True)

def get_model():
    model = MLP()
    return model, optim.SGD(model.parameters(), lr=lr)
valid_ds = TensorDataset(X_val_tensor, y_val_tensor)
valid_dl = DataLoader(valid_ds, batch_size=bs)<jupyter_output><empty_output><jupyter_text>Check the random Model<jupyter_code>model = MLP()
print(loss_func(model(X_train_tensor), y_train_tensor), accuracy(model(X_train_tensor), y_train_tensor))
print(loss_func(model(X_val_tensor), y_val_tensor), accuracy(model(X_val_tensor), y_val_tensor))<jupyter_output>tensor(2.6316) tensor(1.00000e-02 *
       5.3200)<jupyter_text>Define EarlyStopping<jupyter_code>class EarlyStopping:
    """Early stops the training if validation loss doesn't improve after a given patience."""
    def __init__(self, patience=7, verbose=False):
        """
        Args:
            patience (int): How long to wait after last time validation loss improved.
                            Default: 7
            verbose (bool): If True, prints a message for each validation loss improvement. 
                            Default: False
        """
        self.patience = patience
        self.verbose = verbose
        self.counter = 0
        self.best_score = None
        self.early_stop = False
        self.val_loss_min = np.Inf

    def __call__(self, val_loss, model):

        score = -val_loss

        if self.best_score is None:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
        elif score < self.best_score:
            self.counter += 1
            print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
            self.counter = 0
    
    def save_checkpoint(self, val_loss, model):
        '''Saves model when validation loss decrease.'''
        if self.verbose:
            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(self.val_loss_min,
                                                                                            val_loss))
        torch.save(model.state_dict(), '/Users/wh1165/Desktop/AIcrowd_Competition/NLP/model/MLP1_checkpoint.pt')
        self.val_loss_min = val_loss<jupyter_output><empty_output><jupyter_text>Train the MLP model<jupyter_code># to track the training loss as the model trains
train_losses = []
# to track the validation loss as the model trains
valid_losses = []
# to track the average training loss per epoch as the model trains
avg_train_losses = []
# to track the average validation loss per epoch as the model trains
avg_valid_losses = []

# initialize the early_stopping object
early_stopping = EarlyStopping(patience=30, verbose=True)

model, opt = get_model()
epochs = 10001

for epoch in range(epochs):
    model.train() 
    for batch, (xb, yb) in enumerate(train_dl, 1):
        pred = model(xb)
        loss = loss_func(pred, yb)
        loss.backward()
        opt.step()
        opt.zero_grad()
        train_losses.append(loss.item())
    
    model.eval()
    for xb, yb in valid_dl:
        pred = model(xb)
        loss = loss_func(pred, yb)
        valid_losses.append(loss.item())
        
    train_loss = np.average(train_losses)
    valid_loss = np.average(valid_losses)
    avg_train_losses.append(train_loss)
    avg_valid_losses.append(valid_loss)
    
    print_msg = ('epoch:{} '.format(epoch+1) +
                 'train_loss: {:.5f} '.format(train_loss) +
                 'valid_loss: {:.5f}'.format(valid_loss))
    print(print_msg)
    # clear lists to track next epoch
    train_losses = []
    valid_losses = []
    early_stopping(valid_loss, model)
    
    if early_stopping.early_stop:
        print("Early stopping")
        break
    # load the last checkpoint with the best model
    model.load_state_dict(torch.load('checkpoint.pt'))<jupyter_output><empty_output><jupyter_text>Visualize the loss and earlystop<jupyter_code># visualize the loss as the network trained
fig = plt.figure(figsize=(10,8))
plt.plot(range(1,len(avg_train_losses)+1),avg_train_losses, label='Training Loss')
plt.plot(range(1,len(avg_valid_losses)+1),avg_valid_losses,label='Validation Loss')

# find position of lowest validation loss
minposs = avg_valid_losses.index(min(avg_valid_losses))+1 
plt.axvline(minposs, linestyle='--', color='r',label='Early Stopping Checkpoint')

plt.xlabel('epochs')
plt.ylabel('loss')
#plt.ylim(0, 0.5) # consistent scale
plt.xlim(0, len(avg_train_losses)+1) # consistent scale
plt.grid(True)
plt.legend()
plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Test performace<jupyter_code>model.eval()
print('Test Accuracy: ', accuracy(model(X_val_tensor), y_val_tensor))<jupyter_output><empty_output><jupyter_text>Output<jupyter_code>output_test = model(X_test_tensor)<jupyter_output><empty_output><jupyter_text>find the best<jupyter_code>f = output_test.detach().numpy()
ff = argmax(f,axis=1)
df = pd.DataFrame()
df['label'] = ff
df.to_csv('/Users/wh1165/Desktop/AIcrowd_Competition/NLP/input/submit_MLP1.csv', index=None)<jupyter_output><empty_output><jupyter_text>Save Model<jupyter_code>torch.save(model, '/Users/wh1165/Desktop/AIcrowd_Competition/NLP/model/MLP1.pt')
#model = torch.load('model.pt')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12691633815155498, 'fraction_numerical': 0.02146298729741568, 'mean_word_length': 4.257915947035118, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '555745', 'license': 'MIT', 'n_tokens_mistral': 3392, 'n_tokens_neox': 3226, 'n_words': 582, 'path': 'NLP_MLP1_demoRun.ipynb', 'repo_name': 'Warrenheww/rank6_NLP_newstextclassification', 'size': '9132'}