<jupyter_start><jupyter_text>Read data<jupyter_code>INPUT_PATH = 'data/input'
FILENAME = 'tripadvisor_hotel_reviews.csv'
df = pd.read_csv(f'{INPUT_PATH}/{FILENAME}')
print(df.shape)
df.sample(5)
df['Review'].str.split().apply(lambda x: len(x)).describe()
df['Review'].str.split().apply(lambda x: len(x)).hist()
plt.xlabel('Review length')
plt.ylabel('Count reviews')
plt.savefig('img/words_hist.jpg', dpi = 600)
df['Rating'].value_counts().plot.pie(y='Rating', figsize=(5, 5), autopct='%1.0f%%')
plt.savefig('img/pie.jpg', dpi = 600)<jupyter_output><empty_output><jupyter_text>Preprocess<jupyter_code>df['Review'] = df['Review'].str.replace('[^\w\s]','') # remove punctuation marks
df['Review'] =  df['Review'].str.replace('\d+', '') # remove numbers<jupyter_output><empty_output><jupyter_text>Train/validation/test split<jupyter_code>test_size = 0.1
X, y = df['Review'], df['Rating']
X_tr, X_test, y_tr, y_test = train_test_split(X, y, test_size = test_size, stratify = y, shuffle=True)
X_train, X_val, y_train, y_val = train_test_split(X_tr, y_tr, test_size = test_size * (1-test_size), stratify = y_tr, shuffle=True)<jupyter_output><empty_output><jupyter_text>Tokenization<jupyter_code>def tokenize(encoder, X, padding = 2000):
    encoded_X = [encoder.encode(x) for x in X]
    encoded_X = [pad_tensor(x, length=padding) for x in encoded_X]
    return stack_and_pad_tensors(encoded_X).tensor
    
encoder = StaticTokenizerEncoder(X_train.values, tokenize=lambda s: s.split())
X_train = tokenize(encoder, X_train.values)
X_val = tokenize(encoder, X_val.values)
X_test = tokenize(encoder, X_test.values)
encoder.vocab_size
X_train.size(1)<jupyter_output><empty_output><jupyter_text>Create dataloaders for torch model<jupyter_code>train_dataset = TensorDataset(X_train, torch.tensor(y_train.values))
train_loader = DataLoader(train_dataset,shuffle=True,batch_size=BATCH_SIZE)

val_dataset = TensorDataset(X_val, torch.tensor(y_val.values))
val_loader = DataLoader(val_dataset,shuffle=False,batch_size=BATCH_SIZE)

test_dataset = TensorDataset(X_test)
test_loader = DataLoader(test_dataset,shuffle=False,batch_size=BATCH_SIZE,)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>vocab_size = encoder.vocab_size
embedding_size = 100
seq_len = 2000
rnn_size = 32
hidden_size = 16
output_size = 1

class ReviewsModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.emb = nn.Embedding(vocab_size, embedding_size)
        self.rnn = nn.LSTM(embedding_size, rnn_size)
        self.drop1 = nn.Dropout(0.5)
        self.norm1 = nn.BatchNorm1d(seq_len*rnn_size)
        self.fc1 = nn.Linear(seq_len*rnn_size, hidden_size)
        self.drop2 = nn.Dropout(0.5)
        self.norm2 = nn.BatchNorm1d(hidden_size)
        self.fc2 = nn.Linear(hidden_size, output_size)
        self.out = nn.ReLU()
        

    def forward(self, inputs):
        batch_size = inputs.size(0)
        emb = self.emb(inputs)
        
        self.rnn.flatten_parameters()
        rnn, hid = self.rnn(emb)
        rnn = self.drop1(rnn)
        a1 = self.fc1(self.norm1(rnn.view(batch_size, -1)))
        a1 = self.drop2(a1)
        a2 = self.fc2(self.norm2(a1))
        return self.out(a2).float().squeeze()<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>def round_output(y, thresh = [1.5, 2.5, 3.5, 4.5], labels = [1, 2, 3, 4, 5]):
    return pd.cut(y, bins = [-np.inf, *thresh, np.inf], labels = labels).astype(int)
def fit_epoch(model, train_loader, criterion, optimizer):
    model.train()
    running_loss = .0
    running_kappa = .0
    processed_data = 0
    
    for inputs, labels in train_loader:
        inputs = inputs.to(DEVICE)
        labels = labels.to(DEVICE)
        optimizer.zero_grad()
        
        outputs = model(inputs)
        y_pred = round_output(outputs.data.cpu().numpy())

        loss = criterion(outputs, labels.float())
        kappa = cohen_kappa_score(y_pred, labels.data.cpu().numpy(), weights='quadratic')
        
        loss.backward()
        optimizer.step()
        
        running_loss += loss.item() * inputs.size(0)
        running_kappa += kappa * inputs.size(0)
        processed_data += inputs.size(0)
        
        
    train_loss = running_loss / processed_data
    train_kappa = running_kappa / processed_data
    return train_loss, train_kappa

def eval_epoch(model, val_loader, criterion):
    model.eval()
    running_loss = .0
    running_kappa = .0
    processed_data = 0
    for inputs, labels in val_loader:
        inputs = inputs.to(DEVICE)
        labels = labels.to(DEVICE)
        
        with torch.set_grad_enabled(False):
            outputs = model(inputs)
            y_pred = round_output(outputs.data.cpu().numpy())
            loss = criterion(outputs, labels.float())
            kappa = cohen_kappa_score(y_pred, labels.data.cpu().numpy(), weights='quadratic')

        
        running_loss += loss.item() * inputs.size(0)
        running_kappa += kappa * inputs.size(0)
        processed_data += inputs.size(0)
        
        
    val_loss = running_loss / processed_data
    val_kappa = running_kappa / processed_data
    return val_loss, val_kappa
model = ReviewsModel().to(DEVICE)
print(model)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), LEARN_RATE, weight_decay=1e-2)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max')
verbose =1

log_template = "\n[{ep:03d}/{epochs:03d}] train_loss: {t_loss:0.4f} train_kappa: {t_kappa:.4f} \
val_loss {v_loss:0.4f} val_kappa: {v_kappa:.4f}"
history = []

#early stopping params
kappa_best = -1
stop_counter = 0
patience = 15
delta = 0.005

for epoch in range(EPOCHS):
    train_loss, train_kappa = fit_epoch(model, train_loader, criterion, optimizer)
    val_loss, val_kappa = eval_epoch(model, val_loader, criterion)
    scheduler.step(val_kappa)
    
    history.append([train_loss, train_kappa, val_loss, val_kappa])

    log = log_template.format(ep=epoch+1,epochs =EPOCHS, t_loss=train_loss, t_kappa=train_kappa,\
                                   v_loss=val_loss, v_kappa= val_kappa)
    
    # early stopping
    if val_kappa > kappa_best * (1+delta):
        kappa_best = val_kappa
        stop_counter = 0
        # best model
        best_model = copy.deepcopy(model)
    else:
        stop_counter += 1
        if stop_counter >= patience:
            print('EARLY STOP.')
            print(log)
            break
    
    # verbose
    if epoch == 0 or (epoch+1)% verbose == 0 or epoch==EPOCHS-1:
        print(log)
history = list(zip(*history))
def plot_metric(train, val, metric_name, save=True):
    plt.plot(train, label = f'train {metric_name}')
    plt.plot(val, label = f'val {metric_name}')
    plt.xticks(range(len(history[0])))
    plt.ylim(bottom=0)
    plt.legend()
    if save:
        plt.savefig(f'img/{metric_name}.jpg', dpi=600)
    plt.show()
    plt.close()
plot_metric(history[0], history[2], 'loss')
plot_metric(history[1], history[3], 'kappa')
model = copy.deepcopy(best_model)
del best_model<jupyter_output><empty_output><jupyter_text>Evalute model<jupyter_code>def predict(model, test_loader):
    model.eval()
    logits = []
    with torch.no_grad():
        for data in test_loader:
            inputs = data[0]
            inputs = inputs.to(DEVICE)
            outputs = model(inputs).cpu()
            logits.append(outputs)
    return torch.cat(logits, 0).numpy()
train_dataset = TensorDataset(X_train, torch.tensor(y_train.values))
train_loader = DataLoader(train_dataset,shuffle=False,batch_size=BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Scores without optimized thresholds<jupyter_code>y_train_hat = predict(model, train_loader)

y_val_hat = predict(model, val_loader)

y_test_hat = predict(model, test_loader)
print('Quadratic Weighted Kappa')
print('Train: ', cohen_kappa_score(
    round_output(y_train_hat), y_train, weights = 'quadratic'))

print('Validation: ', cohen_kappa_score(
    round_output(y_val_hat), y_val, weights = 'quadratic'))

print('Test: ', cohen_kappa_score(
    round_output(y_test_hat), y_test, weights = 'quadratic'))<jupyter_output>Quadratic Weighted Kappa
Train:  0.7935055220295901
Validation:  0.7363655418887651
Test:  0.7475972431781999<jupyter_text>Scores with optimized thresholds<jupyter_code>class OptimizedRounder(object):
    def __init__(self):
        self.coef_ = 0
    
    def _kappa_loss(self, coef, y_hat, y):
        preds = pd.cut(y_hat, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [1, 2, 3, 4, 5]).astype(int)
        return -cohen_kappa_score(y, preds, weights = 'quadratic')
    
    def fit(self, y_hat, y, initial_coef = [1.5, 2.5, 3.5, 4.5]):
        loss_partial = partial(self._kappa_loss, y_hat = y_hat, y = y)
        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method = 'nelder-mead')['x']
    
    def predict(self, y_hat, coef):
        preds = pd.cut(y_hat, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [1, 2, 3, 4, 5]).astype(int)
        return preds
rounder = OptimizedRounder()
rounder.fit(y_train_hat, y_train)
coefs = rounder.coef_
coefs
print('Quadratic Weighted Kappa')
print('Train: ', cohen_kappa_score(
    rounder.predict(y_train_hat, coefs), y_train, weights = 'quadratic'))

print('Validation: ', cohen_kappa_score(
    rounder.predict(y_val_hat, coefs), y_val, weights = 'quadratic'))

print('Test: ', cohen_kappa_score(
    rounder.predict(y_test_hat, coefs), y_test, weights = 'quadratic'))
labels=[1,2,3,4,5]
sns.heatmap(confusion_matrix(rounder.predict(y_test_hat, coefs), y_test, normalize='true'), xticklabels=labels, yticklabels = labels, annot=True, cmap = 'YlGn')
plt.ylabel('True')
plt.xlabel('Predicted')
plt.title('Confusion matrix (normalized by "True")', fontsize=18)
plt.tight_layout()
plt.savefig('img/conf_matrix.jpg', dpi = 600)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13726285888201278, 'fraction_numerical': 0.020493050623922086, 'mean_word_length': 3.9838220424671387, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '607251', 'license': 'MIT', 'n_tokens_mistral': 3730, 'n_tokens_neox': 3566, 'n_words': 620, 'path': 'project2/Hotel Reviews Rating Model.ipynb', 'repo_name': 'darynarr/ML_UKMA', 'size': '9857'}