<jupyter_start><jupyter_text>Exploratory Data Analysis  Import libraries<jupyter_code>import pandas as pd
import numpy as np
import fasttext
import fasttext.util
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>df = pd.read_csv('Train.csv', sep=';')
df.head()
len(df)
imbalance = df.groupby(by='rate').name.count()
imbalance = np.ceil(21/np.ceil(((imbalance/imbalance.min())*2)))
df['op_len'] = df['opinion'].str.len()<jupyter_output><empty_output><jupyter_text>Data cleaning<jupyter_code>from html import unescape<jupyter_output><empty_output><jupyter_text>Remove html escaping<jupyter_code>df.update(df[df['opinion'].str.contains('&')]['opinion'].apply(unescape))
df.opinion.head()<jupyter_output><empty_output><jupyter_text>remove trailing "<jupyter_code>df['opinion'] = df['opinion'].str[1:-1]
df.opinion.head()
df['opinion'] = df['opinion'].str.lower()
df.opinion.head()
df = df[df['opinion'].str.len() > 2]
df['name'] = df['name'].str.lower()
df['condition'] = df['condition'].str.lower().dropna()
df = df.sample(frac=1.0)
# Clean the data up
import re
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;:]')
BAD_SYMBOLS_RE = re.compile('[^a-z #+_]')
UTF_CODES = re.compile('#\d+')

def clean_text(text):
    """
        text: a string
        
        return: modified initial string
    """
    text = text.lower() # lowercase text
    text = REPLACE_BY_SPACE_RE.sub(' ', text) # replace REPLACE_BY_SPACE_RE symbols by space in text. substitute the matched string in REPLACE_BY_SPACE_RE with space.
    text = BAD_SYMBOLS_RE.sub('', text) # remove symbols which are in BAD_SYMBOLS_RE from text. substitute the matched string in BAD_SYMBOLS_RE with nothing. 
    text = UTF_CODES.sub('', text)
    # We do not remove stop words for LSTM because having a 'not' in a sentence is actually kind of useful
    #text = ' '.join(word for word in text.split() if word not in STOPWORDS) # remove stopwors from text
    return text
df['opinion'] = df['opinion'].apply(clean_text)
df['condition'] = df['condition'].astype('string').dropna().apply(clean_text)
df['name'] = df['name'].astype('string').dropna().apply(clean_text)
train, test = train_test_split(df, test_size=0.1)

import csv
import re

location_train_fasttext = "opinion.train.ft" #will be created
location_test_fasttext = "opinion.test.ft" #will be created

#creates Vowpal Wabbit-formatted file from tsv file
def to_fasttext(df,location_output_file, test = False):
    with open(location_output_file, "w") as outfile:
    #create a reader to read train file
        #for every line
        for i, row in df.iterrows():
            #if test set label doesnt matter/or isnt available
            label = str(row['rate'])
            imb = int(imbalance[row['rate']])
            phrase = row['opinion']
            outfile.write('__label__'+label+' '+phrase + "\n" )<jupyter_output><empty_output><jupyter_text>65.8%<jupyter_code>model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=25, wordNgrams=2)
model.test(location_test_fasttext)<jupyter_output><empty_output><jupyter_text>71.3%<jupyter_code>model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=25, wordNgrams=5)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=25, wordNgrams=7)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.5, epoch=25, wordNgrams=7)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.1, epoch=30, wordNgrams=7)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.3, epoch=30, wordNgrams=5)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.3, epoch=30, wordNgrams=5, neg=10)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.01, epoch=50, wordNgrams=5)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=0.01, epoch=200, wordNgrams=5)
model.test(location_test_fasttext)
model.test(location_test_fasttext)<jupyter_output><empty_output><jupyter_text>BEST MODEL 80%<jupyter_code>model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=25, wordNgrams=5, ws=7, dim=300)
model.test(location_test_fasttext)
model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=20, wordNgrams=4, ws=7, dim=300)
model.test(location_test_fasttext)
preds = test['opinion'].apply(model.predict)
c = 0
for pred, rate in zip(preds, test['rate']):
    if int(pred[0][0][-1:]) == int(rate):
        c+=1
    elif int(pred[0][0][-1:])==0 and int(rate)==10:
        c+=1
print(c/len(test))<jupyter_output>0.7111807453830256<jupyter_text>K-Fold CV<jupyter_code>from sklearn.model_selection import KFold
kf = KFold(n_splits=10)
for train_index, test_index in kf.split(df['opinion']):
    train = df.iloc[train_index, 2:4]
    test = df.iloc[test_index, 2:4]
    to_fasttext(train, location_train_fasttext)
    to_fasttext(test, location_test_fasttext, test=True)
    model = fasttext.train_supervised(input=location_train_fasttext, lr=1.0, epoch=25, wordNgrams=5, ws=7, dim=300)
    model.test(location_test_fasttext)
    c = 0
    preds = test['opinion'].apply(model.predict)
    for pred, rate in zip(preds, test['rate']):
        if int(pred[0][0][-1:]) == int(rate):
            c+=1
        elif int(pred[0][0][-1:])==0 and int(rate)==10:
            c+=1
    print(c/len(test))
results = pd.read_csv('opinion.preds.txt', names=['pred', 'row'], sep=' ')
results = results.set_index('row')
results
original = pd.read_csv('opinion.test.vw', sep = '|', header=None)
original = original.iloc[:,0]
original_rate = original.apply(lambda x: int(x.split(' ')[0]))
original_index = original.apply(lambda x: int(x.split(' ')[2][1:]))
original = pd.DataFrame()
original['index'] = original_index
original['rate'] = original_rate
original = original.set_index('index')['rate']
original<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1514267185473411, 'fraction_numerical': 0.023670557717250323, 'mean_word_length': 6.042237442922374, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '201429', 'license': 'MIT', 'n_tokens_mistral': 2343, 'n_tokens_neox': 2225, 'n_words': 403, 'path': 'log_reg copy 3.ipynb', 'repo_name': 'CrimsonRadiator/ZMUM', 'size': '6168'}