<jupyter_start><jupyter_text>Requirements<jupyter_code>!pip install catboost --quiet<jupyter_output>[K     |████████████████████████████████| 64.8MB 54kB/s 
[?25h<jupyter_text>Imports<jupyter_code>import warnings
warnings.simplefilter('ignore')
import os
import sys
import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm.notebook import tqdm
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression, LinearRegression
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import TruncatedSVD
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from collections import Counter
import catboost as cat, lightgbm as lgb, xgboost as xgb<jupyter_output><empty_output><jupyter_text>Envs<jupyter_code>seed = 2020 # for reproductibility
random.seed(seed)
np.random.seed(seed)<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>def process_prediction(preds):
  final_preds = []
  for pred in preds:
    argmax = pred.argmax()
    if argmax == 0: final_preds.append( -1*pred[0] )
    elif argmax == 1: final_preds.append( 0 )
    else: final_preds.append( pred[2] )
    
  return final_preds


def rmse(true, pred):
  return np.sqrt(mean_squared_error(true, pred))<jupyter_output><empty_output><jupyter_text>Reading the Data<jupyter_code>path = 'drive/My Drive/Zindi/#ZindiWeekendz/'
train = pd.read_csv(path+'Train.csv')
test = pd.read_csv(path+'Test.csv')
sample = pd.read_csv(path+'SampleSubmission.csv')<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>train.head(10)
test.head()
train['label'].value_counts()
train[train['label'].round(2)==0.67]
train[train['label'].isna()]
train[train['agreement'].isna()]<jupyter_output><empty_output><jupyter_text>Features Engeneering Part 1<jupyter_code>train.drop(index=[4798, 4799], inplace=True)
train.reset_index(drop=True, inplace=True)
train['safe_text'] = train['safe_text'].apply(str)
test['safe_text'] = test['safe_text'].apply(str)
train['safe_text'] = train['safe_text'].apply(str.lower)
test['safe_text'] = test['safe_text'].apply(str.lower)
train['safe_text'] = train['safe_text'].apply(lambda x: x.replace('&amp;', ''))
test['safe_text'] = test['safe_text'].apply(lambda x: x.replace('&amp;', ''))
train['safe_text'] = train['safe_text'].apply(lambda x: x.strip().strip('.'))
test['safe_text'] = test['safe_text'].apply(lambda x: x.strip().strip('.'))
train.head(2)<jupyter_output><empty_output><jupyter_text>Part 2<jupyter_code>def clean_stop_words(df):
  stop_words = set(stopwords.words('english'))
  #df['text_clean'] = df['safe_text'].apply(lambda s: " ".join([w for w in s.lower().split() if w not in stop_words and not w.startswith('<')]))
  df['text_clean'] = df['safe_text'].apply(lambda s: " ".join([w for w in s.lower().split() if not w.startswith('<')]))
  df['text_clean'] = df['text_clean'].apply(lambda s: " ".join([w for w in s.lower().split() if not w.startswith('#')]))
df = pd.concat([train, test], axis = 0)
clean_stop_words(df)
train = df[~df.label.isnull()]
test = df[df.label.isnull()]
train['label'] += 1
xtrain = train['text_clean'].values
ytrain = train['label'].values

xtest = test['text_clean'].values
tvect = TfidfVectorizer(min_df=3, max_features=None, 
            strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
            ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,
            stop_words = 'english')

cvect = CountVectorizer(analyzer='word',token_pattern=r'\w{1,}',
            ngram_range=(1, 3), stop_words = 'english')
tvect.fit(df['text_clean'].values.tolist())
cvect.fit(df['text_clean'].values.tolist())
xtrain_tv = tvect.transform(xtrain)
xtest_tv = tvect.transform(xtest)
xtrain_cv = cvect.transform(xtrain)
xtest_cv = cvect.transform(xtest)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>n_fold = 10
fold = StratifiedKFold(n_fold, shuffle=True, random_state=seed)
avg = 0
test_oofs = []

for i, (tr,vr) in enumerate(fold.split(xtrain_tv, ytrain)):
  X,Y = xtrain_tv[tr], np.take(ytrain, tr, axis=0)
  x,y = xtrain_tv[vr], np.take(ytrain, vr, axis=0)

  model = LogisticRegression(C=1.1, max_iter=1000, random_state=seed, n_jobs=-1)
  model.fit(X, Y)
  test_pred = model.predict_proba(xtest_tv)
  pred = model.predict_proba(x)
  pred = process_prediction(pred)

  test_oofs.append(test_pred)

  score = rmse(y-1, pred)
  avg += score
  print(f"Fold {i}: ", score)

print("Avg score : {:.3f}".format(avg/n_fold))<jupyter_output>Fold 0:  0.5841870946124796
Fold 1:  0.5856070381311078
Fold 2:  0.5974970150274683
Fold 3:  0.5878650191316905
Fold 4:  0.5757172734468895
Fold 5:  0.599551556072437
Fold 6:  0.6102478625051501
Fold 7:  0.5860529534500418
Fold 8:  0.6010971207474742
Fold 9:  0.6094025142666933
Avg score : 0.594<jupyter_text>Submission<jupyter_code>final_pred = np.mean(test_oofs, axis=0)
final_pred = process_prediction(final_pred)
submission = test[['tweet_id']]
submission['target'] = final_pred
submission.columns = ['ID', 'target']
submission.head()
submission.describe()
submission.to_csv(f'linear_regression_{avg/n_fold}.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16051454138702462, 'fraction_numerical': 0.04548844146159582, 'mean_word_length': 6.912979351032448, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '481375', 'license': 'MIT', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2068, 'n_words': 347, 'path': 'Competition-Solutions/Text/#ZindiWeekendz Learning- To Vaccinate or Not to Vaccinate/Solution 2/starter_notebook.ipynb', 'repo_name': 'ZindiAfrica/Natural-Language-Processing-NLP-', 'size': '5364'}