<jupyter_start><jupyter_text>EAST AFRICA VIRTUAL HACKATHON 2021: SWAHILI NEWS ML CHALLENGE  Let's Get Started<jupyter_code># import important modules
import numpy as np
import pandas as pd

# sklearn modules
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB # classifier 

from sklearn.metrics import log_loss #evaluation metric
from sklearn.feature_extraction.text import CountVectorizer

# text preprocessing modules
import re 
from string import punctuation 

import warnings
warnings.filterwarnings("ignore")
# seeding
np.random.seed(123)
# load data
path = ''
train = pd.read_csv(path+"train.csv")
test = pd.read_csv(path+"test.csv")
submission = pd.read_csv(path+"sample_submission.csv")
# show top five rows of train data
train.head() 
# show top five rows of test data
test.head()
# show top five rows of submision file
submission.head()
# check the shape of the train data
train.shape
# check the shape of the test data
test.shape
# check missing values in train data
train.isnull().sum()
# check missing values in test data
test.isnull().sum()
# evalute news category distribution
train.category.value_counts()<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code># a mapping dictionary that maps the category values from 0 to 5
category_mapping = {
"kitaifa": 0,
"michezo": 1,
"burudani": 2,
"uchumi": 3,
"kimataifa": 4,
"afya": 5
}

train["category"] = train.category.map(category_mapping)

train.head()
# a simple function to clean text data 

def text_cleaning(text):
    # Clean the text data

    text = re.sub(r"[^A-Za-z0-9]", " ", text)
    text = re.sub(r'\b\d+(?:\.\d+)?\s+', '', text) # remove numbers
    text = text.lower()  # set in lowercase 
        
    # Remove punctuation from text
    text = ''.join([c for c in text if c not in punctuation])
        
    # Return a list of words
    return(text)
#clean the train and test data
train["content"] = train["content"].apply(text_cleaning)
test["content"] = test["content"].apply(text_cleaning)
#split features and target from train data 
X = train["content"]
y = train.category.values
# Transform text data 
vectorizer = CountVectorizer(lowercase=False)

vectorizer.fit(X)

#transform train data 
X_transformed = vectorizer.transform(X)

#transform test data
test_transformed = vectorizer.transform(test["content"])
# split data into train and validate

X_train, X_valid, y_train, y_valid = train_test_split(
    X_transformed,
    y,
    test_size=0.20,
    random_state=42,
    shuffle=True,
    stratify=y,
)<jupyter_output><empty_output><jupyter_text>Create Classifier<jupyter_code># Create a classifier
news_classifier = MultinomialNB()    
# train the news_classifier 
news_classifier.fit(X_train,y_train)
# test model performance on valid data 
y_probas = news_classifier.predict_proba(X_valid)
# evalute model performance by using log_loss in the validation data
log_loss(y_valid, y_probas)
# create prediction from the test data
test_probas = news_classifier.predict_proba(test_transformed)<jupyter_output><empty_output><jupyter_text>Create Submission File<jupyter_code># create submission file 
submission_cols = ['kitaifa', 'michezo', 'burudani','uchumi', 'kimataifa', 'afya'] 
submission_df = pd.DataFrame(test_probas, columns = submission_cols)
submission_df['test_id'] = submission['test_id']   # add  test_id 

#rearange columns 
submission_df = submission_df[['test_id','kitaifa', 'michezo', 'burudani','uchumi', 'kimataifa', 'afya']]

# save submission file 
submission_df.to_csv(path+"first_submission.csv",index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10819037078029883, 'fraction_numerical': 0.006087437742114001, 'mean_word_length': 5.537070524412297, 'pattern_counts': {'":': 6, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '441690', 'license': 'MIT', 'n_tokens_mistral': 1233, 'n_tokens_neox': 1188, 'n_words': 326, 'path': 'StarterNotebook.ipynb', 'repo_name': 'Paul-mwaura/ML-Hackathons', 'size': '3614'}