<jupyter_start><jupyter_text>Spacy Cleaner Proposed first model.Since we would like to use other metadata for model improvement, SpaCy will not suffice.When adding metadata, a PyTorch architecture will be built.<jupyter_code>import pandas as pd
import json
data = pd.read_csv('data/labeled_data.csv', index_col = 0)
data = data[['readme_content', 'Purpose']]
data = data[data['readme_content'] != 'Missing']
data.head(3)
# Misc
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Informational link to database', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Article referencing database links', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Miscellaneous - Scraping database registries', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous; Miscellaneous - Article referencing database links', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Informational Link', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Scraping database registries', 'Miscellaneous')

data['Purpose'] = data['Purpose'].str.replace('Miscellaneous; Miscellaneous', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Articles referencing database links; Miscellaneous - Scraping open science publication databases', 'Miscellaneous')

data['Purpose'] = data['Purpose'].str.replace('Miscellaneous', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Scraping database registries', 'Miscellaneous')

data['Purpose'] = data['Purpose'].str.replace('Miscellaneous - Articles referencing database links', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous ', 'Miscellaneous')
data['Purpose'] = data['Purpose'].str.replace('Miscellaneous- Article with database linkts', 'Miscellaneous')



# Software Dev
data['Purpose'] = data['Purpose'].str.replace('Software development', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Software Development ', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Software Development ', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Storage; Software Development  ', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Software Development; Storage', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Storage; Software Development', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Software Development; Miscellanous - Scraping database registries', 'Software Development')
data['Purpose'] = data['Purpose'].str.replace('Software Development; Miscellaneous', 'Software Development')

# Analysis
data['Purpose'] = data['Purpose'].str.replace('Analysis; Storage', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis; Miscellaneous- Articles referencing databases', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis; Educational; Miscellaneous', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis; Educational', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis; Software Development', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis ', 'Analysis')
data['Purpose'] = data['Purpose'].str.replace('Analysis; Educational', 'Analysis')

# Educational
data['Purpose'] = data['Purpose'].str.replace('Educational, Software Development', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational; Miscellaneous', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational; Miscellaneous- Informational', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational; Software Development', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational ', 'Educational')
data['Purpose'] = data['Purpose'].str.replace('Educational- Informational', 'Educational')

data['Purpose'].unique()
data['Purpose'].value_counts().sort_values()
data = data[data['Purpose'] != 'Storage']
data = data[data['Purpose'] != "Can't categorize/not enough information"]
data = data.reset_index(drop = True)
data
data['Purpose'].value_counts()<jupyter_output><empty_output><jupyter_text>Split Dataset<jupyter_code>from sklearn.model_selection import train_test_split
X = data['readme_content']
y = data['Purpose']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)<jupyter_output><empty_output><jupyter_text>Create SpaCy Tokenizer<jupyter_code>import spacy
import en_core_web_lg
from spacy.lang.en.stop_words import STOP_WORDS
from spacy.lang.en import English

import string

from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.base import TransformerMixin
from sklearn.pipeline import Pipeline
punctuations = string.punctuation

# Create our list of stopwords
nlp = spacy.load('en')
stop_words = spacy.lang.en.stop_words.STOP_WORDS

# Load English tokenizer, tagger, parser, NER and word vectors
parser = English()

# Creating our tokenizer function
def spacy_tokenizer(sentence):
    # Creating our token object, which is used to create documents with linguistic annotations.
    mytokens = parser(sentence)

    # Lemmatizing each token and converting each token into lowercase
    mytokens = [ word.lemma_.lower().strip() if word.lemma_ != "-PRON-" else word.lower_ for word in mytokens ]

    # Removing stop words
    mytokens = [ word for word in mytokens if word not in stop_words and word not in punctuations ]

    # return preprocessed list of tokens
    return mytokens<jupyter_output><empty_output><jupyter_text>Defining a Custom TransformerTo further clean our text data, we’ll also want to create a custom transformer for removing initial and end spaces and converting text into lower case. Here, we will create a custom predictors class wich inherits the TransformerMixin class. This class overrides the transform, fit and get_parrams methods. We’ll also create a clean_text() function that removes spaces and converts text into lowercase.<jupyter_code># Custom transformer using spaCy
class predictors(TransformerMixin):
    def transform(self, X, **transform_params):
        # Cleaning Text
        return [clean_text(text) for text in X]

    def fit(self, X, y=None, **fit_params):
        return self

    def get_params(self, deep=True):
        return {}

# Basic function to clean the text
def clean_text(text):
    # Removing spaces and converting text into lowercase
    return text.strip().lower()<jupyter_output><empty_output><jupyter_text>Vectorization Feature Engineering (BOW or TF-IDF)<jupyter_code>bow_vector = CountVectorizer(tokenizer = spacy_tokenizer, ngram_range=(1,1))
tfidf_vector = TfidfVectorizer(tokenizer = spacy_tokenizer)
tfidf_vector<jupyter_output><empty_output><jupyter_text>Classifier Construction  Training and Parameter Tuning<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.base import BaseEstimator
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier

class ClfSwitcher(BaseEstimator):
    def __init__(
        self, 
        estimator = SGDClassifier(),
    ):
        """
        A Custom BaseEstimator that can switch between classifiers.
        :param estimator: sklearn object - The classifier
        """ 

        self.estimator = estimator


    def fit(self, X, y=None, **kwargs):
        self.estimator.fit(X, y)
        return self


    def predict(self, X, y=None):
        return self.estimator.predict(X)


    def predict_proba(self, X):
        return self.estimator.predict_proba(X)


    def score(self, X, y):
        return self.estimator.score(X, y)
tree = DecisionTreeClassifier()

# Create pipeline using Bag of Words
pipe = Pipeline([('cleaner', predictors()),
                 ('vectorizer', tfidf_vector),
                 ('clf', ClfSwitcher())])
param_grid = [
              {
               'clf__estimator': [SGDClassifier()], # SVM if hinge loss / logreg if log loss
               #'tfidf__max_df': (0.25, 0.5, 0.75, 1.0),
               #'tfidf__stop_words': ['english', None],
               'clf__estimator__penalty': ('l2', 'elasticnet', 'l1'),
               'clf__estimator__max_iter': [50, 80],
               'clf__estimator__tol': [1e-4],
               'clf__estimator__loss': ['hinge', 'log', 'modified_huber'],
               },
              {
               'clf__estimator': [MultinomialNB()],
               #'tfidf__max_df': (0.25, 0.5, 0.75, 1.0),
               #'tfidf__stop_words': [None],
               'clf__estimator__alpha': (1e-2, 1e-3, 1e-1),
               },
              {
               'clf__estimator': [DecisionTreeClassifier()],
               'clf__estimator__max_depth': list(range(1,20,2))}
               ]
#param_grid = {'classifier__max_depth': list(range(1,20,2))}
grid_search = GridSearchCV(pipe, param_grid, cv = 5, verbose=1, return_train_score=True)
grid_search.fit(X_train, y_train)
grid_search.best_score_
#pd.DataFrame(grid_search).mean()
best_estimator = grid_search.best_estimator_
best_estimator.predict(X_test)
from sklearn import metrics
# Predicting with a test dataset
predicted = best_estimator.predict(X_test)
train_predicted = best_estimator.predict(X_train)

# Model Accuracy
print("Best Estimator Test Accuracy:",metrics.accuracy_score(y_test, predicted))
print("Best Estimator Train Accuracy:",metrics.accuracy_score(y_train, train_predicted))
## To think about - would predict_proba save us from a NN?

best_estimator.predict_proba(X_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13178057002758198, 'fraction_numerical': 0.00541424047400143, 'mean_word_length': 5.283697047496791, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '519169', 'license': 'MIT', 'n_tokens_mistral': 3154, 'n_tokens_neox': 2915, 'n_words': 701, 'path': 'SpaCy Preprocessing.ipynb', 'repo_name': 'throughput-ec/repo_classifier', 'size': '9789'}