<jupyter_start><jupyter_text>*Upload submission_db.sqlite3 database*<jupyter_code>from google.colab import files 
uploaded = files.upload()
import pandas as pd
import sqlite3
import numpy as np

def refresh_connection_and_cursor():
  try:
    c.close()
    conn.close()
  except: pass
  conn = sqlite3.connect('/content/submission_db.sqlite3')
  c = conn.cursor()
  return conn, c

conn, c = refresh_connection_and_cursor()

data = pd.read_sql_query(
    "select * from submission_table order by random() limit 100000;", 
    conn)

c.close()
conn.close()
print(data.shape)
data.head()
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 99502 entries, 0 to 99501
Data columns (total 5 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   subreddit_name  99502 non-null  object
 1   subreddit_id    99502 non-null  object
 2   subreddit_subs  99502 non-null  int64 
 3   title           99502 non-null  object
 4   text            99502 non-null  object
dtypes: int64(1), object(4)
memory usage: 3.8+ MB<jupyter_text>*Data Wrangling*<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
def warngle(df, text_len_min=100, subreddit_num=50):
  """
  df: dataframe containg the subreddits and their categories
  text_len_min: minimum length of subreddit text required to be analyzed
  statistical trimming:
  subreddit_num: sampled number of subreddit posts in each category
  """
  print("\ndf shape: ", df.shape)
  
  # filter out the rows with no significant subreddit text
  df = df[[len(i)>100 for i in df['text']]]
  print(f"\ndf shape after text length filter: {text_len_min}:")
  print(df.shape)
  # filter out the rows with low count of subreddit_ids
  subid_count = df['subreddit_id'].value_counts(ascending=False)
  
  plt.figure(figsize=(12,5))
  # y axis is density plot
  sns.distplot(subid_count, kde=False, bins=len(df['subreddit_id'].value_counts()))
  plt.title('Subreddit category histogram')
  plt.xlabel('#of posts per subreddit category')
  plt.ylabel('# of subreddit categories')

  # filter out the categories with low number of posts 
  df = df.groupby('subreddit_id').filter(
      lambda x: (x['subreddit_id'].value_counts() > subreddit_num))
  print(f"\ndf shape after dropping less than {subreddit_num} posts per category filter")
  print(df.shape)

  # converting the title and text columns to string type
  df[['title', 'text']] = df[['title', 'text']].astype('str')

  # filter out non word and non space characters.
  # lower case and remove extra spaces
  df['text'] = df['text'].str.replace("[^\w\s']",' ').str.lower().apply(lambda row: ' '.join(row.split()))
  df['title'] = df['title'].str.replace("[^\w\s']",' ').str.lower().apply(lambda row: ' '.join(row.split()))
  # print(df['text'][:10])

  # merge title and text
  df['joint_text'] = df[['title', 'text']].agg('-'.join, axis=1)
  # print(df.head())

  # Sample a fixed number of posts from each category
  df_sample = df.groupby('subreddit_id').apply(lambda x: x.sample(
      n=subreddit_num, replace=False, random_state=1))#.reset_index(drop = True)
  print(f"\ndf shape after sampling {subreddit_num} posts per subreddit category")
  print(df_sample.shape)
  
  # Number of subreddit categories to classify on
  print(f"\n# of subreddit categories: {len(df_sample['subreddit_name'].value_counts())}")

  return df, df_sample
data, df = warngle(data, text_len_min=100, subreddit_num=50);
print(df.shape)
df.sample(n=5, axis=0)
df['subreddit_name'].unique()<jupyter_output><empty_output><jupyter_text>Spacy Embedding<jupyter_code>#Create the nlp object
nlp = spacy.load("en_core_web_md")
# Use Spacy Embedding to vectorize df['join_text']
X = pd.DataFrame([nlp(str(d)).vector for d in df['joint_text']])
y = df['subreddit_name']
X.shape,  y.shape

from sklearn.model_selection import train_test_split

#split dataset into train and test data
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size=0.2, 
                                                    random_state=1, 
                                                    shuffle=True,
                                                    stratify=y)

X_train.shape, y_train.shape
# column and row
X[0].shape, X.iloc[0].shape<jupyter_output><empty_output><jupyter_text>KNeighbors Classifier<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
# NearestNeighbors is the same as the classifier one but unsupervised
knn = KNeighborsClassifier(n_neighbors = 15)

# Fit the classifier to the data
knn.fit(X_train.values, y_train.values)
knn.score(X_test.values, y_test.values)
def preprocess(text, nlp):
  """
  Vectorizes the 'text' string by spacy pretrained model (nlp). 
  the return value serves as a query to the prediction model
  """
  return nlp(text).vector.reshape(1,-1)

def prediction_spacy(model, input):
  """
  return the nested list of predicted neighbors' indices from 
  the trained dataframe with their distance from the input
  input: a string type text
  """
  kn = model.kneighbors(input)
  return kn

def postprocess(kn, df_ser, n):
  """
  takes the nested list of kneighbors output (kn) and 
  returns first n recommended subreddit names.
  df_ser: pandas Series that has the class names and their respective indices
  """
  return df_ser.iloc[kn[1][0]].drop_duplicates()[:n].reset_index(drop=True)
query = df['joint_text'].iloc[10]
category = df['subreddit_name'].iloc[10]

print("category: {}".format(category))
print(query,"\n")

input = preprocess(query, nlp)
kn = prediction_spacy(knn, input)
pred = postprocess(kn, df['subreddit_name'], 10)
pred
df['joint_text'].iloc[kn[1][0]]
user_input = """Wildfires sweeping through the West Coast killed at least 36 people, CNN reports
- The fires have burned across a total of some 4.5 million acres (1.8 million hectares) in Oregon, California, Washington and Idaho, according to Reuters
- Wildfires are causing dangerous air quality and emitting record amounts of carbon dioxide"""

input = preprocess(user_input, nlp)
kn = prediction_spacy(knn, input)
pred = postprocess(kn, df['subreddit_name'], 5)
pred
df['joint_text'].iloc[kn[1][0][:5]]<jupyter_output><empty_output><jupyter_text>Model Serialization<jupyter_code>import pickle
# pickle knn model
knnPickle = open('knnpickle', 'wb') 
pickle.dump(knn, knnPickle)   

# pickle the dataframe
df['subreddit_name'].to_pickle("./namedf.pkl")

# nlp pickle
nlpPickle = open('nlppickle', 'wb') 
pickle.dump(nlp, nlpPickle) 
loaded_model = pickle.load(open('knnpickle', 'rb'))
loaded_nlp = pickle.load(open('nlppickle', 'rb'))
loaded_df = pd.read_pickle("./namedf.pkl")
input = preprocess(user_input, loaded_nlp)
kn = prediction_spacy(loaded_model, input)
pred = postprocess(kn, loaded_df, 5)
pred<jupyter_output><empty_output><jupyter_text>Latent Semantic Indexing (LSI)<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.decomposition import TruncatedSVD
# Create Pipeline Components

vect = TfidfVectorizer( stop_words='english', 
                       ngram_range=(1,2),
                       lowercase=True,
                       max_df=0.9,
                       min_df=1,
                       max_features=None)

svd = TruncatedSVD(n_components=100, # Just here for demo. 
                   algorithm='randomized',
                   n_iter=10, random_state=1)

rfc = RandomForestClassifier()
lsi = Pipeline([('vect', vect), ('svd', svd)])
pipe = Pipeline([('lsi', lsi), ('clf', rfc)])
parameters = {
    'lsi__vect__max_df': ( 0.75, 1.0),
    'lsi__vect__min_df': (.02, .05),
    'lsi__vect__max_features': (500,1000),
    'lsi__svd__n_components': [10,100,250],
    'clf__n_estimators':(5, 10,),
    'clf__max_depth':(15,20)
}

grid_search = GridSearchCV(pipe,parameters, cv=5, n_jobs=7, verbose=1)
grid_search.fit(df['joint_text'], y)
# The accuracy is less that spacy vectorizer
grid_search.best_score_
grid_search.predict([user_input])
grid_search.best_estimator_
def prediction_est(model, input):
  """
  input: text string or list as input
  model: grid search trained model
  returns a pandas series containing the the recommended subreddit names
  """
  if isinstance(input, str):
    input = [input]
  # probability of each class
  proba = model.predict_proba(input)

  # index of the highest probable classes in descending order
  index_proba = [i[0] for i in sorted(enumerate(proba[0]), key=lambda x:x[1])][::-1]
  # Alternative way
  # idx_max = proba[0].argsort()[::-1]

  # get the most probable class names
  return pd.Series(model.classes_[index_proba][:15])
prediction_est(grid_search, user_input)
# pickle grid search model with rfc+lsi
rfclsi = open('rfclsi_pickle', 'wb') 
pickle.dump(grid_search, rfclsi)   
loaded_model = pickle.load(open('rfclsi_pickle', 'rb'))
prediction_est(loaded_model, user_input)<jupyter_output><empty_output><jupyter_text>Gradient Boost Classifier:<jupyter_code>from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import GradientBoostingClassifier
import numpy as np
from scipy.stats import randint
from sklearn.preprocessing import FunctionTransformer
import spacy
nlp = spacy.load("en_core_web_md")<jupyter_output><empty_output><jupyter_text>* Spacy vectorizer<jupyter_code># def get_word_vectors(docs):
#     return np.array([nlp(doc).vector for doc in docs])
# Instantiate the spacy vectorizer
# embed = FunctionTransformer(get_word_vectors)
# GradientBoostingClassifier
gbc = GradientBoostingClassifier(loss='deviance', learning_rate=0.1, n_estimators=100, 
                               subsample=1.0, criterion='friedman_mse', min_samples_split=2, 
                               min_samples_leaf=1, min_weight_fraction_leaf=0.0, 
                               max_depth=3, min_impurity_decrease=0.0, min_impurity_split=None, 
                               init=None, random_state=None, max_features=None, verbose=0, 
                               max_leaf_nodes=None, warm_start=False, presort='deprecated', 
                               validation_fraction=0.1, n_iter_no_change=None, tol=0.0001, ccp_alpha=0.0)

# spacy embedding is taken out of the pipeline to save time
pipe = Pipeline([
#     ('emb', embed), 
    ('clf', gbc)
    ])
# Gradient Boost Classifier
gbc_par_names = ["max_features", "max_depth", "min_samples_split", "min_samples_leaf"]

max_features_gbc = np.arange(2,30,2)
max_depth_gbc = range(2,16,2)
min_samples_split_gbc = np.linspace(2, 40, 10, endpoint=True).astype(int)
min_samples_leaf_gbc = np.linspace(2, 40, 10, endpoint=True).astype(int)
criterion_gbc = ["gini", "entropy"]
gbc_param_ranges = [max_features_gbc, max_depth_gbc, min_samples_split_gbc, min_samples_leaf_gbc]

param_dist = {
    'clf__max_features': max_features_gbc, 
    'clf__max_depth': max_depth_gbc, 
    'clf__min_samples_split': min_samples_split_gbc, 
    'clf__min_samples_leaf': min_samples_leaf_gbc
    }
rscv = RandomizedSearchCV(pipe, param_distributions=param_dist, n_iter=2, cv=2, verbose=1, n_jobs=-1)
rscv.fit(X, y)
print(rscv.best_score_)
rscv.best_estimator_
# pickle randomized searchCV model with gbc+nlp
gbnlp = open('gbnlp_pickle', 'wb') 
pickle.dump(rscv, gbnlp)   
loaded_model = pickle.load(open('gbnlp_pickle', 'rb'))
input = preprocess(user_input, loaded_nlp)
input.shape
X.shape
input = preprocess(user_input, loaded_nlp)
pred = prediction_est(loaded_model, input)
pred<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11994493202546894, 'fraction_numerical': 0.01815522285320943, 'mean_word_length': 4.240306582506763, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '239013', 'license': 'MIT', 'n_tokens_mistral': 4132, 'n_tokens_neox': 3979, 'n_words': 936, 'path': 'redditmodel.ipynb', 'repo_name': 'Build-Week-Post-Here-2/ds', 'size': '11622'}