<jupyter_start><jupyter_text>Count words frequency for each category<jupyter_code>words_frequency = {}
for category in set(df['main_category'].values):
    all_words = []
    for row in df[df['main_category'] == category]['tokenized_words'].tolist():
        all_words = all_words + ast.literal_eval(row)
    allWordExceptStopDist = nltk.FreqDist(w.lower() for w in all_words)
    most_common = allWordExceptStopDist.most_common(top)
    words_frequency[category] = [word for word, number in most_common]<jupyter_output><empty_output><jupyter_text>Create labels and features set for ML<jupyter_code>features = np.zeros(df.shape[0] * top).reshape(df.shape[0], top)
labels = np.zeros(df.shape[0])
counter = 0
for i, row in df.iterrows():
    c = [word for word, word_count in Counter(ast.literal_eval(row['tokenized_words'])).most_common(top)]
    labels[counter] = list(set(df['main_category'].values)).index(row['main_category'])
    for word in c:
        if word in words_frequency[row['main_category']]:
            features[counter][words_frequency[row['main_category']].index(word)] = 1
    counter += 1

from sklearn.metrics import accuracy_score
from scipy.sparse import coo_matrix
X_sparse = coo_matrix(features)

from sklearn.utils import shuffle
X, X_sparse, y = shuffle(features, X_sparse, labels, random_state=0)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)<jupyter_output><empty_output><jupyter_text>Train and validate data using ML algorithms<jupyter_code>from sklearn.linear_model import LogisticRegression
lr = LogisticRegression()
lr.fit(X_train, y_train)
predictions = lr.predict(X_test)
score = lr.score(X_test, y_test)
print('LogisticRegression')
print('Score: ', score)
print('Top: ', top)
print('Tolerance: ', english_tolerance)
print('Dataset length: ', df.shape[0])
print()
from sklearn.tree import DecisionTreeClassifier
dtc = DecisionTreeClassifier()
dtc.fit(X_train, y_train)
predictions = dtc.predict(X_test)
score = dtc.score(X_test, y_test)
print('DecisionTreeClassifier')
print('Score: ', score)
print('Top: ', top)
print('Tolerance: ', english_tolerance)
print('Dataset length: ', df.shape[0])
print()
from sklearn.svm import LinearSVC
clf = LinearSVC()
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)
score = clf.score(X_test, y_test)
print('SVM')
print('Score: ', score)
print('Top: ', top)
print('Tolerance: ', english_tolerance)
print('Dataset length: ', df.shape[0])<jupyter_output><empty_output><jupyter_text>Save models<jupyter_code>from sklearn.externals import joblib
filename = "Models/LR_model_{}.joblib".format(month)
if not os.path.isfile(filename):
    joblib.dump(lr, filename)

import pickle
words_filename = "Models/word_frequency_{}.picle".format(month)
if not os.path.isfile(words_filename):
    pickle_out = open(words_filename,"wb")
    pickle.dump(words_frequency, pickle_out)
    pickle_out.close()

filename = "Models/LR_maxtrain_{}.joblib".format(month)
if not os.path.isfile(filename):
    from sklearn.linear_model import LogisticRegression
    lr = LogisticRegression()
    lr.fit(X, y)
    joblib.dump(lr, filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13175885643256682, 'fraction_numerical': 0.0046612802983219395, 'mean_word_length': 6.521028037383178, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '156435', 'license': 'MIT', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1114, 'n_words': 220, 'path': '.ipynb_checkpoints/dataset_creation-checkpoint.ipynb', 'repo_name': 'lzomedia/URL-categorization-using-machine-learning', 'size': '3218'}