<jupyter_start><jupyter_text>Read data from .csv file<jupyter_code>dataset_path = 'dataset.csv'    # path to our all urls file
dataset = pd.read_csv(dataset_path,sep=',',encoding='utf-8', error_bad_lines=False)    # reading file
df = pd.DataFrame(dataset)    # converting to a dataframe

df    # print
df = np.array(df)    # converting it into an array
random.shuffle(df)    # shuffling
target = [d[1] for d in df]    # all labels
corpus = [d[0] for d in df]    # all urls corresponding to a label (either good or bad)<jupyter_output><empty_output><jupyter_text>TF-IDF<jupyter_code>vect = TfidfVectorizer(tokenizer=tk, max_features=1000)    # get a vector for each url but use our customized tokenizer
corpus_vect = vect.fit_transform(corpus)    # get the corpus vector

vect.get_feature_names()<jupyter_output><empty_output><jupyter_text>Split train / test data (70:30%)split into training and testing set 70/30 ratio<jupyter_code>test_size = 0.3

data_train, data_test, target_train, target_test = train_test_split(corpus_vect, target, test_size=test_size, random_state=100)<jupyter_output><empty_output><jupyter_text>Create model: using logistic regression<jupyter_code>model = LogisticRegression()
model.fit(data_train, target_train)    # run(fit)<jupyter_output><empty_output><jupyter_text>Evaluate model : Accuracy<jupyter_code>print('Accuracy = %.2f %%' %( model.score(data_test, target_test)*100) )<jupyter_output>Accuracy = 92.73 %<jupyter_text>Save model<jupyter_code>joblib.dump(model, 'url_clf.model')
joblib.dump(vect, 'url_clf.vect')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11477488902980343, 'fraction_numerical': 0.017755231452124286, 'mean_word_length': 6.773399014778325, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '617984', 'license': 'MIT', 'n_tokens_mistral': 605, 'n_tokens_neox': 572, 'n_words': 118, 'path': 'datafun/url_clf/create_model.ipynb', 'repo_name': 'dogterbox/datafun', 'size': '1577'}