<commit_before>from sklearn.datasets import load_files
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
import numpy as np
from sklearn.pipeline import Pipeline


categories = ['neg', 'pos']

twitter_train = load_files('./twitter_data/twitter_data-train', categories=categories, load_content=True, shuffle=True, random_state=42)

#Ignoring decode errors may harm our results, but at least it works now
count_vect = CountVectorizer(decode_error='ignore')
X_train_counts = count_vect.fit_transform(twitter_train.data)
X_train_counts.shape

# Use frequencies, not occurrences
tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)
X_train_tf = tf_transformer.transform(X_train_counts)
print(X_train_tf.shape)

# Tf-idf get rid of redundant terms.
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
print(X_train_tfidf.shape)

# Train the classifier
clf = MultinomialNB().fit(X_train_tfidf, twitter_train.target)
docs_new = ['I am going to rape you.', 'School is cool.']
X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)

predicted = clf.predict(X_new_tfidf)

for doc, category in zip(docs_new, predicted):
    print('%r => %s' % (doc, twitter_train.target_names[category]))

text_clf = Pipeline([('vect', CountVectorizer()),
                     ('tfidf', TfidfTransformer()),
                     ('clf', MultinomialNB())])
text_clf = text_clf.fit(twitter_train.data, twitter_train.target)

twitter_test = load_files('./twitter_data/twitter_data-test', categories=categories, load_content=True, shuffle=True, random_state=42)
docs_test = twitter_test.data
predicted = text_clf.predict(docs_test)
print(np.mean(predicted == twitter_test.target))
<commit_msg>Still throwing encode error<commit_after>from sklearn.datasets import load_files
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
import numpy as np
from sklearn.pipeline import Pipeline


categories = ['neg', 'pos']

twitter_train = load_files('./twitter_data/twitter_data-train', categories=categories, load_content=True, shuffle=True, random_state=42)

#Ignoring decode errors may harm our results, but at least it works now
count_vect = CountVectorizer(decode_error='ignore')
X_train_counts = count_vect.fit_transform(twitter_train.data)
X_train_counts.shape

# Use frequencies, not occurrences
tf_transformer = TfidfTransformer(use_idf=False).fit(X_train_counts)
X_train_tf = tf_transformer.transform(X_train_counts)
print(X_train_tf.shape)

# Tf-idf get rid of redundant terms.
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
print(X_train_tfidf.shape)

# Train the classifier
clf = MultinomialNB().fit(X_train_tfidf, twitter_train.target)
docs_new = ['I am going to rape you.', 'School is cool.']
X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)

predicted = clf.predict(X_new_tfidf)

for doc, category in zip(docs_new, predicted):
    print('%r => %s' % (doc, twitter_train.target_names[category]))

text_clf = Pipeline([('vect', CountVectorizer()),
                     ('tfidf', TfidfTransformer()),
                     ('clf', MultinomialNB())])

#for index in range(0, len(twitter_train.data)):
#    twitter_train.data[index].encode('utf-8').strip()
#text_clf = text_clf.fit(twitter_train.data, twitter_train.target)

twitter_test = load_files('./twitter_data/twitter_data-test', categories=categories, load_content=True,
                          encoding='utf-8', decode_error='ignore', shuffle=True, random_state=42)
docs_test = twitter_test.data
predicted = text_clf.predict(docs_test)
print(np.mean(predicted == twitter_test.target))

{'fraction_non_alphanumeric': 0.10607205845301083, 'fraction_numerical': 0.002771478961955152, 'mean_word_length': 6.33826247689464, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '731860', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1330, 'n_words': 261}