<jupyter_start><jupyter_text>https://github.com/t-davidson/hate-speech-and-offensive-language/blob/master/src/Automated%20Hate%20Speech%20Detection%20and%20the%20Problem%20of%20Offensive%20Language%20Python%203.6.ipynb<jupyter_code>import nltk
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.stem.porter import *
import re
# nltk.download()
stemmer = PorterStemmer()

def preprocess(tweet):
    space_pattern = '\s+'
    url_pattern = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'
        '[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    mention_pattern = '@[\w\-]+'
    text = re.sub(space_pattern, ' ', tweet)
    text = re.sub(url_pattern, '', tweet)
    text = re.sub(mention_pattern, '', tweet)
    return text

def tokenize(tweet):
    tweet = " ".join(re.split("[^a-zA-Z]*", tweet.lower())).strip()
    tokens = [stemmer.stem(t) for t in tweet.split()]
    return tokens

def basic_tokenize(tweet):
    tweet = " ".join(re.split("[^a-zA-Z.,!?]*", tweet.lower())).strip()
    return tweet.split()
vectorizer = TfidfVectorizer(
    preprocessor=preprocess,
    ngram_range=(1, 3),
    stop_words=nltk.corpus.stopwords.words('english'),
    use_idf=True,
    smooth_idf=True,
    norm=None,
    decode_error='replace',
    max_features=10000,
    min_df=5,
    max_df=0.75
)
tweets = df['tweet']
tfidf = vectorizer.fit_transform(tweets).toarray()
vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}
idf_vals = vectorizer.idf_
idf_dict = {i:idf_vals[i] for i in vocab.values()}
from sklearn.model_selection import train_test_split

X = tfidf
y = df['class']

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.2)
from sklearn.model_selection import StratifiedKFold, cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier

cv = StratifiedKFold(n_splits=5)
clf = RandomForestClassifier()
cross_val_score(clf, X_train, y_train, cv=cv, n_jobs=-1)
from sklearn.metrics import classification_report

clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print(classification_report(y_test, y_pred))
from sklearn.metrics import confusion_matrix
import seaborn as sns

confusion_matrix = confusion_matrix(y_test, y_pred)
matrix_proportions = np.zeros((3,3))
for i in range(0,3):
    matrix_proportions[i,:] = confusion_matrix[i,:]/float(confusion_matrix[i,:].sum())
names = ['Hate','Offensive','Neither']
confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)

plt.figure(figsize=(8, 8))
sns.heatmap(confusion_df, annot=True,annot_kws={"size": 10},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')
plt.ylabel('True categories', fontsize=10)
plt.xlabel('Predicted categories', fontsize=10)
plt.tick_params(labelsize=10)
# plt.savefig('rf_base_model.png')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16081460674157302, 'fraction_numerical': 0.021769662921348316, 'mean_word_length': 5.9487804878048784, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '591824', 'license': 'Apache-2.0', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1114, 'n_words': 173, 'path': 'detecting_hate_speech_social_media/base_model.ipynb', 'repo_name': 'robertjankowski/reproducing-dl-papers', 'size': '2848'}