<jupyter_start><jupyter_text>Features<jupyter_code>def getWordFrequencyWithoutBrands(message):
    tweets = pd.read_csv('./tweets/allLabeledTweets.csv')

    allPostsConcat = ''
    for tweet in tweets[message]:
        if(type(tweet)==str):
            allPostsConcat+=' '+ tweet

    # create bag-of-words
    all_words = []

    words = word_tokenize(allPostsConcat)
    words = [word for word in words if not word in brands]
    for word in words:
        if word!='``' and word!="''" and len(word) > 1:
            all_words.append(word)

    all_words = nltk.FreqDist(all_words)
    
    return all_words
# function to find features in every post
def find_features(post, tweet_features):
    words = word_tokenize(post)
    features = {}
    for word in tweet_features:
        features[word] = (word in words)
    return features<jupyter_output><empty_output><jupyter_text>Get training, testing data<jupyter_code>from sklearn import model_selection

train_df, test_df = model_selection.train_test_split(allLabeledTweets, test_size=0.1)
train_df.shape, test_df.shape
train_neutral = train_df[train_df["label"] == 0]
train_positive = train_df[train_df["label"] == 1]
train_negative = train_df[train_df["label"] == 2]

pd.DataFrame(dict(
  neutral=[len(train_neutral)],
  positive=[len(train_positive)],
  negative=[len(train_negative)]  
)).plot(kind='barh');
train_df = pd.concat([
    train_neutral.sample(75),
    train_positive,
    train_negative.sample(75)])

train_df.shape, test_df.shape<jupyter_output><empty_output><jupyter_text>Lowercase raw tweets<jupyter_code>freqWordsLowercase = getWordFrequencyWithoutBrands('message_lowercase')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsLowercase)))
print('Most common words: {}'.format(freqWordsLowercase.most_common(100)))

word_features_lowercase = list(freqWordsLowercase.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"message_lowercase"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_lowercase), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"message_lowercase"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_lowercase), label) for (text, label) in tweets_test]
nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))
txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>precision    recall  f1-score   support

           0       0.82      0.52      0.64        27
           1       0.25      0.29      0.27         7
           2       0.52      0.86      0.65        14

    accuracy                           0.58        48
   macro avg       0.53      0.55      0.52        48
weighted avg       0.65      0.58      0.59        48<jupyter_text>Cleaned tweets<jupyter_code>freqWordsClean = getWordFrequencyWithoutBrands('clean_message')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsClean)))
print('Most common words: {}'.format(freqWordsClean.most_common(100)))

word_features_clean = list(freqWordsClean.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 60.416666666666664
              precision    recall  f1-score   support

           0       0.88      0.56      0.68        27
           1       0.25      0.29      0.27         7
           2       0.52      0.86      0.65        14

    accuracy                           0.60        48
   macro avg       0.55      0.57      0.53        48
weighted avg       0.68      0.60      0.61        48<jupyter_text>No punctuation<jupyter_code>freqWordsCleanNoPunct = getWordFrequencyWithoutBrands('clean_message_no_punct')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsCleanNoPunct)))
print('Most common words: {}'.format(freqWordsCleanNoPunct.most_common(100)))

word_features_clean_no_punct = list(freqWordsCleanNoPunct.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message_no_punct"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean_no_punct), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message_no_punct"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean_no_punct), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 60.416666666666664
              precision    recall  f1-score   support

           0       0.88      0.56      0.68        27
           1       0.29      0.29      0.29         7
           2       0.50      0.86      0.63        14

    accuracy                           0.60        48
   macro avg       0.56      0.57      0.53        48
weighted avg       0.68      0.60      0.61        48<jupyter_text>Stopwords  From list  With punctuation<jupyter_code>freqWordsCleanNoStopwordsFromList = getWordFrequencyWithoutBrands('clean_message_no_stopwords_from_list')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsCleanNoStopwordsFromList)))
print('Most common words: {}'.format(freqWordsCleanNoStopwordsFromList.most_common(100)))

word_features_clean_no_stopwords_from_list = list(freqWordsCleanNoStopwordsFromList.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message_no_stopwords_from_list"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean_no_stopwords_from_list), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message_no_stopwords_from_list"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean_no_stopwords_from_list), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 60.416666666666664
              precision    recall  f1-score   support

           0       0.86      0.67      0.75        27
           1       0.20      0.29      0.24         7
           2       0.53      0.64      0.58        14

    accuracy                           0.60        48
   macro avg       0.53      0.53      0.52        48
weighted avg       0.67      0.60      0.63        48<jupyter_text>No punctuation<jupyter_code>freqWordsCleanNoPunctNoStopwordsFromList = getWordFrequencyWithoutBrands('clean_message_no_punct_no_stopwords_from_list')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsCleanNoPunctNoStopwordsFromList)))
print('Most common words: {}'.format(freqWordsCleanNoPunctNoStopwordsFromList.most_common(100)))

word_features_clean_no_punct_no_stopwords_from_list = list(freqWordsCleanNoPunctNoStopwordsFromList.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message_no_punct_no_stopwords_from_list"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean_no_punct_no_stopwords_from_list), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message_no_punct_no_stopwords_from_list"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean_no_punct_no_stopwords_from_list), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 62.5
              precision    recall  f1-score   support

           0       0.86      0.67      0.75        27
           1       0.27      0.43      0.33         7
           2       0.56      0.64      0.60        14

    accuracy                           0.62        48
   macro avg       0.56      0.58      0.56        48
weighted avg       0.69      0.62      0.65        48<jupyter_text>From most frequently used words excluding brand names  With punctuation<jupyter_code>freqWordsCleanNoFreqStopwords = getWordFrequencyWithoutBrands('clean_message_no_freq_stopwords')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsCleanNoFreqStopwords)))
print('Most common words: {}'.format(freqWordsCleanNoFreqStopwords.most_common(100)))

word_features_clean_no_freq_stopwords = list(freqWordsCleanNoFreqStopwords.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message_no_freq_stopwords"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean_no_freq_stopwords), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message_no_freq_stopwords"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean_no_freq_stopwords), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 64.58333333333334
              precision    recall  f1-score   support

           0       0.82      0.67      0.73        27
           1       0.30      0.43      0.35         7
           2       0.62      0.71      0.67        14

    accuracy                           0.65        48
   macro avg       0.58      0.60      0.58        48
weighted avg       0.69      0.65      0.66        48<jupyter_text>No punctuation<jupyter_code>freqWordsCleanNoPunctNoFreqStopwords = getWordFrequencyWithoutBrands('clean_message_no_punct_no_freq_stopwords')

# print the total number of words and the 100 most common words
print('Number of words: {}'.format(len(freqWordsCleanNoPunctNoFreqStopwords)))
print('Most common words: {}'.format(freqWordsCleanNoPunctNoFreqStopwords.most_common(100)))

word_features_clean_no_punct_no_freq_stopwords = list(freqWordsCleanNoPunctNoFreqStopwords.keys())[:1500]
tweets_train = list(zip(train_df.loc[:,"clean_message_no_punct_no_freq_stopwords"].values,train_df.loc[:,"label"].values))
featuresets_train = [(find_features(text, word_features_clean_no_punct_no_freq_stopwords), label) for (text, label) in tweets_train]

tweets_test = list(zip(test_df.loc[:,"clean_message_no_punct_no_freq_stopwords"].values,test_df.loc[:,"label"].values))
featuresets_test = [(find_features(text, word_features_clean_no_punct_no_freq_stopwords), label) for (text, label) in tweets_test]

nltk_model = SklearnClassifier(MultinomialNB())
nltk_model.train(featuresets_train)
accuracy = nltk.classify.accuracy(nltk_model, featuresets_test)*100
print("{} Accuracy: {}".format("Naive Bayes", accuracy))

txt_features, labels = zip(*featuresets_test)

prediction = nltk_model.classify_many(txt_features)
# print a confusion matrix and a classification report
print(classification_report(labels, prediction))

pd.DataFrame(
    confusion_matrix(labels, prediction),
index = [['actual', 'actual', 'actual'], ['neutral', 'positive', 'negative']],
    columns = [['predicted', 'predicted', 'predicted'], ['neutral', 'positive', 'negative']])<jupyter_output>Naive Bayes Accuracy: 70.83333333333334
              precision    recall  f1-score   support

           0       0.86      0.70      0.78        27
           1       0.38      0.43      0.40         7
           2       0.67      0.86      0.75        14

    accuracy                           0.71        48
   macro avg       0.64      0.66      0.64        48
weighted avg       0.73      0.71      0.71        48
{'fraction_non_alphanumeric': 0.11802408390521818, 'fraction_numerical': 0.04065777547585135, 'mean_word_length': 4.002266839378239, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 35, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '595136', 'license': 'MIT', 'n_tokens_mistral': 5381, 'n_tokens_neox': 4947, 'n_words': 990, 'path': 'Naive_Bayes/trainNB.ipynb', 'repo_name': 'Alliix/twitter-lv-market', 'size': '15446'}