<jupyter_start><jupyter_text>Tweets Analysis  1. Import Libraries<jupyter_code>import numpy as np
import pandas as pd
from time import time
import warnings
from sklearn.svm import SVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import AdaBoostClassifier
from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.metrics import classification_report, fbeta_score, accuracy_score, confusion_matrix, plot_roc_curve, ConfusionMatrixDisplay, make_scorer
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from imblearn.over_sampling import RandomOverSampler<jupyter_output><empty_output><jupyter_text>2. Read the Data<jupyter_code>tweets = pd.read_csv('Tweets\\final_tweets_classified.csv')
tweets = tweets.drop(['Column1', 'index'], axis=1)
tweets = tweets.reset_index(drop=True)
tweets.head()

tweets.info()
tweets.columns<jupyter_output><empty_output><jupyter_text>3. Predective Statistics  Data Preprocessing & Feature Extraction<jupyter_code>tweets.head()
# Drop the neutral class 
tweets = tweets[tweets.Class != 'neutral']
tweets = tweets.reset_index(drop=True)
# Convert the classes negative and positive to 0 and 1 respectively
tweets['Class'] = tweets['Class'].map({'negative': 0, 'positive': 1})
display(tweets.tail(10))
X = tweets['Tweet']  # features
y = tweets['Class'] # classes
# Convert to a vector representation
unsampled_tfidf = TfidfVectorizer() 
unsampled_X = unsampled_tfidf.fit_transform(X)
unsampled_X<jupyter_output><empty_output><jupyter_text>Split the Data<jupyter_code>X_train_unsampled, X_test_unsampled, y_train_unsampled, y_test_unsampled = train_test_split(unsampled_X, y, test_size=0.3, random_state=27)
print("Training set has {} samples.".format(X_train_unsampled.shape[0]))
print("Testing set has {} samples.".format(X_test_unsampled.shape[0]))
upsampled_tfidf = TfidfVectorizer()
upsampled_tfidf.fit(X)
X_train_upsampled, X_test_upsampled, y_train_upsampled, y_test_upsampled = train_test_split(X, y, test_size=0.3, random_state=27)

print("Training set has {} samples.".format(X_train_upsampled.shape[0]))
print("Testing set has {} samples.".format(X_test_upsampled.shape[0]))
X_train_upsampled = upsampled_tfidf.transform(X_train_upsampled)
X_test_upsampled = upsampled_tfidf.transform(X_test_upsampled)
upsampled_X = upsampled_tfidf.transform(X)

X_train_upsampled
ros = RandomOverSampler(random_state=27)

X_train_upsampled, y_train_upsampled = ros.fit_resample(X_train_upsampled, y_train_upsampled)
print("Data before upsampling: {} samples.".format(y_train_unsampled.value_counts()[0]))
print("Data before upsampling: {} samples.".format(y_train_unsampled.value_counts()[1]))
print("Data after upsampling: {} samples.".format(y_train_upsampled.value_counts()[0]))
print("Data after upsampling: {} samples.".format(y_train_upsampled.value_counts()[1]))<jupyter_output>Data after upsampling: 2239 samples.
Data after upsampling: 2239 samples.<jupyter_text>Training PipelineThe goal is to create a function that trains, scores and predicts any model. This will allow for efficient model selection since we can supply the function with multiple models and compare the results to select the most appropriate one. Function Inputs> model: the classifier to be trained > X_train y_train X_test y_test, X and y>     Function Output- result which is an array of the prediction time and scores respectively<jupyter_code>def train_predict_pipeline(model, X_train, y_train, X_test, y_test, X, y):
    print("            {} Training            ".format(model.__class__.__name__))
    results = {}

    start = time()  # Training start
    model = model.fit(X_train, y_train) # Train the model
    end = time()  # Training end
    results['training_time'] = end - start # Store the time

    start = time()  # Prediction start
    predictions_test = model.predict(X_test) # Predict
    predictions_train = model.predict(X_train)
    end = time()  # Prediction end
    results['prediction_time'] = end - start # Store the time


    results['model_accuracy'] = model.score(X_train, y_train) # Overall accuracy

    # Cross validation score
    cross_validation_scores = cross_val_score(model,X, y, cv=10)
    results['model_cross_validation'] = np.mean(cross_validation_scores) 

    # Accuracy scores - for plotting
    results['accuracy_train'] = accuracy_score(y_train, predictions_train)
    results['accuracy_test'] = accuracy_score(y_test, predictions_test)

    # F-scores
    results['fbeta_train'] = fbeta_score(y_train, predictions_train, beta=0.5)
    results['fbeta_test'] = fbeta_score(y_test, predictions_test, beta=0.5)

    # Print the report
    print('            Accuracy Report            ')
    print('Model Accuracy: %.2f' % results['model_accuracy'])
    print('10-Fold Cross Validation: %.2f' % results['model_cross_validation'])
    print('F-beta Score (Training): %.2f' % results['fbeta_train'])
    print('F-beta Score (Testing): %.2f' % results['fbeta_test'])
    print('            Confusion Matrix            ')
    print(confusion_matrix(y_test, predictions_test))
    print(classification_report(y_test, predictions_test))

    display = ConfusionMatrixDisplay.from_estimator(model, X_test, y_test, display_labels=['negative', 'positive'], cmap=plt.cm.Blues)
    display.ax_.set_title('Confusion Matrix Display')
    plt.show()

    # Return the results and the classifier
    return results, model

SVC_classifier = SVC(random_state=0,probability=True)
AdaBoost_classifier = AdaBoostClassifier(random_state=0)
Naivebayes_classifier = MultinomialNB()
results_raw = {}

for classifier in [SVC_classifier, AdaBoost_classifier, Naivebayes_classifier]:
    classifier_name = classifier.__class__.__name__
    results_raw[classifier_name] = {}
    results_raw[classifier_name], classifier = train_predict_pipeline(
classifier, X_train_unsampled, y_train_unsampled, X_test_unsampled, y_test_unsampled, unsampled_X, y)
SVC_classifier = SVC(random_state=0, probability=True)
AdaBoost_classifier = AdaBoostClassifier(random_state=0)
Naivebayes_classifier = MultinomialNB()
results = {}

for classifier in [SVC_classifier, AdaBoost_classifier, Naivebayes_classifier]:
    classifier_name = classifier.__class__.__name__
    results[classifier_name] = {}
    results[classifier_name], classifier = train_predict_pipeline(
        classifier, X_train_upsampled, y_train_upsampled, X_test_upsampled, y_test_upsampled, upsampled_X, y)<jupyter_output>SVC Training            
            Accuracy Report            
Model Accuracy: 1.00
10-Fold Cross Validation: 0.94
F-beta Score (Training): 1.00
F-beta Score (Testing): 0.94
            Confusion Matrix            
[[  4  70]
 [  1 949]]
              precision    recall  f1-score   support

           0       0.80      0.05      0.10        74
           1       0.93      1.00      0.96       950

    accuracy                           0.93      1024
   macro avg       0.87      0.53      0.53      1024
weighted avg       0.92      0.93      0.90      1024<jupyter_text>Model Evaluation<jupyter_code>results
def evaluate(results, accuracy, f1):
    """
    Visualization code to display results of various learners.

    inputs:
      - learners: a list of supervised learners
      - stats: a list of dictionaries of the statistic results from 'train_predict()'
      - accuracy: The score for the naive predictor
      - f1: The score for the naive predictor
    """

    # Create figure
    fig, ax = plt.subplots(2, 3, figsize=(15, 10))

 # Constants
    bar_width = 0.3
    colors = ['#083471', '#1F6EB3', '#56A0CE']

    for k, learner in enumerate(results.keys()):
        for j, metric in enumerate(['training_time', 'accuracy_train', 'fbeta_train', 'prediction_time', 'accuracy_test', 'fbeta_test']):
            ax[j//3, j % 3].bar(k*bar_width, results[learner]
                                [metric], width=bar_width, color=colors[k])

    # Add unique y-labels
    ax[0, 0].set_ylabel("Time (in seconds)")
    ax[0, 1].set_ylabel("Accuracy Score")
    ax[0, 2].set_ylabel("F-score")
    ax[1, 0].set_ylabel("Time (in seconds)")
    ax[1, 1].set_ylabel("Accuracy Score")
    ax[1, 2].set_ylabel("Fbeta-score")

    # Add titles
    ax[0, 0].set_title("Model Training")
    ax[0, 1].set_title("Accuracy Score on Training Subset")
    ax[0, 2].set_title("F-score on Training Subset")
    ax[1, 0].set_title("Model Predicting")
    ax[1, 1].set_title("Accuracy Score on Testing Set")
    ax[1, 2].set_title("Fbeta-score on Testing Set")

    # Add horizontal lines for naive predictors
    ax[0, 1].axhline(y=accuracy, xmin=-0.1, xmax=3.0,
                     linewidth=1, color='k', linestyle='dashed')
    ax[1, 1].axhline(y=accuracy, xmin=-0.1, xmax=3.0,
                     linewidth=1, color='k', linestyle='dashed')
    ax[0, 2].axhline(y=f1, xmin=-0.1, xmax=3.0, linewidth=1,
                     color='k', linestyle='dashed')
    ax[1, 2].axhline(y=f1, xmin=-0.1, xmax=3.0, linewidth=1,
                     color='k', linestyle='dashed')

    # Set y-limits for score panels
    ax[0, 1].set_ylim((0, 1))
    ax[0, 2].set_ylim((0, 1))
    ax[1, 1].set_ylim((0, 1))
    ax[1, 2].set_ylim((0, 1))

    # Create patches for the legend
    patches = []
    for i, learner in enumerate(results.keys()):
        patches.append(mpatches.Patch(color=colors[i], label=learner))
    plt.legend(handles=patches, bbox_to_anchor=(-.80, 2.53),
               loc='upper center', borderaxespad=0., ncol=3, fontsize='x-large')

    # Aesthetics
    plt.suptitle(
        "Performance Metrics for Three Supervised Learning Models", fontsize=16, y=1.10)
    plt.show()

evaluate(results, 0.5, 0.5)

svc_display = plot_roc_curve(SVC_classifier, X_test_upsampled, y_test_upsampled)
ada_display = plot_roc_curve(AdaBoost_classifier, X_test_upsampled, y_test_upsampled, ax=svc_display.ax_)
naive_display = plot_roc_curve(Naivebayes_classifier, X_test_upsampled, y_test_upsampled, ax=ada_display.ax_)
naive_display.figure_.suptitle("ROC curve comparison")

with warnings.catch_warnings():
    warnings.simplefilter("ignore")

plt.show()<jupyter_output>C:\Users\dalal\AppData\Roaming\Python\Python39\site-packages\sklearn\utils\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metric.RocCurveDisplay.from_predictions` or :meth:`sklearn.metric.RocCurveDisplay.from_estimator`.
  warnings.warn(msg, category=FutureWarning)
C:\Users\dalal\AppData\Roaming\Python\Python39\site-packages\sklearn\utils\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metric.RocCurveDisplay.from_predictions` or :meth:`sklearn.metric.RocCurveDisplay.from_estimator`.
  warnings.warn(msg, category=FutureWarning)
C:\Users\dalal\AppData\Roaming\Python\Python39\site-packages\sklearn\utils\deprecation.py:87: FutureWarning: Function plot_roc_curve is deprecated; Function :func:`[...]<jupyter_text>Fine Tuning<jupyter_code>parameters = {'C': [0.1, 1, 1.1, 2, 10, 100],
              'kernel': ['linear']}
scorer = make_scorer(fbeta_score, beta=0.5)
grid = GridSearchCV(SVC(random_state=0), param_grid=parameters, scoring=scorer, refit=True, cv=10, verbose=3)
gridSVC = grid.fit(X_train_upsampled, y_train_upsampled)
gridSVC.best_estimator_
# Original SVC accuracy
results['SVC']['accuracy_test']
# Original SVC fbeta score
results['SVC']['fbeta_test']
# Fine tuned SVC accuracy
accuracy_score(y_test_upsampled, gridSVC.predict(X_test_upsampled))
# Fine tuned SVC fbeta score
fbeta_score(y_test_upsampled, gridSVC.predict(X_test_upsampled), beta=0.5)
train_predict_pipeline(SVC(C=1, kernel='linear', random_state=0), X_train_upsampled, y_train_upsampled, X_test_upsampled, y_test_upsampled, upsampled_X, y)<jupyter_output>SVC Training            
            Accuracy Report            
Model Accuracy: 1.00
10-Fold Cross Validation: 0.94
F-beta Score (Training): 1.00
F-beta Score (Testing): 0.95
            Confusion Matrix            
[[ 15  59]
 [  3 947]]
              precision    recall  f1-score   support

           0       0.83      0.20      0.33        74
           1       0.94      1.00      0.97       950

    accuracy                           0.94      1024
   macro avg       0.89      0.60      0.65      1024
weighted avg       0.93      0.94      0.92      1024
{'fraction_non_alphanumeric': 0.1123015873015873, 'fraction_numerical': 0.02896825396825397, 'mean_word_length': 4.143265306122449, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '493931', 'license': 'MIT', 'n_tokens_mistral': 4541, 'n_tokens_neox': 4236, 'n_words': 922, 'path': 'tweets-analysis.ipynb', 'repo_name': 'dalalbinhumaid/expo-2020-sentiment-analysis', 'size': '12600'}