<jupyter_start><jupyter_text>Preliminary Set Up<jupyter_code># import general packages
import pandas as pd
import numpy as np
# load in data
training = pd.read_csv('../data/train.csv')
testing = pd.read_csv('../data/test.csv')<jupyter_output><empty_output><jupyter_text>Descriptive Statistics / Data Exploration<jupyter_code># training data size
training.count()
# testing data size
testing.count()
# count number of observations in each class
training['target'].value_counts() # 1 = disaster, 0 = not<jupyter_output><empty_output><jupyter_text>Basic NLP<jupyter_code># split into initial training & testing (Kaggle test data is unlabeled)
from sklearn.model_selection import train_test_split
training_i, testing_i = train_test_split(training, test_size=0.2)<jupyter_output><empty_output><jupyter_text>Naive Bayes<jupyter_code># imported needed functions for Naive Bayes
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
# learn basic text classifier using bag of words & naive bayes
textClassNB = Pipeline([('wordVector', CountVectorizer()),
                      ('tfidf', TfidfTransformer()),
                      ('classify', MultinomialNB()),
                    ])
textClassNB = textClassNB.fit(training_i['text'], training_i['target'])
# test performance of naive bayes classifier
predicted = textClassNB.predict(testing_i['text'])
np.mean(predicted == testing_i['target'])<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code># learn basic text classifier using bag of words & SVM
from sklearn.linear_model import SGDClassifier
textClassSVM = Pipeline([('wordVector',CountVectorizer(stop_words='english')),
                        ('tfidf', TfidfTransformer()),
                        ('classifySVM', 
                         SGDClassifier(loss='hinge',
                                       penalty='l2',
                                       max_iter=15,
                                       tol=1e-3,
                                      alpha=1e-3))])
textClassSVM = textClassSVM.fit(training_i['text'], training_i['target'])
# test performance of SVM
predictedSVM = textClassSVM.predict(testing_i['text'])
np.mean(predictedSVM == testing_i['target'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10495133305120609, 'fraction_numerical': 0.004655099449851883, 'mean_word_length': 3.718562874251497, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '631158', 'license': 'MIT', 'n_tokens_mistral': 724, 'n_tokens_neox': 690, 'n_words': 150, 'path': 'michelle/data-exploration.ipynb', 'repo_name': 'big-sky-country/real-or-not', 'size': '2363'}