<jupyter_start><jupyter_text>Data Cleaning & Feature Engineering<jupyter_code>train['text_clean'] = train['text']
test['text_clean'] = test['text']                              
# cleaning: lowercase
train['text_clean'] = train['text_clean'].apply(lambda x: x.lower())
test['text_clean'] = test['text_clean'].apply(lambda x: x.lower())

# cleaning: expand contractions
train['text_clean'] = train['text_clean'].apply(lambda x: contractions.fix(x))
test['text_clean'] = test['text_clean'].apply(lambda x: contractions.fix(x))

# cleaning: remove accents and special characters
train['text_clean'] = (train['text_clean']
                       .str.normalize('NFKD')
                       .str.encode('ascii', errors='ignore')
                       .str.decode('utf-8'))
test['text_clean'] = (test['text_clean']
                      .str.normalize('NFKD')
                      .str.encode('ascii', errors='ignore')
                      .str.decode('utf-8'))

# cleaning: remove URLS
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'https?:\/\/t.co\/[\w]+', '', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'https?:\/\/t.co\/[\w]+', '', x))

# cleaning: remove digits
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'[\d+,*\d+]', '', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'[\d+,*\d+]', '', x))

# cleaning: remove stopwords
stop = stopwords.words('english')
train['text_clean'] = train['text_clean'].apply(lambda x: ' '.join(
    [word for word in x.split() if word not in stop]))
test['text_clean'] = test['text_clean'].apply(lambda x: ' '.join(
    [word for word in x.split() if word not in stop]))  
## Initially, the location was used as a feature. However, this adversely
## affected the model, which is why it is excluded.

# import pycountry
# from sklearn.preprocessing import LabelEncoder
#
# country_list = [country.name for country in pycountry.countries]
# sub_country_list = [subcountry.name for subcountry in pycountry.subdivisions]
# country_list.extend(sub_country_list)

# country_list2 = '|'.join(country_list)

# loc_list_train = []
# for entry in train['text_clean']:
#     match =  re.search(country_list2, entry)
#     if match:
#         loc_list_train.append(match.group())
#     else:
#         loc_list_train.append('Other')
        
# loc_list_test = []
# for entry in test['text_clean']:
#     match =  re.search(country_list2, entry)
#     if match:
#         loc_list_test.append(match.group())
#     else:
#         loc_list_test.append('Other')
        
# train['location_extract'] = loc_list_train
# test['location_extract'] = loc_list_test

# train['location_final'] = train['location'].combine_first(train['location_extract'])
# train['location_final'] = (train['location_final']
#                            .apply(lambda x: x.split(',')[0])
#                            .apply(lambda x: x.lower())
#                            .apply(lambda x: re.sub(r'[_\W]', ' ', x)))

# test['location_final'] = test['location'].combine_first(test['location_extract'])
# test['location_final'] = (test['location_final']
#                           .apply(lambda x: x.split(',')[0])
#                           .apply(lambda x: x.lower())
#                           .apply(lambda x: re.sub(r'[_\W]', ' ', x)))

# train['location_n'] = LabelEncoder().fit_transform(train.loc[:,'location_final'])
# test['location_n'] = LabelEncoder().fit_transform(test.loc[:,'location_final'])
# tweet tokenizer
train['tweet_tokens'] = train['text_clean'].apply(
    lambda x: TweetTokenizer().tokenize(x))
test['tweet_tokens'] = test['text_clean'].apply(
    lambda x: TweetTokenizer().tokenize(x))
# cleaning: remove mentions
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'@[\w]+', '', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'@[\w]+', '', x))

# cleaning: remove punctuation
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'[_\W]', ' ', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'[_\W]', ' ', x))

# cleaning: create tokens
train['text_tokens'] = train['text_clean'].apply(lambda x: word_tokenize(x))
test['text_tokens'] = test['text_clean'].apply(lambda x: word_tokenize(x))

# remove unicode characters
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'[^\x00-\x7F]+', '', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'[^\x00-\x7F]+', '', x))

# remove extra spaces
train['text_clean'] = train['text_clean'].apply(lambda x: re.sub(r'\s+', ' ', x))
test['text_clean'] = test['text_clean'].apply(lambda x: re.sub(r'\s+', ' ', x))
train['text_clean'] = train['text_clean'].str.strip()
test['text_clean'] = test['text_clean'].str.strip()
# spelling correction: word length
def reduce(text):
    pattern = re.compile(r'(.)\1{3,}')
    return pattern.sub(r'\1\1', text)

train['text_clean'] = train['text_clean'].apply(lambda x: reduce(x))
test['text_clean'] = test['text_clean'].apply(lambda x: reduce(x))
train['ch_n'] = train['text'].str.len()
test['ch_n'] = test['text'].str.len()

train['digit_n'] = train['text'].apply(lambda row: len(re.findall(r'(\d)', row)))
test['digit_n'] = test['text'].apply(lambda row: len(re.findall(r'(\d)', row)))

train['alpha_n'] = train['text'].apply(lambda row: len(re.findall(r'(\w)', row)))
test['alpha_n'] = test['text'].apply(lambda row: len(re.findall(r'(\w)', row)))

train['non_alpha_n'] = train['text'].apply(lambda row: len(re.findall(r'(\W)', row)))
test['non_alpha_n'] = test['text'].apply(lambda row: len(re.findall(r'(\W)', row)))

train['word_n'] = train['text'].apply(lambda x: len(str(x).split()))
test['word_n'] = test['text'].apply(lambda x: len(str(x).split()))

train['hashtag_n'] = train['text'].apply(lambda row: len(re.findall(r'#(\w)', row)))
test['hashtag_n'] = test['text'].apply(lambda row: len(re.findall(r'#(\w)', row)))

train['mention_n'] = train['text'].apply(lambda row: len(re.findall(r'@(\w)', row)))
test['mention_n'] = test['text'].apply(lambda row: len(re.findall(r'@(\w)', row)))

train['url_n'] = train['text'].apply(lambda row: len(re.findall(r'(http)', row)))
test['url_n'] = test['text'].apply(lambda row: len(re.findall(r'(http)', row)))

train['clean_ch_n'] = train['text_clean'].str.len()
test['clean_ch_n'] = test['text_clean'].str.len()

train['word_tweet_n'] = train['tweet_tokens'].str.len()
test['word_tweet_n'] = test['tweet_tokens'].str.len()

train['word_token_n'] = train['text_tokens'].str.len()
test['word_token_n'] = test['text_tokens'].str.len()
train.head()
train.tail()
plt.figure(figsize=[14,14])

ax1 = plt.subplot(4,3,1)
ax1.hist(train[train['target'] == 0]['ch_n'], bins=15, label="Not Disaster")
ax1.hist(train[train['target'] == 1]['ch_n'], bins=15, label="Disaster")
ax1.legend()
ax1.set_title('ch_n')
ax1.set_ylabel('tweets')

ax2 = plt.subplot(4,3,2)
ax2.hist(train[train['target'] == 0]['digit_n'], bins=15, label="Not Disaster")
ax2.hist(train[train['target'] == 1]['digit_n'], bins=15, label="Disaster")
ax2.set_xlim([0, 15])
ax2.legend()
ax2.set_title('digit_n')

ax3 = plt.subplot(4,3,3)
ax3.hist(train[train['target'] == 0]['alpha_n'], bins=15, label="Not Disaster")
ax3.hist(train[train['target'] == 1]['alpha_n'], bins=15, label="Disaster")
ax3.legend()
ax3.set_title('alpha_n')

ax4 = plt.subplot(4,3,4)
ax4.hist(train[train['target'] == 0]['non_alpha_n'], bins=15, label="Not Disaster")
ax4.hist(train[train['target'] == 1]['non_alpha_n'], bins=15, label="Disaster")
ax4.set_xlim([0, 50])
ax4.legend()
ax4.set_title('non_alpha_n')
ax4.set_ylabel('tweets')

ax5 = plt.subplot(4,3,5)
ax5.hist(train[train['target'] == 0]['hashtag_n'], bins=10, label="Not Disaster")
ax5.hist(train[train['target'] == 1]['hashtag_n'], bins=10, label="Disaster")
ax5.set_xlim([0, 6])
ax5.legend()
ax5.set_title('hashtag_n')

ax6 = plt.subplot(4,3,6)
ax6.hist(train[train['target'] == 0]['mention_n'], bins=10, label="Not Disaster")
ax6.hist(train[train['target'] == 1]['mention_n'], bins=10, label="Disaster")
ax6.set_xlim([0, 6])
ax6.legend()
ax6.set_title('mention_n')

ax7 = plt.subplot(4,3,7)
ax7.hist(train[train['target'] == 0]['url_n'], bins=5, label="Not Disaster")
ax7.hist(train[train['target'] == 1]['url_n'], bins=5, label="Disaster")
ax7.legend()
ax7.set_title('url_n')
ax7.set_ylabel('tweets')

ax8 = plt.subplot(4,3,8)
ax8.hist(train[train['target'] == 0]['clean_ch_n'], bins=10, label="Not Disaster")
ax8.hist(train[train['target'] == 1]['clean_ch_n'], bins=10, label="Disaster")
ax8.legend()
ax8.set_title('clean_ch_n')
ax8.set_xlabel('count')

ax9 = plt.subplot(4,3,9)
ax9.hist(train[train['target'] == 0]['word_tweet_n'], bins=10, label="Not Disaster")
ax9.hist(train[train['target'] == 1]['word_tweet_n'], bins=10, label="Disaster")
ax9.set_xlim([0, 30])
ax9.legend()
ax9.set_title('word_tweet_n')
ax9.set_xlabel('count')

ax10 = plt.subplot(4,3,10)
ax10.hist(train[train['target'] == 0]['word_token_n'], bins=10, label="Not Disaster")
ax10.hist(train[train['target'] == 1]['word_token_n'], bins=10, label="Disaster")
ax10.legend()
ax10.set_title('word_token_n')
ax10.set_xlabel('count')

plt.tight_layout()
plt.show()
feature_exploration = ['ch_n', 'digit_n', 'alpha_n', 'non_alpha_n', 'hashtag_n', 
                       'mention_n', 'url_n', 'clean_ch_n', 'word_tweet_n', 
                       'word_token_n', 'target']

train[feature_exploration].groupby('target').mean()<jupyter_output><empty_output><jupyter_text>Model Training<jupyter_code>features = ['ch_n', 'mention_n', 'url_n', 'word_tweet_n', 'text_clean']

X_train, X_test, y_train, y_test = train_test_split(train[features], train['target'], test_size=0.25)
def add_feature(X, feature_to_add):
    '''
    Returns sparse feature matrix with added feature.
    feature_to_add can also be a list of features.
    '''
    return hstack([X, csr_matrix(feature_to_add).T], 'csr')
vect = TfidfVectorizer(max_df=0.90, min_df=10, ngram_range=(1,3))
vect.fit(X_train['text_clean'])

X_train_vect = vect.transform(X_train['text_clean'])
X_test_vect = vect.transform(X_test['text_clean'])

features_vect = ['ch_n', 'mention_n', 'url_n', 'word_tweet_n']

for feature in features_vect:
    X_train_vect = add_feature(X_train_vect, X_train[feature])
    X_test_vect = add_feature(X_test_vect, X_test[feature])<jupyter_output><empty_output><jupyter_text>Naive Bayes Classifier<jupyter_code>model_bayes = MultinomialNB().fit(X_train_vect, y_train)
predictions_bayes = model_bayes.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_bayes)}')
print(f'f1: {f1_score(y_test, predictions_bayes)}')
alpha =  np.linspace(0.1, 10, 100)
param_bayes = [{'alpha': alpha}]

grid_bayes = GridSearchCV(MultinomialNB(), param_bayes, scoring='f1', cv=5)
grid_bayes.fit(X_train_vect, y_train)
best_alpha = grid_bayes.best_params_['alpha']

grid_bayes.best_estimator_
model_bayes_final = MultinomialNB(alpha=best_alpha)
model_bayes_final.fit(X_train_vect, y_train)
predictions_bayes_final = model_bayes_final.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_bayes_final)}')
print(f'f1: {f1_score(y_test, predictions_bayes_final)}')<jupyter_output>accuracy: 0.7888655462184874
f1: 0.7365661861074705<jupyter_text>Logistic Regression<jupyter_code>model_log = LogisticRegression(max_iter=5000)
model_log.fit(X_train_vect, y_train)

predictions_log = model_log.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_log)}')
print(f'f1: {f1_score(y_test, predictions_log)}')
param_log = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],
            'solver': ['newton-cg', 'lbfgs', 'liblinear']}

grid_log = GridSearchCV(LogisticRegression(max_iter=5000), param_log, scoring='f1')
grid_log.fit(X_train_vect, y_train)
best_C_log = grid_log.best_params_['C']
grid_log.best_estimator_
model_log_final = LogisticRegression(C=best_C_log, max_iter=4500)
model_log_final.fit(X_train_vect, y_train)
predictions_log_final = model_log_final.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_log_final)}')
print(f'f1: {f1_score(y_test, predictions_log_final)}')<jupyter_output>accuracy: 0.7862394957983193
f1: 0.7382636655948553<jupyter_text>Support Vector Classifier<jupyter_code>model_svm = SVC()
model_svm.fit(X_train_vect, y_train)

predictions_svm = model_svm.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_svm)}')
print(f'f1: {f1_score(y_test, predictions_svm)}')
param_svm = {'C': [0.1, 1, 10, 100], 
              'gamma': [1, 0.1, 0.01, 0.001],
              'kernel': ['rbf', 'sigmoid']}

grid_svm = GridSearchCV(SVC(), param_svm, scoring='f1', verbose=1)
grid_svm.fit(X_train_vect, y_train)
model_svm_final = grid_svm.best_estimator_
model_svm_final.fit(X_train_vect, y_train)

predictions_svm_final = model_svm_final.predict(X_test_vect)

print(f'accuracy: {accuracy_score(y_test, predictions_svm_final)}')
print(f'f1: {f1_score(y_test, predictions_svm_final)}')<jupyter_output>accuracy: 0.7415966386554622
f1: 0.6890012642225031<jupyter_text>Final Predictions<jupyter_code>vect_final = TfidfVectorizer(max_df=0.90, min_df=10, ngram_range=(1,3))
vect_final.fit(test['text_clean'])

X_train_vect_final = vect_final.transform(train['text_clean']) 
X_test_vect_final = vect_final.transform(test['text_clean'])

for feature in features_vect:
    X_train_vect_final = add_feature(X_train_vect_final, train[feature])
    X_test_vect_final = add_feature(X_test_vect_final, test[feature])
model_final = model_log_final
model_final.fit(X_train_vect_final, train['target'])
predictions_final = model_final.predict(X_test_vect_final)
test['target'] = predictions_final<jupyter_output><empty_output><jupyter_text>Exporting Results<jupyter_code>output = pd.DataFrame({'id': test['id'], 'target': test['target']})
output.to_csv('disaster_tweets_submission.csv', index=False)
print('Your submission was successfully saved!')<jupyter_output>Your submission was successfully saved!
{'fraction_non_alphanumeric': 0.19545910328033234, 'fraction_numerical': 0.0275748460106002, 'mean_word_length': 6.201134605466735, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '361828', 'license': 'MIT', 'n_tokens_mistral': 5576, 'n_tokens_neox': 5327, 'n_words': 799, 'path': 'disaster_tweets.ipynb', 'repo_name': 'katrinaalaimo/disaster-tweets', 'size': '13962'}