<jupyter_start><jupyter_text>Machine Learning: Classification Predict Team 4  Problem Statement:  Global Warming Sentiment Analysis using Twitter Data  Import libraries and datasets<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import re<jupyter_output><empty_output><jupyter_text>Train dataset<jupyter_code>train_df = pd.read_csv('data/train.csv')
train_df.head()<jupyter_output><empty_output><jupyter_text>Test dataset<jupyter_code>test_df = pd.read_csv('data/test.csv')
test_df.head()<jupyter_output><empty_output><jupyter_text>Sample solution<jupyter_code>ss = pd.read_csv('data/sample_submission.csv')
ss.head()<jupyter_output><empty_output><jupyter_text>1. Data Cleaning<jupyter_code>df = train_df.copy()
df.head()
def clean_text(df):
    data = df.copy()
    # Removing mentions and hashtags
    data.message = data.message.apply(lambda x: re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)","",x))
    # Removing https/http links
    data.message = data.message.apply(lambda x: re.sub('http[s]?://\S+', '', x))
    # Replacing non words or punctuation with spaces
    data.message = data.message.apply(lambda x: re.sub('\W',' ',x))
    # Removing numbers
    data.message = data.message.apply(lambda x: re.sub('\d+','',x.lower()))
    return data
df = clean_text(df)
df.head()
test = clean_text(test_df)
test.head()<jupyter_output><empty_output><jupyter_text>1.1 Vectorize<jupyter_code>def vectorize_features(df,v):
    data = df.copy()
    messages = list(data.message)
    train_vector_x = v.fit_transform(messages)
    return train_vector_x
def vectorize_test(df,v):
    data = df.copy()
    messages = list(data.message)
    train_vector_x = v.transform(messages)
    return train_vector_x
from sklearn.feature_extraction.text import CountVectorizer
v = CountVectorizer()

train_vectors_X = vectorize_features(df,v)
test_vectors_X = vectorize_test(test,v)
train_y = list(df.sentiment)<jupyter_output><empty_output><jupyter_text>3. Model Training: Classification  Fitting Model<jupyter_code>from sklearn import svm
clf_svm = svm.SVC(kernel='linear')
clf_svm.fit(train_vectors_X,train_y)
sentiment_pred = clf_svm.predict(test_vectors_X)
sentiment_pred
# Predicted sentiment results
test['sentiment'] = list(sentiment_pred)
test.head()
# Sample Solution results
ss.head()
from sklearn.metrics import f1_score
f1_score(list(ss.sentiment), list(sentiment_pred), average='macro')
from sklearn.linear_model import LogisticRegression
clf_log = LogisticRegression()
clf_log.fit(train_vectors_X,train_y)
y_pred = clf_log.predict(test_vectors_X)
f1_score(list(ss.sentiment), list(y_pred), average='macro')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13213885778275475, 'fraction_numerical': 0.004479283314669653, 'mean_word_length': 6.570621468926554, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '366305', 'license': 'MIT', 'n_tokens_mistral': 991, 'n_tokens_neox': 970, 'n_words': 167, 'path': 'Classification Predict Team 4 - Draft.ipynb', 'repo_name': 'explore-thulani/classification-predicts-team4', 'size': '2679'}