<jupyter_start><jupyter_text>Decision Tree  Installs and imports  Install all required libraries<jupyter_code># Uncomment line below to install all required libraries
# !pip3 install -r ../requirements.txt -q<jupyter_output><empty_output><jupyter_text>Import required libraries<jupyter_code>import nltk
import re
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier<jupyter_output><empty_output><jupyter_text>Load cleaned tweets dataset<jupyter_code>df = pd.read_csv('./cleaned_tweets.csv')
np.random.seed(450)<jupyter_output><empty_output><jupyter_text>Drop text<jupyter_code>df = df[['sentiment', 'Snowball_Stem']]
df.head()<jupyter_output><empty_output><jupyter_text>Drop rows with NaN<jupyter_code>df.isna().sum()
df = df.dropna()
df.isna().sum()
X= df['Snowball_Stem']
y = df['sentiment']<jupyter_output><empty_output><jupyter_text>Reduce dataframe size<jupyter_code>reduced_df = pd.concat([df[df.sentiment != 0][:700000], df[df.sentiment == 0][:700000]])
reduced_df.shape
rX= reduced_df['Snowball_Stem']
ry = reduced_df['sentiment']
X_train, X_test, y_train, y_test = train_test_split(rX, ry)
X_train.shape, X_test.shape, y_train.shape, y_test.shape<jupyter_output><empty_output><jupyter_text>Unigram and bigram vectorizer<jupyter_code>v= TfidfVectorizer(ngram_range = (1, 2))
v.fit(X)
xtrain = v.transform(X_train)
xtest = v.transform(X_test)<jupyter_output><empty_output><jupyter_text>Encoding labels<jupyter_code>Encoder = LabelEncoder()
y_train = Encoder.fit_transform(y_train)
y_test = Encoder.fit_transform(y_test)<jupyter_output><empty_output><jupyter_text>DT<jupyter_code>dt = DecisionTreeClassifier()
dt.fit(xtrain,y_train)
dt_pred = dt.predict(xtest)
accuracy_score(dt_pred,y_test)<jupyter_output><empty_output><jupyter_text>Saving the model<jupyter_code>import pickle
DT_path = './DT_model.pickle'
pickle.dump(dt, open(DT_path, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11455108359133127, 'fraction_numerical': 0.008845643520566122, 'mean_word_length': 10.034146341463414, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '276593', 'license': 'MIT', 'n_tokens_mistral': 867, 'n_tokens_neox': 838, 'n_words': 138, 'path': 'Model/DT.ipynb', 'repo_name': 'Varun487/NLP_Live_Tweet_Sentiment_Analysis', 'size': '2261'}