<jupyter_start><jupyter_text>BDDA Assignment question 1<NAME>; 015031BDA-01  Text Classification of Corona Tweets - NLP<jupyter_code>from pyspark import SparkContext
#Creating Spark Session
from pyspark.sql import SparkSession
#Initializing the TextClassifier app
spark= SparkSession.builder.appName('TextClassifierApp').getOrCreate()
#Reading file
df = spark.read.csv('Corona_NLP_train.csv', header=True, inferSchema=True)<jupyter_output><empty_output><jupyter_text>About the data<jupyter_code>df.show()
df.columns
df.printSchema()
#dropping the duplicate values
df = df.dropDuplicates()
print(df.count())
df.toPandas()['Sentiment'].isnull().sum()
df = df.dropna(subset=('Sentiment'))
#filtering out data in the dataset
df = df.filter(df.Sentiment.isin(sentiments))
#defining the list of different sentiments
sentiments = ['Positive','Negative','Neutral','Extremely Positive','Extremely Negative']<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code>#Computes the character length of string data
from pyspark.sql.functions import length
#Viewing the length of each and every tweet
df=df.withColumn('length', length(df['OriginalTweet']))
#Viewing the length of each and every tweet
df.show()
#renaming the column
df=df.withColumnRenamed("Sentiment","sentiment")
#calculating the mean length of different sentiments
df.groupby('Sentiment').mean().show()<jupyter_output>+------------------+------------------+
|         Sentiment|       avg(length)|
+------------------+------------------+
|Extremely Negative| 209.6656891495601|
|          Positive|193.66195905675045|
|           Neutral| 151.2949846860643|
|          Negative| 189.6651596908269|
|Extremely Positive| 215.0605167724388|
+------------------+------------------+<jupyter_text>Processing<jupyter_code>#Performing Tokeization
from pyspark.ml.feature import Tokenizer, StopWordsRemover, CountVectorizer, IDF, StringIndexer

tokenizer=Tokenizer(inputCol="OriginalTweet", outputCol="token_text")
#Removing Stopwords from the comments using count vectorizer, inverse document frequency and string indexer to transform the text
stopremove=StopWordsRemover(inputCol="token_text", outputCol="stop_tokens")
count_vec=CountVectorizer(inputCol="stop_tokens", outputCol="c_vec")
idf=IDF(inputCol="c_vec", outputCol="tf_idf")
# converting the labels in numbers
label_to_num = StringIndexer(inputCol="sentiment", outputCol='label')
#feature transformer that merges multiple columns into a vector column.
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.linalg import Vector
cleaned = VectorAssembler(inputCols=['tf_idf','length'], outputCol='features')<jupyter_output><empty_output><jupyter_text>Creating Model<jupyter_code>from pyspark.ml.classification import DecisionTreeClassifier
#instantitating decision tree classifier

dtc=DecisionTreeClassifier()<jupyter_output><empty_output><jupyter_text>Creating Pipeline<jupyter_code>from pyspark.ml import Pipeline

#building a pieline for pre-processing the text 

pipeline= Pipeline(stages=[label_to_num, tokenizer, stopremove, count_vec, idf, cleaned])
#fitting the model

cleaned_data = pipeline.fit(df)
#transforming the data

cleaned_data = cleaned_data.transform(df)
#extracting labels and features 

cleaned_data = cleaned_data.select(['label', 'features'])
#viewing the selected columns

cleaned_data.show()<jupyter_output>+-----+--------------------+
|label|            features|
+-----+--------------------+
|  3.0|(78306,[0,3,6,12,...|
|  0.0|(78306,[0,3,6,51,...|
|  2.0|(78306,[398,10236...|
|  4.0|(78306,[4,7,33,34...|
|  3.0|(78306,[7,10,36,5...|
|  0.0|(78306,[0,1,10,23...|
|  1.0|(78306,[1,5,8,21,...|
|  2.0|(78306,[3,6,88,99...|
|  0.0|(78306,[1,7,9,11,...|
|  0.0|(78306,[13,27,48,...|
|  1.0|(78306,[8,21,28,3...|
|  2.0|(78306,[0,3,57,71...|
|  4.0|(78306,[4,8,17,21...|
|  3.0|(78306,[0,7,17,19...|
|  0.0|(78306,[0,3,6,70,...|
|  3.0|(78306,[0,3,6,16,...|
|  2.0|(78306,[0,3,6,72,...|
|  2.0|(78306,[16,17,117...|
|  1.0|(78306,[10,96,98,...|
|  1.0|(78306,[5,7,9,11,...|
+-----+--------------------+
only showing top 20 rows<jupyter_text>Training the Model<jupyter_code>#Spilting the data into train and test

(training, testing)=cleaned_data.randomSplit([0.7,0.3])<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>#fitting the model on the dataset

dtpredictor = dtc.fit(training)
#transforming the model

test_results = dtpredictor.transform(testing)
#viewing the results of the test data
test_results.show()<jupyter_output>+-----+--------------------+--------------------+--------------------+----------+
|label|            features|       rawPrediction|         probability|prediction|
+-----+--------------------+--------------------+--------------------+----------+
|  0.0|(78306,[0,1,5,7,1...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[0,3,6,51,...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[0,4,16,31...|[186.0,51.0,8.0,2...|[0.37575757575757...|       3.0|
|  0.0|(78306,[0,18,51,5...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[2,18,92,2...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[5,16,19,1...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[5,16,50,1...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[36,102,13...|[3008.0,2728.0,13...|[0.28710508733416...|       0.0|
|  0.0|(78306,[45,262,26...|[1521.0,1356.0,22...|[0.25575920632251...|       2.0|
|  1.0|(78306,[0[...]<jupyter_text>Importing Evaluator<jupyter_code>from pyspark.ml.evaluation import MulticlassClassificationEvaluator 
#instantiating the classfication evaluator
acc_eval=MulticlassClassificationEvaluator()
#evaluating the accuracy of the model
acc=acc_eval.evaluate(test_results)
print ("Accuracy of the model is::", acc)<jupyter_output>Accuracy of the model is:: 0.2921386154999538
{'fraction_non_alphanumeric': 0.225050916496945, 'fraction_numerical': 0.12763068567549218, 'mean_word_length': 4.01531914893617, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '70075', 'license': 'Apache-2.0', 'n_tokens_mistral': 2636, 'n_tokens_neox': 2199, 'n_words': 343, 'path': 'Text Classification of Corona Tweets - NLP.ipynb', 'repo_name': 'divikmathur/BDDA-UCR-Projects', 'size': '5898'}