<jupyter_start><jupyter_text>Install PySpark<jupyter_code># Install the dependencies
!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://archive.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop3.2.tgz
!tar xf spark-3.0.1-bin-hadoop3.2.tgz
!pip install -q findspark
# Set the environment variables for running PySpark in the collaboration environmentimport os
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-3.0.1-bin-hadoop3.2"
# Run the local session to test the installation
import findspark
findspark.init('spark-3.0.1-bin-hadoop3.2')
from pyspark.sql import SparkSession
spark = SparkSession.builder.master('local[*]').getOrCreate()
df = spark.read.csv('drive/MyDrive/dataset/spam.csv',inferSchema=True, sep=',', header=True)
df.show(5)
df = df.withColumnRenamed('v1','class').withColumnRenamed('v2','text')<jupyter_output><empty_output><jupyter_text>Clean Data<jupyter_code>from pyspark.sql.functions import length
df= df.withColumn('length', length(df['text']))
df.show(5)
df = df.filter((df["class"]=='ham') | (df["class"]=='spam'))
df.groupBy('class').mean().show()<jupyter_output>+-----+------------------+
|class|       avg(length)|
+-----+------------------+
|  ham| 71.07626943005181|
| spam|138.89558232931728|
+-----+------------------+<jupyter_text>Feature Transformation  Creating Assemblers/Features<jupyter_code>from pyspark.ml.feature import Tokenizer, StopWordsRemover, CountVectorizer, IDF, StringIndexer
tokenizer = Tokenizer(inputCol='text',outputCol='token_text')
stop_word_remover = StopWordsRemover(inputCol='token_text',outputCol='stop_tokens')
count_vec = CountVectorizer(inputCol='stop_tokens',outputCol='c_vec')
idf = IDF(inputCol='c_vec',outputCol='tf_idf')
ham_spam_to_num = StringIndexer(inputCol='class',outputCol='label')
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.linalg import Vector
cleaned = VectorAssembler(inputCols=['tf_idf','length'],outputCol='features')<jupyter_output><empty_output><jupyter_text>Naive Bayes Model<jupyter_code>from pyspark.ml.classification import NaiveBayes
nb = NaiveBayes()<jupyter_output><empty_output><jupyter_text>PipeLine<jupyter_code>from pyspark.ml import Pipeline
pipeline = Pipeline(stages=[
                            ham_spam_to_num,
                            tokenizer,
                            stop_word_remover,
                            count_vec,
                            idf,
                            cleaned
])
cleaner = pipeline.fit(df)
clean_df = cleaner.transform(df)
clean_df.show(5)<jupyter_output>+-----+--------------------+------+-----+--------------------+--------------------+--------------------+--------------------+--------------------+
|class|                text|length|label|          token_text|         stop_tokens|               c_vec|              tf_idf|            features|
+-----+--------------------+------+-----+--------------------+--------------------+--------------------+--------------------+--------------------+
|  ham|Go until jurong p...|   111|  0.0|[go, until, juron...|[go, jurong, poin...|(13378,[7,10,31,6...|(13378,[7,10,31,6...|(13379,[7,10,31,6...|
|  ham|Ok lar... Joking ...|    29|  0.0|[ok, lar..., joki...|[ok, lar..., joki...|(13378,[0,23,293,...|(13378,[0,23,293,...|(13379,[0,23,293,...|
| spam|Free entry in 2 a...|   155|  1.0|[free, entry, in,...|[free, entry, 2, ...|(13378,[2,13,19,2...|(13378,[2,13,19,2...|(13379,[2,13,19,2...|
|  ham|U dun say so earl...|    49|  0.0|[u, dun, say, so,...|[u, dun, say, ear...|(13378,[0,68,78,1...|(13378,[0,68,7[...]<jupyter_text>Train Model and Evaluation<jupyter_code>clean_df= clean_df.select(['label','features'])
(train, test) = clean_df.randomSplit([0.7,0.3], seed=42)
pred = nb.fit(train)
res = pred.transform(test)
res.show()
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
eval = MulticlassClassificationEvaluator()
acc = eval.evaluate(res)
print(f'Accuracy: {acc * 100}')<jupyter_output>Accuracy: 91.85519252666279
{'fraction_non_alphanumeric': 0.24820943442825388, 'fraction_numerical': 0.05655717461101507, 'mean_word_length': 2.9092664092664093, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '610964', 'license': 'MIT', 'n_tokens_mistral': 1561, 'n_tokens_neox': 1404, 'n_words': 209, 'path': 'nlp/google_colab_spark_nlp_spam.ipynb', 'repo_name': 'GrishenkovP/Spark', 'size': '4049'}