<jupyter_start><jupyter_text><jupyter_code>!pip install pyspark

import sys
from pyspark.sql import SparkSession
from pyspark.sql.functions import count
from pyspark import SparkFiles
#file_url="https://raw.githubusercontent.com/databricks/LearningSparkV2/master/chapter2/py/src/data/mnm_dataset.csv"
spark = SparkSession.builder.appName("My cokies").getOrCreate()
#spark.sparkContext.addFile(file_url)

mnm_df = spark.read.format("csv").option("header", "true").option("inferSchema", "true").load("/content/mnm_dataset.csv")
#mnm_df = spark.read.csv("file:///content/mnm_dataset.csv", header=True, inferSchema= True)

mnm_df.head(2)
state_wise_counts= mnm_df.select('*').groupBy('State','Color').agg(count("Count").alias("Total")).orderBy("Total", ascending=False)

# Show the resulting aggregations for all
state_wise_counts.show(n=30, truncate=False)
print("Total Rows={}".format(state_wise_counts.count()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16276595744680852, 'fraction_numerical': 0.005319148936170213, 'mean_word_length': 8.60204081632653, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '181624', 'license': 'MIT', 'n_tokens_mistral': 345, 'n_tokens_neox': 329, 'n_words': 45, 'path': 'pyspark_mnm_cookie_counts.ipynb', 'repo_name': 'sanat259/PG-DBDA', 'size': '940'}