<jupyter_start><jupyter_text>Exercise 3.2 Import the pySpark libraries<jupyter_code>import sys
!conda install --yes --prefix {sys.prefix} -c conda-forge pyspark
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, split, size<jupyter_output><empty_output><jupyter_text>Connect to a local Spark session<jupyter_code>spark = SparkSession.builder.appName("Packt").getOrCreate()<jupyter_output><empty_output><jupyter_text>Read the raw data from a CSV file<jupyter_code>data = spark.read.csv('../../Datasets/netflix_titles_nov_2019.csv', header='true')<jupyter_output><empty_output><jupyter_text>Take only the movies<jupyter_code>movies = data.filter((col('type') == 'Movie') & (col('release_year') == 2019))<jupyter_output><empty_output><jupyter_text>Add a column with the number of actors<jupyter_code>transformed = movies.withColumn('count_cast', size(split(movies['cast'], ',')))<jupyter_output><empty_output><jupyter_text>Select a subset of columns to store<jupyter_code>selected = transformed.select('title', 'director', 'count_cast', 'cast', 'rating', 'release_year', 'type')<jupyter_output><empty_output><jupyter_text>Write the contents of the DataFrame to disk<jupyter_code>selected.write.csv('transformed.csv', header='true')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1402961808261886, 'fraction_numerical': 0.00779423226812159, 'mean_word_length': 10.068965517241379, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '507093', 'license': 'MIT', 'n_tokens_mistral': 457, 'n_tokens_neox': 439, 'n_words': 78, 'path': 'Chapter03/Exercise03.02/spark_etl.ipynb', 'repo_name': 'anand-ns/The-Artificial-Intelligence-Infrastructure-Workshop', 'size': '1283'}