<jupyter_start><jupyter_text>Import Pyspark<jupyter_code>import pyspark 
sc = pyspark.SparkContext('local[*]')<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>%%time
data = sc.textFile('*.csv.bz2')
data.take(3)<jupyter_output>CPU times: user 0 ns, sys: 0 ns, total: 0 ns
Wall time: 137 ms<jupyter_text>Mapping<jupyter_code>%%time
rdd1 = data.map(lambda x: x.split(',')[8]).filter(lambda x : 'UniqueCarrier' not in x)
rdd1.take(10)
rdd2 = rdd1.map(lambda x:(x,1))
rdd2.take(3)<jupyter_output><empty_output><jupyter_text>Shuffling & Reducing<jupyter_code>%%time
rdd4 = rdd2.reduceByKey(lambda x,y: x+y)
rdd4.take(30)<jupyter_output>CPU times: user 80 ms, sys: 30 ms, total: 110 ms
Wall time: 10min 18s
{'fraction_non_alphanumeric': 0.15760111576011157, 'fraction_numerical': 0.04741980474198047, 'mean_word_length': 8.205128205128204, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '200271', 'license': 'MIT', 'n_tokens_mistral': 341, 'n_tokens_neox': 313, 'n_words': 57, 'path': 'Assignment4_docker_sparkcontext.ipynb', 'repo_name': 'spu-bigdataanalytics/assignment4-dray19', 'size': '717'}