<jupyter_start><jupyter_text>BROADCAST VARIABLES ```Broadcast objects to the executors, such that they're alwaysthere whenever needed``````Just use sc.broadcast() to ship off whatever you want``````pythonThen use.value() to get the object back```  Improved version<jupyter_code>def load_movie_name():
    movie_name = {}
    with open('/home/ubuntu/spark_project/01-basics_andexampl/ml-100k/u.item') as f:
        for line in f:
            fields = line.split('|')
            movie_name[int(fields[0])] = fields[1]
    return movie_name
name_dict = sc.broadcast(load_movie_name())
lines  = sc.textFile('/home/ubuntu/spark_project/01-basics_andexampl/ml-100k/u.data')
movies = lines.map(lambda x: (int(x.split()[1], 1))
movie_count = movies.reduceByKey(lambda x,y: x+y)
sorted_movie_count = movies_counts.sortBy(lambda x: x[1], ascending=False)
sorted_movie_with_name = sorted_movie_count.map(lambda :(name_dict.value[x[0]], x[1]))
results = sorted_movie_with_name.collect()
for result in results:
    print(result[0], " ", result[1])
def loadMovieNames():
    movieNames = {}
    with open("/home/ubuntu/spark_project/01-basics_andexampl/ml-100k/u.item") as f:
        for line in f:
            if isinstance(line, str):
                fields = line.split('|')
                movieNames[int(fields[0])] = fields[1]
    return movieNames
nameDict = sc.broadcast(loadMovieNames())

lines = sc.textFile("/home/ubuntu/spark_project/01-basics_andexampl/ml-100k/u.data")
movies = lines.map(lambda x: (int(x.split()[1], 1)))
movies = lines.map(lambda x: (int(x.split()[1]), 1))
movieCounts = movies.reduceByKey(lambda x, y: x + y)

flipped = movieCounts.map( lambda x : (x[1], x[0]))
sortedMovies = flipped.sortByKey()

sortedMoviesWithNames = sortedMovies.map(lambda countMovie : (nameDict.value[countMovie[1]], countMovie[0]))

results = sortedMoviesWithNames.collect()

for result in results:
    print (result)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1498455200823893, 'fraction_numerical': 0.02008238928939238, 'mean_word_length': 4.567335243553009, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '509745', 'license': 'MIT', 'n_tokens_mistral': 730, 'n_tokens_neox': 673, 'n_words': 136, 'path': '02-Advanced_Examples_of_Spark_Programs/Popular_Movies.ipynb', 'repo_name': 'egbertli/Apache-Spark-Projects-with-Python', 'size': '1942'}