<jupyter_start><jupyter_text>Movie Recommender<jupyter_code>movies = spark.read.format("csv") \
.option('InferSchema','True') \
.option('Header','True') \
.load("movies.csv")
movies.show()
ratings = spark.read.format("csv") \
.option('InferSchema','True') \
.option('Header','True') \
.load("ratings.csv")
ratings.show()
from pyspark.sql.functions import *
user2 = movies.join(ratings, movies.movieId == ratings.movieId) \
.filter(ratings.userId == 2) \
.orderBy(desc("rating"))

user2.show(truncate=False)

print(user2.count())
print(movies.count())
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.recommendation import ALS
from pyspark.sql.functions import *
from pyspark.ml.feature import StringIndexer, VectorAssembler
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
 (training, test) = ratings.randomSplit([0.8, 0.2])
als = ALS(maxIter=5, regParam=0.01, rank=5, userCol="userId", itemCol="movieId", ratingCol="rating",coldStartStrategy="drop")
dir(ALS)
model = als.fit(training)
predictions = model.transform(test)
evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating", predictionCol="prediction")
rmse = evaluator.evaluate(predictions)
 print("Root-mean-square error = " + str(rmse))
dir(model)
userRecs = model.recommendForAllUsers(10)
userRecs.printSchema()
userRecs.show(truncate=False)
 userRec2 = userRecs.filter(userRecs.userId == 2) \
        .withColumn('ratings', explode(userRecs.recommendations)) \
        .select('userId', 'ratings.movieId', 'ratings.rating')
userRec2.join(movies, movies.movieId == userRec2.movieId) \
        .show(truncate=False)<jupyter_output>+------+-------+---------+-------+--------------------------------------------------+--------------------------------+
|userId|movieId|rating   |movieId|title                                             |genres                          |
+------+-------+---------+-------+--------------------------------------------------+--------------------------------+
|2     |7982   |8.559359 |7982   |Tale of Two Sisters, A (Janghwa, Hongryeon) (2003)|Drama|Horror|Mystery|Thriller   |
|2     |70946  |8.168563 |70946  |Troll 2 (1990)                                    |Fantasy|Horror                  |
|2     |48322  |7.472438 |48322  |Jackass Number Two (2006)                         |Comedy|Documentary              |
|2     |5909   |7.3413954|5909   |Visitor Q (Bizita Q) (2001)                       |Comedy|Drama|Horror             |
|2     |5048   |7.3184037|5048   |Snow Dogs (2002)                                  |Adventure|Children|Comedy       |
|2     |6975   |7.270154 |6975   |Funny Games (1[...]
{'fraction_non_alphanumeric': 0.21610639083856667, 'fraction_numerical': 0.05282600664942741, 'mean_word_length': 2.4062893081761008, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '612666', 'license': 'MIT', 'n_tokens_mistral': 954, 'n_tokens_neox': 855, 'n_words': 147, 'path': 'lesson6/examples/resources/SparkML.ipynb', 'repo_name': 'enldobre/das_2021', 'size': '2707'}