<jupyter_start><jupyter_text><jupyter_code>import os
# Find the latest version of spark 3.0  from http://www.apache.org/dist/spark/ and enter as the spark version
# For example:
spark_version = 'spark-3.0.3'
# spark_version = 'spark-3.<enter version>'
os.environ['SPARK_VERSION']=spark_version

# Install Spark and Java
!apt-get update
!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q http://www.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz
!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz
!pip install -q findspark

# Set Environment Variables
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = f"/content/{spark_version}-bin-hadoop2.7"

# Start a SparkSession
import findspark
findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("BigData-HW-2").getOrCreate()
from pyspark import SparkFiles
url = "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Video_Games_v1_00.tsv.gz"
spark.sparkContext.addFile(url)
video_games_df = spark.read.csv(SparkFiles.get("amazon_reviews_us_Video_Games_v1_00.tsv.gz"), sep="\t", header=True)
video_games_df.show()
filtered_video_games_df = video_games_df.select(["star_rating", "helpful_votes", "total_votes", "vine", "verified_purchase"])
filtered_video_games_df.show(10)
# Filter for greater than 20 total votes
total_votes_df = filtered_video_games_df.filter(filtered_video_games_df["total_votes"] >= 20)
total_votes_df.show(10)
# Filter for greater than 50% helpful vote percentage
helpful_votes_df = total_votes_df.filter(total_votes_df["helpful_votes"]/total_votes_df["total_votes"] >= 0.5)
helpful_votes_df.show(10)
# Paid reviews
paid_df = helpful_votes_df.filter(helpful_votes_df['vine']== 'Y')
paid_df.describe().show(10)
# Unpaid reviews
unpaid_df = helpful_votes_df.filter(helpful_votes_df['vine']== 'N')
unpaid_df.describe().show()
paid_five_star_number = paid_df[paid_df['star_rating']== 5].count()
paid_number = paid_df.count()
percentage_five_star_vine = round(float(paid_five_star_number) / float(paid_number),4)
print(f'Number of paid reviews {paid_number}')
print(f'Number of paid five star reviews {paid_five_star_number}')
print(f'Percantage of paid reviews that are five stars {percentage_five_star_vine * 100}%')
unpaid_five_star_number = unpaid_df[unpaid_df['star_rating']== 5].count()
unpaid_number = unpaid_df.count()
percentage_five_star_non_vine = round(float(unpaid_five_star_number) / float(unpaid_number),4)
print(f'Number of unpaid reviews {unpaid_number}')
print(f'Number of unpaid five star reviews {unpaid_five_star_number}')
print(f'Percantage of paid reviews that are five stars {percentage_five_star_non_vine * 100}%')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12900874635568513, 'fraction_numerical': 0.018221574344023325, 'mean_word_length': 7.605015673981192, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '593324', 'license': 'ADSL', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1001, 'n_words': 179, 'path': 'level-2/level-2.ipynb', 'repo_name': 'AlexanderLorin/big-data-challenge', 'size': '2744'}