<jupyter_start><jupyter_text>Dependencies<jupyter_code>import os
# Find the latest version of spark 3.0  from http://www-us.apache.org/dist/spark/ and enter as the spark version
# For example:
spark_version = 'spark-3.0.1'
# spark_version = 'spark-3.<spark version>'
os.environ['SPARK_VERSION'] = spark_version

# Install Spark and Java
!apt-get update
!apt-get install openjdk-11-jdk-headless -qq > /dev/null
!wget -q https://archive.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz
!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz
!pip install -q findspark

# Set Environment Variables
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-11-openjdk-amd64"
os.environ["SPARK_HOME"] = f"/content/{spark_version}-bin-hadoop2.7"

# Start a SparkSession
import findspark
findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("BigData-HW-1").config("spark.driver.extraClassPath","/content/postgresql-42.2.9.jar").getOrCreate()<jupyter_output><empty_output><jupyter_text>Amazon Data<jupyter_code>from pyspark import SparkFiles

# load data
url = "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Video_Games_v1_00.tsv.gz"
spark.sparkContext.addFile(url)
video_games_df = spark.read.csv(SparkFiles.get("amazon_reviews_us_Video_Games_v1_00.tsv.gz"), sep="\t", header=True)
video_games_df.show()
# filter by votes
filtered_video_games_df = video_games_df.select(["star_rating", "helpful_votes", "total_votes", "vine", "verified_purchase"])
filtered_video_games_df.show()
# 20+ votes
total_votes_df = filtered_video_games_df.filter(filtered_video_games_df["total_votes"] >= 20)
total_votes_df.show()
# 50% + helpful vote
helpful_votes_df = total_votes_df.filter(total_votes_df["helpful_votes"]/total_votes_df["total_votes"] >= 0.5)
helpful_votes_df.show()<jupyter_output>+-----------+-------------+-----------+----+-----------------+
|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+-----------+-------------+-----------+----+-----------------+
|          1|           21|         34|   N|                N|
|          1|           21|         35|   N|                Y|
|          1|          147|        175|   N|                Y|
|          2|           55|         60|   N|                N|
|          1|           51|         65|   N|                Y|
|          4|           31|         36|   N|                N|
|          2|           19|         34|   N|                Y|
|          5|           28|         31|   N|                N|
|          2|          151|        198|   N|                N|
|          1|           49|         51|   N|                Y|
|          1|           23|         28|   N|                N|
|          5|           88|        110|   N|                N|
|          2|           99|        119|   N|           [...]<jupyter_text>Describe Data<jupyter_code># paid
paid_df = helpful_votes_df.filter(helpful_votes_df['vine']== 'Y')
paid_df.describe().show()<jupyter_output>+-------+------------------+-----------------+------------------+----+-----------------+
|summary|       star_rating|    helpful_votes|       total_votes|vine|verified_purchase|
+-------+------------------+-----------------+------------------+----+-----------------+
|  count|                94|               94|                94|  94|               94|
|   mean| 4.202127659574468|54.59574468085106|61.787234042553195|null|             null|
| stddev|0.9791348741656414|65.26098459822538| 68.90976994895392|null|             null|
|    min|                 1|              111|               102|   Y|                N|
|    max|                 5|               97|                88|   Y|                N|
+-------+------------------+-----------------+------------------+----+-----------------+<jupyter_text>The mean of Vine reviews was 4.20 with a standard deviation of 0.98.<jupyter_code># unpaid
unpaid_df = helpful_votes_df.filter(helpful_votes_df['vine']== 'N')
unpaid_df.describe().show()<jupyter_output>+-------+------------------+------------------+------------------+-----+-----------------+
|summary|       star_rating|     helpful_votes|       total_votes| vine|verified_purchase|
+-------+------------------+------------------+------------------+-----+-----------------+
|  count|             40471|             40471|             40471|40471|            40471|
|   mean|  3.34765634651973|47.428405524943784|55.891057794470115| null|             null|
| stddev|1.6418850112078023|117.53763370687005|127.40280622961905| null|             null|
|    min|                 1|                10|               100|    N|                N|
|    max|                 5|               999|               999|    N|                Y|
+-------+------------------+------------------+------------------+-----+-----------------+<jupyter_text>The mean of non-Vine reviews was 3.35 with a standard deviation of 1.64, showing greater variance.  Analysis<jupyter_code># unpaid 5 star
unpaid_five_star_number = unpaid_df[unpaid_df['star_rating']== 5].count()
print(f'Number of unpaid five star reviews {unpaid_five_star_number}')
# total unpaid reviews
unpaid_number = unpaid_df.count()
print(f'Number of unpaid reviews {unpaid_number}')
# percent unpaid that are 5 star
percentage_five_star_non_vine = round(float(unpaid_five_star_number) / float(unpaid_number),4)
print(f'Percantage of unpaid reviews that are five stars {percentage_five_star_non_vine * 100}%')
# paid 5 star
paid_five_star_number = paid_df[paid_df['star_rating']== 5].count()
print(f'Number of paid five star reviews {paid_five_star_number}')
# total paid reviews
paid_number = paid_df.count()
print(f'Number of paid reviews {paid_number}')
# percent paid that are 5 star
percentage_five_star_vine = round(float(paid_five_star_number) / float(paid_number),4)
print(f'Percantage of paid reviews that are five stars {percentage_five_star_vine * 100}%')<jupyter_output>Percantage of paid reviews that are five stars 51.06%
{'fraction_non_alphanumeric': 0.21013039117352056, 'fraction_numerical': 0.0663657639585423, 'mean_word_length': 1.7382151029748283, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '435985', 'license': 'ADSL', 'n_tokens_mistral': 2166, 'n_tokens_neox': 1861, 'n_words': 361, 'path': 'level-2/big_dataL2.ipynb', 'repo_name': 'epurwant0/big-data-challenge', 'size': '5982'}