<jupyter_start><jupyter_text>Install Java, Spark, and Findspark<jupyter_code>!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q http://www-us.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz
!tar xf spark-2.3.2-bin-hadoop2.7.tgz
!pip install -q findspark<jupyter_output><empty_output><jupyter_text>Set Environmental Variables<jupyter_code>import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-2.3.2-bin-hadoop2.7"<jupyter_output><empty_output><jupyter_text>Start a Spark Session<jupyter_code>import findspark
findspark.init()<jupyter_output><empty_output><jupyter_text>Dataframe Basics<jupyter_code># Import our SparkSession so we can use it
from pyspark.sql import SparkSession
# Create our SparkSession, this can take a couple minutes locally
spark = SparkSession.builder.appName("basics").getOrCreate()
# Load in data
from pyspark import SparkFiles
url = "https://s3.amazonaws.com/dataviz-curriculum/day_1/food.csv"
spark.sparkContext.addFile(url)
df = spark.read.csv(SparkFiles.get("food.csv"), sep=",", header=True)
df.show()
# Let's show the data
df.show()
# Print our schema
df.printSchema()
# Show the columns
df.columns
# Describe our data
df.describe()
# Import struct fields that we can use
from pyspark.sql.types import StructField, StringType, IntegerType, StructType
# Next we need to create the list of struct fields
schema = [StructField("food", StringType(), True), StructField("price", IntegerType(), True),]
schema
# Pass in our fields
final = StructType(fields=schema)
final
# Read our data with our new schema
dataframe = spark.read.csv(SparkFiles.get("food.csv"), sep=",", header=True, schema=final)
dataframe
# Print it out
dataframe.printSchema()<jupyter_output>root
 |-- food: string (nullable = true)
 |-- price: integer (nullable = true)<jupyter_text>Accessing data<jupyter_code>dataframe['price']
type(dataframe['price'])
dataframe.select('price')
type(dataframe.select('price'))
dataframe.select('price').show()<jupyter_output>+-----+
|price|
+-----+
|    0|
|   12|
|   10|
+-----+<jupyter_text>Manipulating Columns<jupyter_code># Add new column
dataframe.withColumn('newprice', dataframe['price']).show()
# Update column name
dataframe.withColumnRenamed('price','newerprice').show()
# Double the price
dataframe.withColumn('doubleprice',dataframe['price']*2).show()
# Add a dollar to the price
dataframe.withColumn('add_one_dollar',dataframe['price']+1).show()
# Half the price
dataframe.withColumn('half_price',dataframe['price']/2).show()
# Collecting a column as a list
dataframe.select("price").collect()<jupyter_output><empty_output><jupyter_text>Converting PySpark DataFrame to Pandas DataFrame<jupyter_code>import pandas as pd
pandas_df = dataframe.toPandas() 
pandas_df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14708994708994708, 'fraction_numerical': 0.011287477954144622, 'mean_word_length': 6.644204851752021, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '218465', 'license': 'ADSL', 'n_tokens_mistral': 1018, 'n_tokens_neox': 991, 'n_words': 205, 'path': '07-Extra-Content/Big-Data-Google-Colab/day-1/05-Ins_Pyspark_Dataframes_Basics/Solved/spark_dataframe_basics.ipynb', 'repo_name': 'anirudhmungre/sneaky-lessons', 'size': '2835'}