<jupyter_start><jupyter_text>PySpark Linear RegressionThis notebook demonstrates Linear Regression in PySpark.Several [Spark examples](/tree/examples/spark) are included with TAP.More examples are available on the Spark website: http://spark.apache.org/examples.htmlPySpark API documentation: http://spark.apache.org/docs/latest/api/python/<jupyter_code># Import the needed libraries
import pyspark
from pyspark.mllib.linalg import Vectors
from pyspark.ml.regression import LinearRegression

# Create a SparkContext in local mode
sc = pyspark.SparkContext("local")

# Create a SqlContext from the SparkContext
sqlContext = pyspark.SQLContext(sc)
# Setup some fictional housing data
data = [ 
    # price, sqft, bedrooms
    ( 300000.0, Vectors.dense( 2000.0, 3.0 ) ),
    ( 500000.0, Vectors.dense( 4000.0, 4.0 ) ),
    ( 250000.0, Vectors.dense( 1500.0, 2.0 ) ),
    ( 165000.0, Vectors.dense( 1200.0, 1.0 ) ),
    ( 325000.0, Vectors.dense( 2500.0, 3.0 ) ),
    ( 275000.0, Vectors.dense( 1900.0, 3.0 ) ) 
]

# Create a DataFrame
df = sqlContext.createDataFrame(data, ['price', 'features'])

# Convert to a Pandas DataFrame for easy display
df.toPandas()
# Setup LinearRegression
lr = LinearRegression(maxIter=5, regParam=0.0, labelCol="price")

# Train the model
model = lr.fit(df)

# View properties of the trained model
print "intercept: " + str(model.intercept)
print "weights: " + str(model.weights)
# Setup data that we want to do predictions for
dataToPredict = [
    # sqft, bedrooms
    ( Vectors.dense(2700.0, 3.0), ),
    ( Vectors.dense(1700.0, 3.0), ),
    ( Vectors.dense(1700.0, 2.0), ),
    ( Vectors.dense(1000.0, 1.0), )
]

# Create a DataFrame
dfToPredict = sqlContext.createDataFrame(dataToPredict, ["features"])
     
# Use the model to predict housing prices
predictions = model.transform(dfToPredict)

# Convert to a Pandas DataFrame for easy display
predictions.toPandas()<jupyter_output><empty_output><jupyter_text>Stop the Spark Context<jupyter_code># Stop the context when you are done with it. When you stop the SparkContext resources 
# are released and no further operations can be performed within that context
sc.stop()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11635364177737059, 'fraction_numerical': 0.052679798442510306, 'mean_word_length': 4.473684210526316, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '41416', 'license': 'Apache-2.0', 'n_tokens_mistral': 842, 'n_tokens_neox': 736, 'n_words': 203, 'path': 'jupyter-default-notebooks/notebooks/examples/spark/pyspark-linear-regression.ipynb', 'repo_name': 'trustedanalytics-ng/jupyter', 'size': '2183'}