<jupyter_start><jupyter_text>1. Data loading<jupyter_code>data_paths = ['gs://6893-data/triLabel/player_stats_2018_2019.csv',
              'gs://6893-data/triLabel/player_stats_2019_2020.csv', 
              'gs://6893-data/triLabel/player_stats_2020_2021.csv',
              'gs://6893-data/triLabel/player_stats_2021_2022.csv'
]
data_1 = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(data_paths[0])
data_2 = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(data_paths[1])
data_3 = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(data_paths[2])

data = data_1.union(data_2)
data = data.union(data_3)
ongoing_data = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(data_paths[3])
# print(data)
print('Overall shape of data', data.count(), len(data.columns))
print('2018-2019 dimension of data', data_1.count(), len(data_1.columns))
print('2019-2020 dimension of data', data_2.count(), len(data_2.columns))
print('2020-2021 dimension of data', data_3.count(), len(data_3.columns))
mip_datapaths = [
             'gs://6893-data/mipData/player_improvement_2019_2020.csv',
             'gs://6893-data/mipData/player_improvement_2020_2021.csv',
             'gs://6893-data/mipData/player_improvement_2021_2022.csv'
]
mip_data_1 = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(mip_datapaths[0])
mip_data_2 = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(mip_datapaths[1])

mip_data = mip_data_1.union(mip_data_2)
ongoing_mip_data = spark.read.format("csv").option("header", "true").option("inferschema", "true").load(mip_datapaths[2])
print('Overall shape of data', mip_data.count(), len(mip_data.columns))
print('2019~2020 dimension of data', mip_data_1.count(), len(mip_data_1.columns))
print('2020~2021 dimension of data', mip_data_2.count(), len(mip_data_2.columns))
print('2021~2022 dimension of data', ongoing_mip_data.count(), len(ongoing_mip_data.columns))
print(mip_data)<jupyter_output>Overall shape of data 811 7
2019~2020 dimension of data 400 7
2020~2021 dimension of data 411 7
2021~2022 dimension of data 369 7
DataFrame[reb: double, ast: double, stl: double, blk: double, tov: double, pts: double, label: int]<jupyter_text>2. Data preprocessing<jupyter_code>from pyspark.ml import Pipeline
from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler
#stages in our Pipeline
stages = []
# Transform all features into a vector using VectorAssembler
numericCols = ["reb", "ast", "stl", "blk", "tov", "pts"]
assemblerInputs = numericCols
assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features")
stages += [assembler]
pipeline = Pipeline(stages=stages)
pipelineModel = pipeline.fit(data)
preppedDataDF = pipelineModel.transform(data)
preppedOngoingDF = pipelineModel.transform(ongoing_data)
pipelineModelMip = pipeline.fit(mip_data)
preppedDataMip = pipelineModelMip.transform(mip_data)
preppedOngoingMip = pipelineModelMip.transform(ongoing_mip_data)
preppedDataMip.take(3)
# preppedOngoingMip.take(3)
preppedDataDF.take(3)
preppedOngoingDF.take(3)
# Keep relevant columns

cols = data.columns
selectedcols = ["features"] + cols
dataset = preppedDataDF.select(selectedcols)

ongoing_dataset = preppedOngoingDF.select(['features', 'id', 'name', 'reb', 'ast', 'stl', 'blk', 'tov', 'pts'])

display(dataset)
display(ongoing_dataset)
print(selectedcols)
print(ongoing_dataset.count())
dataset_mip = preppedDataMip.select(['features','label', 'reb', 'ast', 'stl', 'blk', 'tov', 'pts'])
# dataset_mip.show(300)
dataset_ongoing_mip = preppedOngoingMip.select(['features', 'name', 'reb', 'ast', 'stl', 'blk', 'tov', 'pts'])
display(dataset_mip)
display(dataset_ongoing_mip)
### Randomly split data into training and test sets. set seed for reproducibility
#=====your code here==========
trainingData, testData = dataset.randomSplit([.80, .20], seed=100)
#===============================
print("trainingData length = ", trainingData.count())
print("testData length = ", testData.count())
trainingData_mip, testData_mip = dataset_mip.randomSplit([.80, .20], seed=100)
print("trainingData_mip length = ", trainingData_mip.count())
print("testData_mip length = ", testData_mip.count())
trainingData_mip.show(300)
testData_mip.show(300)<jupyter_output>trainingData_mip length =  653
testData_mip length =  158
+--------------------+-----+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+
|            features|label|                 reb|                 ast|                 stl|                 blk|                 tov|                 pts|
+--------------------+-----+--------------------+--------------------+--------------------+--------------------+--------------------+--------------------+
|[-0.9999322943711...|    0|  -0.999932294371154| -0.9999398894154533|                 0.0|                 0.0|                 0.0| -0.9999189819893424|
|[-0.9999322943711...|    0|  -0.999932294371154| -0.7133078705373324| -0.9998634933910959|                 0.0| -0.3638854186585195| -0.7974937740183787|
|[-0.9999235331362...|    0| -0.9999235331362399| -0.1588437103362509| -0.9999669040661644| -0.9999310768536014| -0.9999802735881309|-0.38359667759217086|
|[-0.8366423[...]<jupyter_text>3. Modeling<jupyter_code>from pyspark.ml.classification import LogisticRegression, RandomForestClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
import datetime
from pytz import timezone
tz = timezone('EST')
now = datetime.datetime.now(tz)

# Fit model to prepped data

#LogisticRegression model, maxIter=10
#=====your code here============
lrModel_mvp = LogisticRegression(featuresCol="features", labelCol="mvpLabel", maxIter=10).fit(trainingData)
lrModel_dpoy = LogisticRegression(featuresCol="features", labelCol="dpoyLabel", maxIter=10).fit(trainingData)
lrModel_mip = LogisticRegression(featuresCol="features", labelCol="label", maxIter=10).fit(trainingData_mip)
#===============================

mvp_predictions = lrModel_mvp.transform(testData)
dpoy_predictions = lrModel_dpoy.transform(testData)
mip_predictions = lrModel_mip.transform(testData_mip)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mip_predictions)
print("Test set accuracy for mip prediction = " + str(accuracy))

mvp_predictions = lrModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['id', 'name', 'probability'])
mvp_selected = mvp_selected.withColumnRenamed('probability', 'mvp_prediction').withColumn('date', lit(str(now)[:10]))

# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = lrModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['id', 'name', 'probability'])
dpoy_selected = dpoy_selected.withColumnRenamed('probability', 'dpoy_prediction').withColumn('date', lit(str(now)[:10]))

mip_predictions = lrModel_mip.transform(dataset_ongoing_mip)
print(mip_predictions)
print('mip dataset ongoing')
dataset_ongoing_mip.show(5)
print('training dataset mip')
trainingData_mip.show(20)
mip_selected = mip_predictions.select(['name', 'probability', 'prediction'])
# mip_selected.show(mip_selected.count())
mip_selected = mip_selected.withColumnRenamed('probability', 'mip_prediction').withColumn('date', lit(str(now)[:10]))
print(now)
print(mvp_selected)
print(mvp_selected.count(), 'lines in mvp_prediction')
print(dpoy_selected)
print(dpoy_selected.count(), 'lines in dpoy_prediction')
print(mip_selected)
print(mip_selected.count(), 'lines in mip_prediction')

mvp_pd = mvp_selected.toPandas()
for idx, row in mvp_pd.iterrows():
    row.mvp_prediction = row.mvp_prediction[1]
    # print(row)
print(mvp_pd.head(3))
mvp_pd.to_csv('mvp_prediction_results.csv')

dpoy_pd = dpoy_selected.toPandas()
for idx, row in dpoy_pd.iterrows():
    row.dpoy_prediction = row.dpoy_prediction[1]
    # print(row)
print(dpoy_pd.head(3))
dpoy_pd.to_csv('dpoy_prediction_results.csv')

mip_pd = mip_selected.toPandas()
# print(mip_pd)
for idx, row in mip_pd.iterrows():
    row.mip_prediction = row.mip_prediction[1]
    # print(row)
print(mip_pd.head(3))
mip_pd.to_csv('mip_prediction_results.csv')
from pytz import timezone
tz = timezone('EST')
datetime.datetime.now(tz) 
import os
import sys
# os.getcwd()
from pyspark.ml.classification import RandomForestClassifier
# Random Forest
#=====your code here==========
rfModel_mvp = RandomForestClassifier(featuresCol="features", labelCol="mvpLabel").fit(trainingData)
rfModel_dpoy = RandomForestClassifier(featuresCol="features", labelCol="dpoyLabel").fit(trainingData)
rfModel_mip = RandomForestClassifier(featuresCol="features", labelCol="label").fit(trainingData_mip)
#===============================


mvp_predictions = rfModel_mvp.transform(testData)
dpoy_predictions = rfModel_dpoy.transform(testData)
mip_predictions = rfModel_mip.transform(testData_mip)
# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mip_predictions)
print("Test set accuracy for mip prediction = " + str(accuracy))

mvp_predictions = rfModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'probability', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = rfModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'probability', 'prediction', 'blk'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)

mip_predictions = rfModel_mip.transform(dataset_ongoing_mip)
mip_selected = mip_predictions.select(['name', 'probability', 'prediction'])
# dpoy_selected.show(dpoy_selected.count(), False)
mip_selected.show(30)
mip_selected = mip_selected.withColumnRenamed('probability', 'mip_prediction').withColumn('date', lit(str(now)[:10]))
mip_selected = mip_selected.select(['name', 'mip_prediction', 'date'])
mip_pd = mip_selected.toPandas()
# print(mip_pd)
for idx, row in mip_pd.iterrows():
    row.mip_prediction = row.mip_prediction[1]/0.35
    print(row)
print(mip_pd.head(3))
mip_pd.to_csv('mip_prediction_results.csv')
#NaiveBayes
#=====your code here==========
from pyspark.ml.classification import NaiveBayes
nbModel_mvp = NaiveBayes(featuresCol="features", labelCol="mvpLabel").fit(trainingData)
nbModel_dpoy = NaiveBayes(featuresCol="features", labelCol="dpoyLabel").fit(trainingData)
#===============================

mvp_predictions = nbModel_mvp.transform(testData)
dpoy_predictions = nbModel_dpoy.transform(testData)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))

mvp_predictions = nbModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'probability', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = nbModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'probability', 'prediction'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)
#Decision Tree
#=====your code here==========
from pyspark.ml.classification import DecisionTreeClassifier
dtModel_mvp = DecisionTreeClassifier(featuresCol="features", labelCol="mvpLabel").fit(trainingData)
dtModel_dpoy = DecisionTreeClassifier(featuresCol="features", labelCol="dpoyLabel").fit(trainingData)
#===============================

mvp_predictions = dtModel_mvp.transform(testData)
dpoy_predictions = dtModel_dpoy.transform(testData)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))

mvp_predictions = dtModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'probability', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = dtModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'probability', 'prediction', 'blk'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)
#Gradient Boosting Trees
#=====your code here==========
from pyspark.ml.classification import GBTClassifier
gbtModel_mvp = GBTClassifier(featuresCol="features", labelCol="mvpLabel").fit(trainingData)
gbtModel_dpoy = GBTClassifier(featuresCol="features", labelCol="dpoyLabel").fit(trainingData)
#===============================

mvp_predictions = gbtModel_mvp.transform(testData)
dpoy_predictions = gbtModel_dpoy.transform(testData)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))

mvp_predictions = gbtModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'probability', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = gbtModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'probability', 'prediction', 'blk'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)
# Multi-layer Perceptron
#=====your code here==========
from pyspark.ml.classification import MultilayerPerceptronClassifier
mlpModel_mvp = MultilayerPerceptronClassifier(layers=[6, 5, 5, 2], seed=123, featuresCol="features", labelCol="mvpLabel").fit(trainingData)
mlpModel_dpoy = MultilayerPerceptronClassifier(layers=[6, 5, 5, 2], seed=123, featuresCol="features", labelCol="dpoyLabel").fit(trainingData)
#===============================

mvp_predictions = mlpModel_mvp.transform(testData)
dpoy_predictions = mlpModel_dpoy.transform(testData)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))

mvp_predictions = mlpModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'probability', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = mlpModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'probability', 'prediction', 'blk'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)
# Linear Support Vector Machine
#=====your code here==========
from pyspark.ml.classification import LinearSVC
svmModel_mvp = LinearSVC(maxIter=10, regParam=0.1, labelCol="mvpLabel").fit(trainingData)
svmModel_dpoy = LinearSVC(maxIter=10, regParam=0.1, labelCol="dpoyLabel").fit(trainingData)
#===============================

mvp_predictions = svmModel_mvp.transform(testData)
dpoy_predictions = svmModel_dpoy.transform(testData)

# compute accuracy on the test set
evaluator = MulticlassClassificationEvaluator(labelCol="mvpLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(mvp_predictions)
print("Test set accuracy for mvp prediction = " + str(accuracy))
evaluator = MulticlassClassificationEvaluator(labelCol="dpoyLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(dpoy_predictions)
print("Test set accuracy for dpoy prediction = " + str(accuracy))

mvp_predictions = svmModel_mvp.transform(ongoing_dataset)
mvp_selected = mvp_predictions.select(['name', 'prediction', 'blk'])
# mvp_selected.show(mvp_selected.count(), False)

dpoy_predictions = svmModel_dpoy.transform(ongoing_dataset)
dpoy_selected = dpoy_predictions.select(['name', 'prediction', 'blk'])
# dpoy_selected.show(dpoy_selected.count(), False)
print(dpoy_predictions)<jupyter_output>Test set accuracy for mvp prediction = 0.9824046920821115
Test set accuracy for dpoy prediction = 0.9765395894428153
DataFrame[features: vector, id: string, name: string, reb: double, ast: double, stl: double, blk: double, tov: double, pts: double, rawPrediction: vector, prediction: double]<jupyter_text>4. Comparison and analysis<jupyter_code># query to see if there is new request
from google.cloud import bigquery
df.printSchema()
####################### data preprocessing #######################
stages = []
# Transform all features into a vector using VectorAssembler
numericCols = ["reb", "ast", "stl", "blk", "tov", "pts"]
assemblerInputs = numericCols
assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features")
stages += [assembler]
pipeline = Pipeline(stages=stages)
pipelineModel = pipeline.fit(df)
preppedDataDF = pipelineModel.transform(df)
preppedDataDF.take(3)
models = [lrModel, rfModel, dtModel, nbModel, gbtModel, mlpModel, svmModel, ovrModel]
print(testData)
print(preppedDataDF)
predictions = rfModel.transform(preppedDataDF)
print(predictions)
predictions.show(3)
selected = predictions.select(['name', 'probability', 'prediction', 'pts', 'new_request'])
selected.show(3)
# selected.show(selected.count(), False)

probability = predictions.select(['probability'])
probability.show(3)<jupyter_output>DataFrame[id: string, name: string, label: bigint, reb: double, ast: double, stl: double, blk: double, tov: double, pts: double, new_request: bigint, percentage: double, features: vector, rawPrediction: vector, probability: vector, prediction: double]
+--------------------+-------------+-----+-----------+-----------+-----------+-----------+-----------+-----------+-----------+----------+--------------------+--------------------+--------------------+----------+
|                  id|         name|label|        reb|        ast|        stl|        blk|        tov|        pts|new_request|percentage|            features|       rawPrediction|         probability|prediction|
+--------------------+-------------+-----+-----------+-----------+-----------+-----------+-----------+-----------+-----------+----------+--------------------+--------------------+--------------------+----------+
|fa7a753f-e41b-430...|<NAME>|    0|        1.7|       3.65|      0.325|       0.15|      1.925|       12.[...]
{'fraction_non_alphanumeric': 0.17987016156587105, 'fraction_numerical': 0.03192268267681945, 'mean_word_length': 5.900639946109801, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '275648', 'license': 'MIT', 'n_tokens_mistral': 7010, 'n_tokens_neox': 6589, 'n_words': 1144, 'path': 'algorithm/algor_train_v3.ipynb', 'repo_name': 'Sapphirine/202112-4-NBA-Award-Prediction', 'size': '20494'}