<jupyter_start><jupyter_text>Import data for buy clicks to get number of add clicks per user<jupyter_code>sqlContext = SQLContext(sc)
add_clicks = sqlContext.read.load('file:///home/cloudera/Desktop/bigdatastudy/big-data-4/ad-clicks.csv', 
                          format='com.databricks.spark.csv', 
                          header='true',inferSchema='true')
adclicks_df = pd.read_csv('file:///home/cloudera/Desktop/bigdatastudy/big-data-4/ad-clicks.csv')
adclicks_df = adclicks_df.rename(columns=lambda x: x.strip()) #remove whitespaces from headers
adclicks_df.head(5)
adclicks_df['adCount'] = 1
adclicks_df.head(5)
gameclicks_df = pd.read_csv('file:///home/cloudera/Desktop/bigdatastudy/big-data-4/game-clicks.csv')
gameclicks_df = gameclicks_df.rename(columns=lambda x: x.strip()) #remove whitespaces from headers
gameclicks_df['clickCount'] = 1
gameclicks_df.head(5)
buyclicks_df = pd.read_csv('file:///home/cloudera/Desktop/bigdatastudy/big-data-4/buy-clicks.csv')
buyclicks_df = buyclicks_df.rename(columns=lambda x: x.strip()) #removes whitespaces from headers 
buyclicks_df.head(5)
user_purchases = buyclicks_df[['userId','price']] #select only userid and price
user_purchases.head(5)
user_adclicks = adclicks_df[['userId','adCount']]
user_adclicks.head(5)
user_gameclicks = gameclicks_df[['userId','clickCount']]
user_gameclicks.head(5) #as we saw before, this line displays first five lines
ads_per_user = user_adclicks.groupby('userId').sum()
ads_per_user = ads_per_user.reset_index()
ads_per_user.columns = ['userId', 'TotalAddClicks'] #rename the columns
ads_per_user.head(5)

game_clicks_per_user = user_gameclicks.groupby('userId').sum()
game_clicks_per_user = game_clicks_per_user.reset_index()
game_clicks_per_user.columns = ['userId', 'TotalGameClicks'] #rename the columns
game_clicks_per_user.head(5)
revenue_per_user = user_purchases.groupby('userId').sum()
revenue_per_user = revenue_per_user.reset_index()
revenue_per_user.columns = ['userId', 'MoneySpent'] #rename the columns
revenue_per_user.head(5)
combined_df = ads_per_user.merge(game_clicks_per_user, on='userId') #userId, adCount, clickCount
combined_df = combined_df.merge(revenue_per_user, on='userId') #userId, adCount, clickCount, price 
combined_df.head(5) #display how the merged table looks
clusters_df = combined_df[['TotalAddClicks','MoneySpent', 'TotalGameClicks']]
clusters_df.head(5)
clusters_df.shape
# Convert to Kmeans format 
sqlContext = SQLContext(sc)
p_df = sqlContext.createDataFrame(cluster_df)
parsed_data = p_df.rdd.map(lambda line: array([line[0], line[1], line[2]]))

p_df = p_df.na.drop()
p_df.columns
p_df.head(5)
# use vector assembler to get all features 
featuresUsed = ['totalAdClicks', 'revenue','totalGameClicks']
assembler = VectorAssembler(inputCols=featuresUsed, outputCol="features_unscaled")
assembled = assembler.transform(p_df)
scaler = StandardScaler(inputCol="features_unscaled", outputCol="features", withStd=True, withMean=True)
scalerModel = scaler.fit(assembled)
scaledData = scalerModel.transform(assembled)
# Select the features column make the data persist: 
scaledData = scaledData.select("features")
scaledData.persist()
kmeans = KMeans(k=3, seed=1)
model = kmeans.fit(scaledData)
transformed = model.transform(scaledData)
centers = model.clusterCenters()
centers
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.datasets.samples_generator import make_blobs

threedee = plt.figure(figsize=(12,10)).gca(projection='3d')
threedee.scatter(scaledData[:,0], scaledData[:,1], scaledData[:,2] )
threedee.set_xlabel('totalAdClicks')
threedee.set_ylabel('revenue')
threedee.set_zlabel('totalGameClicks')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13056379821958458, 'fraction_numerical': 0.009171837065012138, 'mean_word_length': 7.351351351351352, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '320997', 'license': 'MIT', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1367, 'n_words': 222, 'path': 'ucsd_spark_ml/Cluster Analysis Of Pink Flamingo.ipynb', 'repo_name': 'gauravbrills/ml-sprint', 'size': '3707'}