<jupyter_start><jupyter_text>Setup<jupyter_code># Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
PROJECT_ID = "clustering"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", PROJECT_ID)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)<jupyter_output><empty_output><jupyter_text>Get the Data<jupyter_code>!ls ./datasets/
import pandas as pd

cust = pd.read_csv('./datasets/2017-10-25_BCDentists.csv')<jupyter_output><empty_output><jupyter_text>Inspect Data<jupyter_code>cust.head()
cust.info()
cust.describe()
cust['districts'].value_counts()
cust['Province'].value_counts()
cust['cdsbc_subclassidname'].value_counts() 
cust['School'].value_counts() 
cust['Gender'].value_counts() 
cust['languages'].value_counts()<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code>cust['Grad Year'].hist(bins=200,figsize=(20,15))
save_fig("attribute_histogram_plots")
plt.show()

cust.plot(kind="scatter", x="Longitude", y="Latitude", alpha=0.6, figsize=(20,15))
save_fig("better_visualization_plot")
from pandas.tools.plotting import scatter_matrix # For older versions of Pandas
# from pandas.plotting import scatter_matrix

attributes = ["median_house_value", "median_income", "total_rooms",
              "housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12, 8))
save_fig("scatter_matrix_plot")
Ideas<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13802083333333334, 'fraction_numerical': 0.018229166666666668, 'mean_word_length': 7.5377777777777775, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '589809', 'license': 'MIT', 'n_tokens_mistral': 740, 'n_tokens_neox': 690, 'n_words': 116, 'path': 'quote_formulary/20171115-tc-cluster.ipynb', 'repo_name': 'HSCanada/br_notebook', 'size': '1920'}