<jupyter_start><jupyter_text>Import data and reduce as needed.<jupyter_code>df = pd.read_csv("../DataSet/Merged_Reduced_SDSS_GZ_Catalog.csv")
# Drop unknowns
df = df.drop(df[df["P_DK"]>0.5].index)
df = df.drop(df[np.isnan(df["dered_g"])].index)
df = df.drop(df[np.isnan(df["dered_r"])].index)
df = df.drop(df[np.isnan(df["dered_i"])].index)
df.reset_index()
conditions = [(df["P_EL"] > df["P_CS"]), (df["P_EL"] < df["P_CS"])]
cases = ["Elliptical","Spiral"]
df["Type"] = np.select(conditions,cases,default="Unknown")
df["dered_g-dered_r"] = df["dered_g"] - df["dered_r"]
df["dered_r-dered_i"] = df["dered_r"] - df["dered_i"]
df.to_csv("../DataSet/GalaxyZooSDSSDataset.csv")<jupyter_output><empty_output><jupyter_text>Choose what parameter space you want to work in.<jupyter_code>X = "dered_g-dered_r"; Y = "dered_g"<jupyter_output><empty_output><jupyter_text>Format the full data, training data, and test data<jupyter_code>data = np.zeros([len(df[X]),2]); data[:,0] = df[X]; data[:,1] = df[Y]<jupyter_output><empty_output><jupyter_text>I split the full data in half here, but there's fancier ways to do this using KFOLD.<jupyter_code>half = int(np.ceil(len(df[X])/2))
training_data = np.zeros([half,2]); training_data[:,0] = df[X][0:half]
training_data[:,1] = np.array(df[Y][0:half]); training_labels = df.Type[0:half]<jupyter_output><empty_output><jupyter_text>Get some classifier just to see one result.<jupyter_code>classifier = get_classifier(5)<jupyter_output><empty_output><jupyter_text>Create coordinates for the entire parameter space and predict labels for each point.<jupyter_code>x,y = np.meshgrid(np.arange(min(df[X]),max(df[X]),0.1),np.arange(min(df[Y]),max(df[Y]),.1))
test_data = np.zeros([len(x.flatten()),2]); test_data[:,0] = x.flatten(); test_data[:,1] = y.flatten()
test_labels = classifier.predict(test_data)

new_dict = {X:test_data[:,0],Y:test_data[:,1],"Type":test_labels}
tr = pd.DataFrame.from_dict(new_dict)
plt.clf(); 
plt.figure(figsize=(11,8))

plt.scatter(tr[X].loc[tr["Type"]=="Elliptical"],tr[Y].loc[tr["Type"]=="Elliptical"],c="#9b59b6",alpha=0.3,zorder=0,s=100,label="")
plt.scatter(tr[X].loc[tr["Type"]=="Spiral"],tr[Y].loc[tr["Type"]=="Spiral"],c="#3498db",alpha=0.3,zorder=0,s=100,label="")

plt.scatter(df[X].loc[df["Type"]=="Elliptical"],df[Y].loc[df["Type"]=="Elliptical"],c="#9b59b6",edgecolor="black",label="Elliptical",s=30,zorder=2)
plt.scatter(df[X].loc[df["Type"]=="Spiral"],df[Y].loc[df["Type"]=="Spiral"],c="#3498db",edgecolor="black",label="Spiral",s=30,zorder=2)

plt.xlim(-2.5,5); plt.ylim(11,25)
plt.legend(); plt.xlabel(X,size="large"); plt.ylabel(Y,size="large")
plt.title("Plot the Data Set")
plt.savefig("DataSet.png")
plt.show()<jupyter_output><empty_output><jupyter_text>Test for completeness and Purity<jupyter_code>E_Tests = np.array([get_tests(i,"Elliptical") for i in range(1,20)])
S_Tests = np.array([get_tests(i,"Spiral") for i in range(1,20)])
plt.clf()
f, (ax,bx) = plt.subplots(1,2,figsize=(15,6))
ax.plot(np.arange(1,20),E_Tests[:,0],label="Completeness for Ellipticals",c="#9b59b6",linewidth=6)
ax.plot(np.arange(1,20),E_Tests[:,1],label="Purity for Ellipticals",c="#3498db",linewidth=6)
ax.legend(); ax.set_xlabel("Neighbors",size="large")
bx.plot(np.arange(1,20),S_Tests[:,0],label="Completeness for Spirals",c="#9b59b6",linewidth=6)
bx.plot(np.arange(1,20),S_Tests[:,1],label="Purity for Spirals",c="#3498db",linewidth=6)
bx.legend(); bx.set_xlabel("Neighbors",size="large")
f.suptitle("Nearest Neighbors vs Completeness and Purity")
ax.grid(color="grey",alpha=0.2); bx.grid(color="grey",alpha=0.2)
plt.savefig("NearestNeighborsvsCompletenessandPurity.png")
plt.show()
plt.clf(); 
f, (ax,bx,cx) = plt.subplots(1,3,figsize=(16,6))

tr1_labels = get_classifier(1).predict(test_data)
tr1 = pd.DataFrame.from_dict({X:test_data[:,0],Y:test_data[:,1],"Type":tr1_labels})

ax.scatter(tr[X].loc[tr1["Type"]=="Elliptical"],tr[Y].loc[tr1["Type"]=="Elliptical"],c="#9b59b6",alpha=0.3,zorder=0,s=100,label="")
ax.scatter(tr[X].loc[tr1["Type"]=="Spiral"],tr[Y].loc[tr1["Type"]=="Spiral"],c="#3498db",alpha=0.3,zorder=0,s=100,label="")
ax.scatter(df[X].loc[df["Type"]=="Elliptical"],df[Y].loc[df["Type"]=="Elliptical"],c="#9b59b6",edgecolor="black",label="Elliptical",s=30,zorder=2)
ax.scatter(df[X].loc[df["Type"]=="Spiral"],df[Y].loc[df["Type"]=="Spiral"],c="#3498db",edgecolor="black",label="Spiral",s=30,zorder=2)

tr2_labels = get_classifier(9).predict(test_data)
tr2 = pd.DataFrame.from_dict({X:test_data[:,0],Y:test_data[:,1],"Type":tr2_labels})
bx.scatter(tr[X].loc[tr2["Type"]=="Elliptical"],tr[Y].loc[tr2["Type"]=="Elliptical"],c="#9b59b6",alpha=0.3,zorder=0,s=100,label="")
bx.scatter(tr[X].loc[tr2["Type"]=="Spiral"],tr[Y].loc[tr2["Type"]=="Spiral"],c="#3498db",alpha=0.3,zorder=0,s=100,label="")
bx.scatter(df[X].loc[df["Type"]=="Elliptical"],df[Y].loc[df["Type"]=="Elliptical"],c="#9b59b6",edgecolor="black",label="Elliptical",s=30,zorder=2)
bx.scatter(df[X].loc[df["Type"]=="Spiral"],df[Y].loc[df["Type"]=="Spiral"],c="#3498db",edgecolor="black",label="Spiral",s=30,zorder=2)

tr3_labels = get_classifier(12).predict(test_data)
tr3 = pd.DataFrame.from_dict({X:test_data[:,0],Y:test_data[:,1],"Type":tr3_labels})
cx.scatter(tr[X].loc[tr3["Type"]=="Elliptical"],tr[Y].loc[tr3["Type"]=="Elliptical"],c="#9b59b6",alpha=0.3,zorder=0,s=100,label="")
cx.scatter(tr[X].loc[tr3["Type"]=="Spiral"],tr[Y].loc[tr3["Type"]=="Spiral"],c="#3498db",alpha=0.3,zorder=0,s=100,label="")
cx.scatter(df[X].loc[df["Type"]=="Elliptical"],df[Y].loc[df["Type"]=="Elliptical"],c="#9b59b6",edgecolor="black",label="Elliptical",s=30,zorder=2)
cx.scatter(df[X].loc[df["Type"]=="Spiral"],df[Y].loc[df["Type"]=="Spiral"],c="#3498db",edgecolor="black",label="Spiral",s=30,zorder=2)

ax.set_xlim(-2.5,5); ax.set_ylim(11,25)
bx.set_xlim(-2.5,5); bx.set_ylim(11,25)
cx.set_xlim(-2.5,5); cx.set_ylim(11,25)
ax.set_title("N = 1");bx.set_title("N = 9");cx.set_title("N = 12")
plt.legend(); bx.set_xlabel(X,size="large"); ax.set_ylabel(Y,size="large")
f.suptitle("Classification Boundaries")
plt.savefig("ClassificationBoundaries.png")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.25936835215185733, 'fraction_numerical': 0.045655375552282766, 'mean_word_length': 12.28695652173913, 'pattern_counts': {'":': 4, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '549751', 'license': 'MIT', 'n_tokens_mistral': 2845, 'n_tokens_neox': 2730, 'n_words': 223, 'path': 'NearestNeighbors/NearestNeighborsWorkBook.ipynb', 'repo_name': 'AstroLudwig/MachineLearningClass', 'size': '6111'}