<jupyter_start><jupyter_text>Now apply algorithm to catalog at large<jupyter_code># Removes points with nan. It is assumed that these are all magnitude measurements, so they are set to their max.
flat = catalog.flatten()
for i in range(flat.size):
    if (np.isnan(flat[i])) or (not np.isfinite(flat[i])):
        catalog[i//271][i%271] = 99.999
        print(i)
cat_predictions=knn.predict(catalog) # Contains the predicted good/bad points 
pos = np.zeros((catalog.shape[0],2))
allinfo = np.zeros(np.shape(catalog))

j = 0 # Iteration variable for positions array
for i in range(catalog.shape[0]): # Cycles through each object in catalog
    # Checks to make sure point is "good"
    # Good defined by: 1. S/N > 20     2. Sharpness < .25     3. Roundness < 1     4. Crowding < .1 
    #                  5. Object type = "Bright Star"     6. ML algorithm picks as good
    if ((catalog[i][5] >= 20)&(abs(catalog[i][6]) < .25)&(abs(catalog[i][7]) < 1)
        &(catalog[i][9] < .1)&(catalog[i][10] == 1)&(cat_predictions[i] == 'green')):
        pos[j][0] = catalog[i][2] # Assigns X position with offset
        pos[j][1] = catalog[i][3] # Assigns Y position with offset
        allinfo[j] = catalog[i]
        j = j + 1
                
# Trims all zeros from end of positions array, left from objects with low S/N
pos = pos[~np.all(pos == 0, axis=1)]
info = allinfo[~np.all(allinfo == 0, axis=1)]

# Saves high S/N object X,Y positions
np.savetxt("sn2010ae_ML1.reg", pos, '%5.2f')
info.shape<jupyter_output><empty_output><jupyter_text>Re-train, this time with lower signal to noise<jupyter_code>asci = open("/Users/tktakaro/Documents/Type1ax_HST/machine_learning/sn2010ae_set2.reg", "r+")

def mysplit(s, delim=None):
    return [x for x in s.split(delim) if x]

text = asci.readlines()
regionX = np.zeros(2676)
regionY = np.zeros(2676)
key = []
for i in range(2676):
    A = mysplit(text[i+3], ' # ')
    regionX[i] = mysplit(mysplit(A[0], 'point')[0],',')[0][1:]
    regionY[i] = mysplit(mysplit(A[0], 'point')[0],',')[1][:-1]
    if A[1][-3] == 'l':
        key.append('green')
    elif A[1][-3] == 'u':
        key.append('blue')
    else:
        print("Error: neither blue nor green")

# Remove points from info (that were removed by hand)
j = 0
for i in range(info.shape[0]):
    if (info[i][2] == regionX[j]) & (info[i][3] == regionY[j]):
        j = j + 1
    else:
        info[i] = np.zeros(271)
info = info[~np.all(info == 0, axis=1)]
dataset = pandas.DataFrame(info)
dataset[271] = key
array = dataset.values
X = array[:,0:271]
Y = array[:,271]
validation_size = 0.20
seed = 7

# Test options and evaluation metric
num_folds = 10
num_instances = len(X_train)
seed = 7
scoring = 'accuracy'
X_train, X_validation, Y_train, Y_validation = cross_validation.train_test_split(X, Y,
                                                    test_size=validation_size, random_state=seed)
# Spot Check Algorithms
models = []
models.append(('LR', LogisticRegression()))
models.append(('LDA', LinearDiscriminantAnalysis()))
models.append(('KNN', KNeighborsClassifier()))
models.append(('CART', DecisionTreeClassifier()))
models.append(('NB', GaussianNB()))
models.append(('SVM', SVC()))
# evaluate each model in turn
results = []
names = []
for name, model in models:
    kfold = cross_validation.KFold(n=num_instances, n_folds=num_folds, random_state=seed)
    cv_results = cross_validation.cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
    results.append(cv_results)
    names.append(name)
    msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
    print(msg)
# Compare Algorithms
fig = plt.figure()
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()
# Make predictions on validation dataset
knn = KNeighborsClassifier()
knn.fit(X_train, Y_train)
predictions = knn.predict(X_validation)
print(accuracy_score(Y_validation, predictions))
print(confusion_matrix(Y_validation, predictions))
print(classification_report(Y_validation, predictions))
cat_predictions=knn.predict(catalog) # Contains the predicted good/bad points 
pos = np.zeros((catalog.shape[0],2))
allinfo = np.zeros(np.shape(catalog))

j = 0 # Iteration variable for positions array
for i in range(catalog.shape[0]): # Cycles through each object in catalog
    # Checks to make sure point is "good"
    # Good defined by: 1. S/N > 5     2. Sharpness < .25     3. Roundness < 1     4. Crowding < .1 
    #                  5. Object type = "Bright Star"     6. ML algorithm picks as good
    if ((catalog[i][5] >= 10)&(abs(catalog[i][6]) < .25)&(abs(catalog[i][7]) < 1)
        &(catalog[i][9] < .1)&(catalog[i][10] == 1)&(cat_predictions[i] == 'green')):
        pos[j][0] = catalog[i][2] # Assigns X position with offset
        pos[j][1] = catalog[i][3] # Assigns Y position with offset
        allinfo[j] = catalog[i]
        j = j + 1
                
# Trims all zeros from end of positions array, left from objects with low S/N
pos = pos[~np.all(pos == 0, axis=1)]
info = allinfo[~np.all(allinfo == 0, axis=1)]

# Saves high S/N object X,Y positions
np.savetxt("sn2010ae_ML2.reg", pos, '%5.2f')
info.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1579351372097486, 'fraction_numerical': 0.030896181155248514, 'mean_word_length': 3.742493175614195, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '137268', 'license': 'MIT', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1847, 'n_words': 475, 'path': 'jupyter_notebooks/.ipynb_checkpoints/2010ae_MachineLearning_OLD-checkpoint.ipynb', 'repo_name': 'TTakaro/Type-Iax-HST', 'size': '5211'}