<jupyter_start><jupyter_text>Eye HeuristicsThe data has two normal distributions where one is for images with eyes and one is for images without.<jupyter_code>import SimpleITK as sitk
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.tree import DecisionTreeClassifier
import matplotlib.pyplot as plt
import glob
import os
import pickle
import numpy as np
IMG_SAVE_DIR = "img_viz"
IMG_VIZ_EXT = "png"
all_subject_dirs = glob.glob(os.path.join(EYE_DIR,'*'))
all_subject_ids = [ os.path.basename(sd) for sd in all_subject_dirs]
img_files = glob.glob("./img_viz/*/*")
average_intensities = []
for imf in img_files:
    im = sitk.ReadImage(imf)
    arr = sitk.GetArrayFromImage(im)
    average_intensities.append(np.average(arr.flatten()))
plt.hist(average_intensities,bins=100)
LE_data = glob.glob("./img_viz/LE/*")
LE_data.sort()
RE_data = glob.glob("./img_viz/RE/*")
RE_data.sort()
with open("./failed_le.list",'r') as f:
    LE_fails = f.readlines()
with open("./failed_re.list",'r') as f:
    RE_fails = f.readlines()
LE_fails = [a.split('\n')[0] for a in LE_fails[1:]]
LE_fails.sort()
RE_fails = [a.split('\n')[0] for a in RE_fails[1:]]
RE_fails.sort()<jupyter_output><empty_output><jupyter_text>prep features<jupyter_code>features = []
for image_fn in LE_data:
    features.append(np.average(sitk.GetArrayFromImage(sitk.ReadImage(image_fn)).flatten()))
for image_fn in RE_data:
    features.append(np.average(sitk.GetArrayFromImage(sitk.ReadImage(image_fn)).flatten()))
features = np.array(features)
features = np.reshape(features, (-1,1))
print(features.shape)<jupyter_output>(1610, 1)<jupyter_text>prep labels<jupyter_code>labels = []
for sub in all_subject_ids:
    labels.append(sub in LE_fails)
for sub in all_subject_ids:
    labels.append(sub in RE_fails)
labels = np.array(labels)
labels = np.reshape(labels, (-1,1))
print(labels.shape)<jupyter_output>(1610, 1)<jupyter_text>train test split<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(features, labels, shuffle=True, test_size=0.33)
clf = DecisionTreeClassifier(max_depth=1)  # only one decision
clf.fit(x_train, y_train)
print("accuracy {}%".format(clf.score(x_test, y_test)*100))
print(classification_report(y_test, clf.predict(x_test)))
with open("./classifier.pickle", 'wb') as f:
    pickle.dump(clf, f)
clf.predict([[3.4]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14073462649607923, 'fraction_numerical': 0.012381345439537762, 'mean_word_length': 6.504643962848297, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '136728', 'license': 'MIT', 'n_tokens_mistral': 957, 'n_tokens_neox': 926, 'n_words': 177, 'path': 'hw/hw07-abpwrs/HeuristicTrain.ipynb', 'repo_name': 'abpwrs/ece-5490-sp19', 'size': '2423'}