<jupyter_start><jupyter_text>Check If All Class 1 Bad Pixels Are Indeed Just Noisy Pixels---<jupyter_code>quirks_store[classes_store == 1].shape
fig = figure()#figsize=(6,6))
ax  = fig.add_subplot(111)
# ax.plot([nan,nan])
corrections = []
for cnow in np.where(classes_store == 1)[0]:
    # ax.lines.pop()
    ax.clear()
    ax.plot(quirks_store[cnow] - median(quirks_store[cnow]))
    ax.set_title('Entry:' + str(cnow) + '/ Class:' + str(classes_store[cnow]))
    fig.canvas.draw()
    display.display(plt.gcf())
    display.clear_output(wait=True)
    checkClass = input('Is this a Noisy Pixel? ');
    if checkClass != '':
        corrections.append([cnow, checkClass])
# for cnow in np.where(classes_store == 1)[0]:
#     plt.plot(quirks_store[cnow])
#     display.clear_output(wait=True)
#     display.display(plt.gcf())
#     checkClass = input('Is this a Noisy Pixel? ');print(checkClass)
#     plt.clf()
#     display.clear_output(wait=True)<jupyter_output><empty_output><jupyter_text>Check If All Class 4 Bad Pixels Are Indeed Just CR Pixels---<jupyter_code>quirks_store[classes_store == 4].shape
fig = figure()#figsize=(6,6))
ax  = fig.add_subplot(111)

CRs = np.where(classes_store == 4)[0]
corrections = []
for cnow in :
    # ax.lines.pop()
    ax  = fig.add_subplot(111)
    ax.plot((quirks_store[cnow] - min(quirks_store[cnow])) / (max(quirks_store[cnow]) - min(quirks_store[cnow])), lw=2)
    ax.set_title('Entry:' + str(cnow) + '/ Class:' + str(classes_store[cnow]))
    ax.annotate(str(cnow), [110, 0.5], fontsize=50)
    fig.canvas.draw()
    display.display(plt.gcf())
    time.sleep(.05)
    display.clear_output(wait=True)
    ax.lines.pop()
    ax.texts.pop()
    if cnow > 500 and cnow < 1000:
        display.display(plt.clf())
        ax = fig.add_subplot(111)
#     checkClass = input('Is this a Cosmic Ray? ');
#     if checkClass != '':
#         corrections.append([cnow, checkClass])
# for cnow in np.where(classes_store == 1)[0]:
#     plt.plot(quirks_store[cnow])
#     display.clear_output(wait=True)
#     display.display(plt.gcf())
#     checkClass = input('Is this a Noisy Pixel? ');print(checkClass)
#     plt.clf()
#     display.clear_output(wait=True)
corrections
np.where(classes_store == 4)[0][69]
classes_store[[140,260, 380]] = 2
plot(quirks_store[140])
plot(quirks_store[260])
plot(quirks_store[380])
((quirks_store.T - np.min(quirks_store,axis=1)) / (np.max(quirks_store,axis=1) - np.min(quirks_store, axis=1))).shape
((quirks_store.T - np.min(quirks_store,axis=1)) / (np.max(quirks_store,axis=1) - np.min(quirks_store, axis=1))).T[classes_store == 4].T.shape
np.sum(classes_store == 2) // 100
quirk_store_norm = ((quirks_store.T - np.min(quirks_store,axis=1)) / (np.max(quirks_store,axis=1) - np.min(quirks_store, axis=1))).T
quirksNow = quirk_store_norm[classes_store == classNow][k*stepsize:(k+1)*stepsize].T
quirksNow.shape
classes_store_bak = np.copy(classes_store)
classNow  = 5
stepsize  = 50
fig = figure(figsize=(16,30))
for k in range( np.sum(classes_store == classNow) // stepsize):
    quirksNow = quirk_store_norm[classes_store == classNow][k*stepsize:(k+1)*stepsize]
#     upper     = np.where(quirksNow[:,-1] > 0.5)[0]
    lower     = np.where(quirksNow[:,-1] < 0.5)[0]
    classes_store[classes_store == classNow][lower] = np.ones(len(classes_store[classes_store == classNow][lower]))*6
#     ax = fig.add_subplot(np.int(np.ceil(np.sum(classes_store == classNow) // stepsize / 2)), 2, k+1)
#     plot(quirksNow[lower].T);
plot(quirk_store_norm[classes_store == 6].T);<jupyter_output><empty_output><jupyter_text>classes_store_new = np.copy(classes_store)classes_store_new[(classes_store == 5)*(quirk_store_norm[:,-1] < 0.5)]  = 6 classes_store_new[(classes_store == 5)*(quirk_store_norm[:,-1] >= 0.5)] = classes_store[(classes_store == 5)*(quirk_store_norm[:,-1] >= 0.5)]classes_store_new[classes_store_new == 6]np.savetxt('myclasses_new_FINAL_for_cnaw_mask_CV3_dark_frames.txt', classes_store_new.astype(int), fmt='%d')<jupyter_code>plot(quirk_store_norm[classes_store_new == 6].T);
mask  = mask0[0].data[0]
darks = darks0[0].data

nQuirks     = sum(mask != 0)
nFrames     = darks.shape[0]
nCols,nRows = darks.shape[1:]
med_darkImage = np.median(darks,axis=0)
darks.shape, med_darkImage.shape
darks_reshape = darks.reshape()
pp.
std_med_darkImage = np.std(darks - med_darkImage,axis=0)
mad_med_darkImage = scale.mad(darks - med_darkImage,axis=0)
# fig = figure(figsize=(16,16))
queries = transpose(darks,axes=(1,2,0))[mad_med_darkImage > 1.25*mean(mad_med_darkImage)]
fig = figure(figsize=(12,12))
ax  = fig.add_subplot(111)
ax.plot(queries.T - median(queries[:,:10],axis=1));
ax.set_xlim(0,108);
# ax.set_ylim(0,2**16);
quirks = np.zeros((nQuirks, nFrames))
norms  = np.zeros((nCols*nRows - nQuirks, nFrames))
qCnt   = 0
nCnt   = 0

for icol in range(darks.shape[1]):
    for irow in range(darks.shape[2]):
        if mask[icol,irow] != 0:
            # quirky.append(darks[:,icol,irow])
            quirks[qCnt]  = darks[:,icol,irow]
            qCnt          = qCnt + 1
            assert(qCnt <= nQuirks)
        else:
            norms[nCnt]   = darks[:,icol,irow]
            # normy.append(darks[:,icol,irow])
            nCnt          = nCnt + 1
norms.shape
np.median(norms[randUse],axis=1).shape
np.median(norms[randUse],axis=1).shape
nDisplay = 100000
randUse  = np.random.randint(0,nCols*nRows - nQuirks,nDisplay)
plot((norms[randUse].T - np.median(norms[randUse],axis=1)));
xlim(-1,110);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1757421234747769, 'fraction_numerical': 0.03278091422327445, 'mean_word_length': 4.564336372847011, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '634625', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2298, 'n_tokens_neox': 2207, 'n_words': 313, 'path': 'notebooks/RTN Classification-Backup.ipynb', 'repo_name': 'exowanderer/BadPixelDetector', 'size': '5491'}