<jupyter_start><jupyter_text>Tracking> Simple Store, Pull, and Visualize data in nosql format<jupyter_code>#export
import shelve
import matplotlib.pyplot as plt
import os
from fastcore.foundation import *
import numpy as np
test_dict1 = {'name':'resnet18',
               'epoch_loss':[8.,4.,2.],
               'epoch_accuracy':[.90,.94,.96]
            }
test_dict2 = {'name':'resnet34',
               'notes':f'test note 2',
               'epoch_loss':[6.,3.,1.],
               'epoch_accuracy':[.91,.95,.97],
               'batch_loss':[10.,8.,6.1,4.5,3.2,2.5]
            }
test_dict3 = {'name':'resnet50',
               'notes':f'test note 3',
               'epoch_loss':[5.,2.,.5],
               'epoch_accuracy':[.93,.97,.99],
               'batch_loss':[9.,7.,5.1,3.5,2.2,1.5]
            }<jupyter_output><empty_output><jupyter_text>Shelve Writing<jupyter_code>#export
def create(filename,keys): 
  with shelve.open(filename) as d: 
    if type(keys) == str: d[keys] = L()
    else: 
        for key in keys: d[key] = L()
#export
def append(filename,new_dict,key='exp'):
    '''Append a new_dict to list store in key - create db if needed'''
    if not os.path.exists(filename): create(filename,key)
    with shelve.open(filename) as d: 
        if key not in list(d.keys()): d[key] = L()
        tmp = d[key]
        tmp.append(new_dict)
        d['exp'] = tmp   
filename = 'test.shelve'
if os.path.exists(filename): os.remove(filename)
  
for new_dict in [test_dict1,test_dict2,test_dict3]: append(filename,new_dict)
with shelve.open(filename) as d: assert d['exp'] == [test_dict1,test_dict2,test_dict3]
#export
def delete(filename,exp_num,key='exp'):
    '''delete an item from list stored in key'''
    with shelve.open(filename) as d: 
        tmp = d[key]
        if type(exp) == int: 
            tmp.pop(exp_num)
        if type(exp) == str:         
            for i,e in eumerate(tmp): 
              if e['name'] == name: tmp.pop(i)
        d[key] = tmp<jupyter_output><empty_output><jupyter_text>Shelve Querying<jupyter_code>#export
def print_keys(filename,last_only=True, with_type=False, key='exp'):
    with shelve.open(filename) as d:
        if last_only and not with_type: print(list(d[key][-1].keys()))
        if last_only and with_type: print({k:type(v) for k,v in d['exp'][-1].items()})
        if not last_only and not with_type:
            a = L()
            for o in d['exp']: a = a + L(o.keys())
            a = a.unique()
            print(a)
        if not last_only and with_type:
            a = {}
            for o in d['exp']: a.update({k:type(v) for k,v in o.items()})
            print(a)
print_keys(filename)
print('')
print_keys(filename,with_type=True)
#export
def get_stat(filename,exp_num,stat,key='exp',display=True):
    '''get a specific stat (ie loss) from key for a given expirament
    Goes well with partial
    '''
    with shelve.open(filename) as d: 
      if display: print(f'd[{key}][{exp_num}][{stat}]: {d[key][exp_num][stat]}')
      return (f'd[{key}][{exp_num}][{stat}]',d[key][exp_num][stat])
a,b = get_stat(filename,-1,'epoch_loss',display=True)
assert b == [5.0, 2.0, 0.5]
#export 
def get_stats(filename,exp_num,stats,key='exp',display=True):
    return [get_stat(filename,exp_num,stat,key,display) for stat in stats]
a,b,c = get_stats(filename,-1,['epoch_loss','epoch_accuracy','batch_loss'],display=True)
assert a[1] == [5.0, 2.0, 0.5]
assert b[1] == [0.93, 0.97, 0.99]
assert c[1] == [9.0, 7.0, 5.1, 3.5, 2.2, 1.5]
#export 
def print_best(filename,stat,best='min', key='exp'):
    with shelve.open(filename) as d: exps = len(d[key])
    if best == 'min':
        out = (np.inf,None)
        for i in range(exps):
          a,b = get_stat(filename,i,stat,display=False)
          if min(b) < out[0]: out = (min(b),i)
    if best == 'max':
        out = (-np.inf,None)
        for i in range(exps):
          a,b = get_stat(filename,i,stat,display=False)
          if max(b) > out[0]: out = (max(b),i)
            
    print(f'{stat} {best} value = {out[0]} | best idx = {out[1]-exps}')
print_best(filename,'epoch_loss',best='min')
print_best(filename,'epoch_accuracy',best='max')<jupyter_output>epoch_loss min value = 0.5 | best idx = -1
epoch_accuracy max value = 0.99 | best idx = -1<jupyter_text>Shelve Graphing<jupyter_code>#export
def graph_stat(filename,stat,idxs=[-1,-2,-3], key='exp',name='name',figsize=(12,6)):
    with shelve.open(filename) as d: 
        fig,ax = plt.subplots(figsize=figsize)
        for e in L(d[key])[idxs]:
            try:
              vals = e[stat]
              ax.plot(range(len(e[stat])),e[stat],label=e[name])
              ax.legend();ax.set_title(f'{stat}')
            except:
              print(f'Unable to plot {stat} for {e[name]}')
              
def graph_stats(filename,stats,idxs=[-1,-2,-3],key='exp',name='name',figsize=(12,6)):
    for stat in stats: 
        graph_stat(filename,stat,idxs,key,name,figsize)
graph_stats(filename,['batch_loss','epoch_accuracy'],idxs=[-1,-2,-3])<jupyter_output>Unable to plot batch_loss for resnet18
{'fraction_non_alphanumeric': 0.17478792661274414, 'fraction_numerical': 0.026237916748865654, 'mean_word_length': 3.007905138339921, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '531243', 'license': 'Apache-2.0', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1884, 'n_words': 380, 'path': '03_Tracking.ipynb', 'repo_name': 'Isaac-Flath/perconv', 'size': '5069'}