<jupyter_start><jupyter_text>Find which data exists<jupyter_code>#settings.PROJECT_DATA_ROOT = '../../data/test'
settings.PROJECT_DATA_ROOT = '/home/projects/worm_movement/Data/WALDO_Data/'
#settings.PROJECT_DATA_ROOT = '/home/projects/worm_movement/Data/waldo_paper_review/WALDO/'

# #'../../data/test/20130318_131056/blob_files/'
# pl = pathlib.Path(settings.PROJECT_DATA_ROOT)
# # path = pl / eid / 'blob_files'
# # print(path)
# print(pl.absolute())
# for i in pl.glob('*'):
    
#     summary_list = list(i.glob('blob_files/*.summary'))
#     if not len(summary_list):
#         continue
#     print(i.name, summary_list[0].name)
    
    
#'../../data/test/20130318_131056/blob_files/'
pl = pathlib.Path(settings.PROJECT_DATA_ROOT)
print(pl.absolute())
files = {}
for i in pl.glob('*'):
    
    summary_list = list(i.glob('blob_files/*.summary'))
    if not len(summary_list):
        continue
    #print(i.name, summary_list[0].name)
    files[i.name] = summary_list[0].name.split('.summary')[0]

# names = sorted(list(files.keys()))
# for n in names:
#     print "'{n}',".format(n=n) , '#  --', files[n]<jupyter_output>/home/projects/worm_movement/Data/WALDO_Data<jupyter_text>Check if data grabbing works<jupyter_code>def check_name(eid, basepath=pl):
    for e in basepath.glob('*'):
        #print(e.name)
        if eid in e.name:
            if '-' in e.name:
                print(eid, e.name)
                return e.name
    return eid
            
eid = '20150608_121725'
check_name(eid, basepath=pl)

check_name('20150615_124728')
eid = '20150615_170754'
eid = '20150608_121725'
eid2 = check_name(eid)
path =  pl / eid2 / 'blob_files'
print(path)
test_list = fs.pull_blobs_from_eid(eid=eid2, path=path, 
                                   min_time=10, min_timepoints=100, dt=1.0)


print('test list is ', len(test_list), 'tracks')
df = test_list[0]
df.head()
fig, ax = plt.subplots(figsize=(12,3))

for df in test_list:
    d = pd.rolling_mean(df[['minutes', 'speed']], 60, center=True)
    ax.plot(d['minutes'], d['speed'], alpha=0.3)
plt.show()<jupyter_output><empty_output><jupyter_text>Start importing temp data<jupyter_code>h1 = [
    '20150518_182454',  # copper-10worms-N2-day1-standard_25degree
    '20150518_182501',  # copper-10worms-N2-day1-standard_25degree
    '20150518_182452',  # copper-10worms-N2-day1-standard_25degree
    ]
    
    
c1 = ['20150518_121756', # copper-10worms-N2-day1-standard_15degree
      '20150518_121757', # copper-10worms-N2-day1-standard_15degree
      '20150518_121809',  # copper-10worms-N2-day1-standard_15degree
    ]

h2 = [
'20150615_124720', # 	10 	copper-10worms-N2-day1-25degrees 	180
'20150615_124728', # 	10 	copper-10worms-N2-day1-25degrees 	180
'20150615_124738', # 	10 	copper-10worms-N2-day1-25degrees 	180
]
c2 = [
'20150615_170747', # 	10 	copper-10worms-N2-day1-15degrees 	180
'20150615_170754', # 	10 	copper-10worms-N2-day1-15degrees 	180
'20150615_170759', # 	10 	copper-10worms-N2-day1-15degrees 	180
]

h3 = [
'20150625_132445', #  -- copper-10worms-N2-day1-25 degree
#'20150625_132448', #  -- copper-10worms-N2-day1-25degree  -- Weirdness! huge speed spike in beginning
'20150625_132449', #  -- copper-10worms-N2-day1-25degree    
    
]
c3 = [

'20150625_180232', #  -- copper-10worms-N2-day1-15degree
#'20150625_180236', #  -- copper-10worms-N2-day1-15degree
'20150625_180237', #  -- copper-10worms-N2-day1-15degree

]

h = h1 + h2 + h3
c = c1 + c2 + c3
print('hot')
print(h)
print('cold')
print(c)
eids20 = [
'20150511_121312', #  -- copper-10worms-N2-day1-standard
'20150511_121317', #  -- copper-10worms-N2-day1-standard
'20150511_160123', #  -- copper-10worms-N2-day1-populationtest
'20150608_121725', #  -- copper-10worms-N2-day1-standard
'20150622_121933', #  -- copper-40worms-N2-day1-standard
'20150622_121935', #  -- copper-20worms-N2-day1-standard
'20150622_121940', #  -- copper-60worms-N2-day1-standard
'20150629_165904', #  -- copper-10worms-N2-day1-standard
'20150702_150024', #  -- copper-10worms-N2-day1-standard             
]
def process_eid_list(eids, pl, total_seconds, min_time=20, min_timepoints=1000, dt=1.0):
    paths = [pl / eid / 'blob_files' for eid in eids]
    blobs = []
    plate_dfs = []
    total_seconds = 10801
    for eid, path in zip(eids, paths):
        if not path.exists():
            print('pass')
            continue
        print(eid, path)
        some_blobs = fs.pull_blobs_from_eid(eid=eid, path=path, min_time=20, min_timepoints=1000, dt=1.0)
        blobs.append(some_blobs)
        d = fs.aggregate_track_dfs(some_blobs, total_seconds=total_seconds)
        plate_dfs.append(d)
        
    allblobs = []
    agg_df = []
    for bs in blobs:
        allblobs = allblobs + bs
        agg_df = fs.aggregate_track_dfs(allblobs, total_seconds=total_seconds)
    return paths, blobs, plate_dfs, agg_df

total_seconds = 10801
h_paths, h_blobs, h_plate_dfs, h_df = process_eid_list(h, pl=pl, total_seconds=total_seconds)
h_blobs[0][0].columns
# for p, df in zip(h_paths, h_plate_dfs):
#     print(p)
#     plt.plot(df['time'], df['mean'])
#     plt.show()

rt_paths, rt_blobs, rt_plate_dfs, rt_df = process_eid_list(eids20, pl=pl, total_seconds=total_seconds)
# START WORING HERE!
print('DONE')
c_paths, c_blobs, c_plate_dfs, c_df = process_eid_list(c, pl=pl, total_seconds=total_seconds)
h_df = h_df[::10]
rt_df = rt_df[::10]
c_df = c_df[::10]
fig = plt.figure(figsize=(12,5))
ax = plt.subplot(axisbg='white')
#fig, ax = plt.subplots(figsize=(12,5))
stat = 'mean'

ax.plot(h_df['time']/ 60, h_df[stat], color='red', alpha=0.5, label='T=25', lw=5)
ax.plot(rt_df['time']/ 60, rt_df[stat], color='k', alpha=0.5, label='T=20', lw=5)
ax.plot(c_df['time']/ 60, c_df[stat], color='blue', alpha=0.5, label='T=15', lw=5)

#ax.legend(loc='best', fontsize=20, frameon=False)
ax.set_ylim([0, .25])
# ax.set_ylabel('mean speed (bl / s)', size=20)
# ax.set_xlabel('time (min)', size=20)
#plt.yticks(fontsize=24)
plt.yticks([0, 0.1, 0.2], [], fontsize=24)
#plt.xticks([0, 60, 120, 180], [0, 60, 120, 180], fontsize=24)
plt.xticks([0, 30, 60, 90, 120, 150, 180], [], fontsize=24)

#plt.xticks([0, 60, 120, 180], [], fontsize=24)

plt.savefig('temp-combined-all-small-notick.eps')
plt.show()
#ax.set_ylim([0, 2])
fig = plt.figure(figsize=(12,5))
ax = plt.subplot(axisbg='white')
#fig, ax = plt.subplots(figsize=(12,5))
stat = 'mean'
for df in h_plate_dfs:
    ax.plot(df['time'][::10]/ 60, df[stat][::10], color='red', lw=0.5, alpha=0.4)
    
ax.plot(h_df['time']/ 60, h_df[stat], color='red', alpha=0.5, label='T=25', lw=5)
#ax.plot(rt_df['time']/ 60, rt_df[stat], color='k', alpha=0.7, label='T=20', lw=3)
ax.plot(c_df['time']/ 60, c_df[stat], color='blue', alpha=0.5, label='T=15', lw=5)

#ax.legend(loc='best', fontsize=20, frameon=False)
ax.set_ylim([0, .25])
# ax.set_ylabel('mean speed (bl / s)', size=20)
# ax.set_xlabel('time (min)', size=20)
#plt.yticks(fontsize=24)
plt.yticks([0, 0.1, 0.2], [], fontsize=24)
#plt.xticks([0, 60, 120, 180], [0, 60, 120, 180], fontsize=24)
plt.xticks([0, 30, 60, 90, 120, 150, 180], [], fontsize=24)

#plt.xticks([0, 60, 120, 180], [], fontsize=24)

plt.savefig('temp-combined-t25-small-notick.eps')
plt.show()
#ax.set_ylim([0, 2])
#fig, ax = plt.subplots(figsize=(12,5))
fig = plt.figure(figsize=(12,5))
ax = plt.subplot(axisbg='white')
stat = 'q2'
stat = 'mean'

    


# for df in t20_dfs:
#     ax.plot(df['time']/ 60, df[stat], color='black', alpha=0.5, label='T=20')
for df in c_plate_dfs:
    ax.plot(df['time'][::10]/ 60, df[stat][::10], color='blue', lw=0.5, alpha=0.5)
    
    
ax.plot(h_df['time']/ 60, h_df[stat], color='red', alpha=0.5, label='T=25', lw=5)
#ax.plot(rt_df['time']/ 60, rt_df[stat], color='k', alpha=0.7, label='T=20', lw=3)
ax.plot(c_df['time']/ 60, c_df[stat], color='blue', alpha=0.5, label='T=15', lw=5)

#ax.legend(loc='best', fontsize=20, frameon=False)
#ax.set_ylim([0, 20])
ax.set_ylim([0, .25])

#ax.set_ylabel('mean speed (bl / s)', size=20)
#ax.set_xlabel('time (min)', size=20)
#plt.savefig('temp-speed-means2.png')
plt.yticks([0, 0.1, 0.2], [0, 0.1, 0.2], fontsize=24)
plt.xticks([0, 30, 60, 90, 120, 150, 180], [0, 30, 60, 90, 120, 150, 180], fontsize=24)
plt.yticks([0, 0.1, 0.2], [], fontsize=24)
plt.xticks([0, 30, 60, 90, 120, 150, 180], [], fontsize=24)
plt.savefig('temp-combined-t15-small-notick.eps')
plt.show()
#ax.set_ylim([0, 2])
#fig, ax = plt.subplots(figsize=(12,5))
fig = plt.figure(figsize=(12,5))
ax = plt.subplot(axisbg='white')
stat = 'q2'
stat = 'mean'

ax.plot(h_df['time']/ 60, h_df[stat], color='red', alpha=0.5, label='T=25', lw=3)

# for df in t20_dfs:
#     ax.plot(df['time']/ 60, df[stat], color='black', alpha=0.5, label='T=20')
for df in h_plate_dfs:
    #print(df.head(70))
    ax.plot(df['time']/ 60, df[stat], color='red', lw=0.5, alpha=0.4)

for df in c_plate_dfs:
    ax.plot(df['time']/ 60, df[stat], color='blue', lw=0.5, alpha=0.5)
    
ax.plot(c_df['time']/ 60, c_df[stat], color='blue', alpha=0.5, label='T=15', lw=3)

ax.legend(loc='best', fontsize=20, frameon=False)
ax.set_ylim([0, 20])

ax.set_ylabel('mean speed (pxl / s)', size=20)
ax.set_xlabel('time (min)', size=20)
#plt.savefig('temp-speed-means2.png')
plt.yticks(fontsize=14)
plt.xticks(fontsize=14)
plt.savefig('temp-combined-both.eps')
plt.show()
#ax.set_ylim([0, 2])<jupyter_output>/home/visitors/peterwinter/.pyenv/versions/anaconda-2.1.0/lib/python2.7/site-packages/matplotlib/font_manager.py:1289: UserWarning: findfont: Could not match :family=Helvetica:style=normal:variant=normal:weight=normal:stretch=normal:size=14.0. Returning /home/visitors/peterwinter/.pyenv/versions/general/local/lib/python2.7/site-packages/matplotlib/mpl-data/fonts/afm/phvl8a.afm
  UserWarning)
/home/visitors/peterwinter/.pyenv/versions/anaconda-2.1.0/lib/python2.7/site-packages/matplotlib/font_manager.py:1289: UserWarning: findfont: Could not match :family=Helvetica:style=normal:variant=normal:weight=normal:stretch=normal:size=20.0. Returning /home/visitors/peterwinter/.pyenv/versions/general/local/lib/python2.7/site-packages/matplotlib/mpl-data/fonts/afm/phvl8a.afm
  UserWarning)<jupyter_text>Paper observation windows<jupyter_code>windows = pd.read_csv('/home/visitors/peterwinter/Dropbox/research/literature_dat.csv')
# windows =[
# {'authors':'Stephens et al. (2008)', 'acc':1, 'obs':60},
# {'authors':'Yu et al. (2013)', 'acc':0, 'obs':4},
# {'authors':'Silva et al. (2011)', 'acc':0, 'obs':1},
# {'authors':'Yemini et al. (2013)', 'acc':30, 'obs':15},
# ]
# w = pd.DataFrame(windows)
w = windows
w
#w = w.sort('delay (min)')
w = w.sort('duration (min)', ascending=False)
w
import matplotlib.patches as patches
from matplotlib.lines import Line2D

# axis params
tick_label_size = 12


# get data ready
# xs = list(w['delay (min)'])
# x0 = [0 for _ in xs]
# widths = list(w['duration (min)'])
# height = 1
# steps = []


# Make figure

fig, ax = plt.subplots(figsize=(12,5))
ax = plt.axes(frameon=False)
ax.get_xaxis().tick_bottom() # Turn off ticks at top of plot
ax.axes.get_yaxis().set_visible(False)


color = ax._get_lines.color_cycle.next()

for y, (i, row) in enumerate(w.iterrows()):
    x = row['delay (min)']
    width = row['duration (min)']
    print(y, x, width)
    step = patches.Rectangle((x, y), height=height*0.8, width=width,
                             fill=True, ec='k', #fc=color, ec=color,
                             alpha=0.5)
    ax.add_patch(step)
    if x !=0:
        ax.plot([0, x], [y+0.5, y+0.5], 'k', alpha=0.8, lw=2)
    
#     ax.text(120, y+0.5, row['key'],
#             verticalalignment='bottom', horizontalalignment='left',
#             #transform=ax.transAxes,
#             color='k', fontsize=14)
ax.plot([0, 0], [0, y+1], 'k')
        

ax.set_xlim([0, 180])

xmin, xmax = ax.get_xaxis().get_view_interval()
ymin, ymax = ax.get_yaxis().get_view_interval()
ax.add_artist(Line2D((xmin, xmax), (ymin, ymin), color='black', linewidth=2))
ax.add_artist(Line2D((xmin, xmin), (ymin, ymax), color='black', linewidth=2))
ax.set_xlabel('Time (min)', size=15)
#plt.yticks(fontsize=tick_label_size)
plt.xticks(fontsize=tick_label_size)
plt.savefig('literature-coverage-chart.png')
plt.show()
480 / 60<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1952662721893491, 'fraction_numerical': 0.09944115713346482, 'mean_word_length': 4.295474325500435, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '214281', 'license': 'MIT', 'n_tokens_mistral': 5968, 'n_tokens_neox': 5210, 'n_words': 851, 'path': 'workbooks/Temperature-Figure/temp_fig.ipynb', 'repo_name': 'amarallab/waldo', 'size': '12168'}