<jupyter_start><jupyter_text>Seem to need to do one run before the profilingEvery time the first run looks different than the rest, so we run one and throw it out.<jupyter_code>_, _ = run_memory_profile(num_files, sampling_interval, size=image_size, 
                          memory_limit=memory_limit, combine_method='average')<jupyter_output><empty_output><jupyter_text>Memory profile without sigma clipping<jupyter_code>n_repetitions = 4
def run_them(runs, clipping=False):
    for combine_method in runs.keys():
        for _ in range(n_repetitions):
            mem_use, img_size = run_memory_profile(num_files, sampling_interval, size=image_size, 
                                                   memory_limit=memory_limit, combine_method=combine_method,
                                                   sigma_clip=clipping)
            gc.collect()
            runs[combine_method]['times'].append(np.arange(len(mem_use)) * sampling_interval)
            runs[combine_method]['memory'].append(mem_use)
            runs[combine_method]['image_size'] = img_size
            runs[combine_method]['memory_limit'] = memory_limit
            runs[combine_method]['clipping'] = clipping
run_them(runs)
styles = ['solid', 'dashed', 'dotted']
plt.figure(figsize=(20, 10))

for idx, method in enumerate(runs.keys()):
    style = styles[idx % len(styles)]
    for i, data in enumerate(zip(runs[method]['times'], runs[method]['memory'])):
        time, mem_use = data 
        if i == 0:
            label = 'Memory use in {} combine (repeated runs same style)'.format(method)
            alpha = 1.0
        else:
            label = ''
            alpha = 0.4
        plt.plot(time, mem_use, linestyle=style, label=label, alpha=alpha)

plt.vlines(-40 * sampling_interval, mem_use[0], mem_use[0] + memory_limit/1e6, colors='red', label='Memory use limit')
plt.vlines(-20 * sampling_interval, mem_use[0], mem_use[0] + runs[method]['image_size']/1e6, label='size of one image')

plt.grid()
clipped = 'ON' if runs[method]['clipping'] else 'OFF'

plt.title('ccdproc commit {}; {} repetitions per method; sigma_clip {}'.format(commit, n_repetitions, clipped),
          fontsize=20)
plt.xlabel('Time (sec)', fontsize=20)
plt.ylabel('Memory use (MB)', fontsize=20)

plt.legend(fontsize=20)
plt.savefig('commit_{}_reps_{}_clip_{}_memlim_{}GB.png'.format(commit, n_repetitions, clipped, memory_limit/1e9))<jupyter_output><empty_output><jupyter_text>Memory profile with sigma clipping<jupyter_code>run_them(runs_clip, clipping=True)
plt.figure(figsize=(20, 10))

for idx, method in enumerate(runs_clip.keys()):
    style = styles[idx % len(styles)]
    for i, data in enumerate(zip(runs_clip[method]['times'], runs_clip[method]['memory'])):
        time, mem_use = data 
        if i == 0:
            label = 'Memory use in {} combine (repeated runs same style)'.format(method)
            alpha = 1.0
        else:
            label = ''
            alpha = 0.4
        plt.plot(time, mem_use, linestyle=style, label=label, alpha=alpha)

plt.vlines(-40 * sampling_interval, mem_use[0], mem_use[0] + memory_limit/1e6, colors='red', label='Memory use limit')
plt.vlines(-20 * sampling_interval, mem_use[0], mem_use[0] + runs_clip[method]['image_size']/1e6, label='size of one image')

plt.grid()
clipped = 'ON' if runs_clip[method]['clipping'] else 'OFF'

plt.title('ccdproc commit {}; {} repetitions per method; sigma_clip {}'.format(commit, n_repetitions, clipped),
          fontsize=20)
plt.xlabel('Time (sec)', fontsize=20)
plt.ylabel('Memory use (MB)', fontsize=20)

plt.legend(fontsize=20)
plt.savefig('commit_{}_reps_{}_clip_{}_memlim_{}GB.png'.format(commit, n_repetitions, clipped, memory_limit/1e9))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13879957127545553, 'fraction_numerical': 0.016881028938906754, 'mean_word_length': 3.7797695262483995, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '116034', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1206, 'n_words': 253, 'path': 'ccdproc/tests/run_profile.ipynb', 'repo_name': 'jryon/ccdproc', 'size': '3732'}