<commit_before>import pandas as pd
import csv
from progressivis import Scheduler
from progressivis.io import CSVLoader
from progressivis.table.constant import Constant
from progressivis.table.table import Table
from progressivis.datasets import get_dataset
from benchmarkit import BenchEnv, BenchmarkIt, InputProc
from progressivis.core.storage import StorageEngine
#from progressivis.core.storage.zarr import ZARRStorageEngine
import sys
import os
import os.path
import subprocess
import glob
from collections import OrderedDict
from progressivis import Print, Scheduler
from progressivis.stats import Min, Max, RandomTable

import dask.dataframe as dd

def cleanup_hdf5():
    for d in glob.glob('/var/tmp/progressivis_*'):
        subprocess.call(['/bin/rm','-rf', d])
    for d in glob.glob('/tmp/progressivis_*'):
        subprocess.call(['/bin/rm','-rf', d])



def make_df(n, L):
    s=Scheduler()
    random = RandomTable(10, rows=n*L, scheduler=s)
    s.start()<commit_msg>Fix path for storage in bench<commit_after>import pandas as pd
import csv
from progressivis import Scheduler
from progressivis.io import CSVLoader
from progressivis.table.constant import Constant
from progressivis.table.table import Table
from progressivis.datasets import get_dataset
from benchmarkit import BenchEnv, BenchmarkIt, InputProc
from progressivis.storage import StorageEngine
#from progressivis.storage.zarr import ZARRStorageEngine
import sys
import os
import os.path
import subprocess
import glob
from collections import OrderedDict
from progressivis import Print, Scheduler
from progressivis.stats import Min, Max, RandomTable

import dask.dataframe as dd

def cleanup_hdf5():
    for d in glob.glob('/var/tmp/progressivis_*'):
        subprocess.call(['/bin/rm','-rf', d])
    for d in glob.glob('/tmp/progressivis_*'):
        subprocess.call(['/bin/rm','-rf', d])



def make_df(n, L):
    s=Scheduler()
    random = RandomTable(10, rows=n*L, scheduler=s)
    s.start()
{'fraction_non_alphanumeric': 0.09137055837563451, 'fraction_numerical': 0.003045685279187817, 'mean_word_length': 4.972727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3714690', 'n_tokens_mistral': 606, 'n_tokens_neox': 583, 'n_words': 188}