<commit_before>import datetime
from os import path

from fabric.api import env, run, sudo

def postgresql(database, file):
    owner = sudo("stat -c %%U `dirname %s`" % file)
    sudo("chown postgres `dirname %s`" % file)
    sudo("pg_dump %s > %s" % (database, file), user='postgres')
    sudo("chown %s `dirname %s` %s" % (owner, file, file))

class TmpDir(object):
    def __enter__(self):
        self.dir_name = run("mktemp -d")
        return self.dir_name
    
    def __exit__(self, type, value, traceback):
        sudo("rm -rf %s" % self.dir_name)

def generate_name():
    hostname = run("hostname")
    date = datetime.datetime.now().isoformat('_')
    return "%s_%s" % (hostname, date)

def archive_dir(source_path, target_path, archive_name):
    file_name = path.join(target_path, "%s.tar" % archive_name)
    run('cd %s; tar c * > "%s"' % (source_path, file_name))
    return file_name<commit_msg>Add compression.<commit_after>import datetime
from os import path

from fabric.api import env, run, sudo

def postgresql(database, file):
    owner = sudo("stat -c %%U `dirname %s`" % file)
    sudo("chown postgres `dirname %s`" % file)
    sudo("pg_dump %s > %s" % (database, file), user='postgres')
    sudo("chown %s `dirname %s` %s" % (owner, file, file))

class TmpDir(object):
    def __enter__(self):
        self.dir_name = run("mktemp -d")
        return self.dir_name
    
    def __exit__(self, type, value, traceback):
        sudo("rm -rf %s" % self.dir_name)

def generate_name():
    hostname = run("hostname")
    date = datetime.datetime.now().isoformat('_')
    return "%s_%s" % (hostname, date)

def archive_dir(source_path, target_path, archive_name):
    file_name = path.join(target_path, "%s.tar" % archive_name)
    run('cd %s; tar c * > "%s"' % (source_path, file_name))
    return file_name

def compress(file_name):
    run('bzip2 -9 "%s"' % file_name)
    return "%s.bz2" % file_name
{'fraction_non_alphanumeric': 0.158004158004158, 'fraction_numerical': 0.0015592515592515593, 'mean_word_length': 3.63855421686747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6400279', 'n_tokens_mistral': 695, 'n_tokens_neox': 678, 'n_words': 181}