<commit_before>import gzip
import bz2
import json
import cPickle as pickle
import os


def load(fn):
    if not os.path.exists(fn):
        raise IOError('Path does not exist [%s]' % fn)
    try:
        base, ext = os.path.splitext(fn)
    except ValueError:
        raise ValueError('File needs an extension')
    if ext[-3:] == '.gz':
        fp = gzip.GzipFile(fn)
        ext = ext[:-3]
    elif ext[-4:] == '.bz2':
        fp = bz2.BZ2File(fn)
        ext = ext[:-4]
    else:
        fp = open(fn)
    if ext == '.js':
        return json.load(fp)
    elif ext == '.pkl':
        return pickle.load(fp)
    else:
        raise ValueError('File [%s] has unknown extension [%s]' % (fn, ext))


def dump(obj, fn):
    try:
        base, ext = os.path.splitext(fn)
    except ValueError:
        raise ValueError('File needs an extension')
    if ext[-3:] == '.gz':
        fp = gzip.GzipFile(fn, 'w')
        ext = ext[:-3]
    elif ext[-4:] == '.bz2':
        fp = bz2.BZ2File(fn, 'w')
        ext = ext[:-4]
    else:
        fp = open(fn, 'w')
    if ext == '.js':
        return json.dump(obj, fp)
    elif ext == '.pkl':
        return pickle.dump(obj, fp, -1)
    else:
        raise ValueError('File [%s] has unknown extension [%s]' % (fn, ext))<commit_msg>Fixed parse Project (tp) Signed-off-by: <NAME> <<EMAIL>><commit_after>import gzip
import bz2
import json
import cPickle as pickle
import os


def _getext(fn):
    return fn.rsplit('/', 1)[-1].split('.', 1)[-1]


def load(fn):
    if not os.path.exists(fn):
        raise IOError('Path does not exist [%s]' % fn)
    try:
        ext = _getext(fn)
    except ValueError:
        raise ValueError('File needs an extension')
    if ext[-3:] == '.gz':
        fp = gzip.GzipFile(fn)
        ext = ext[:-3]
    elif ext[-4:] == '.bz2':
        fp = bz2.BZ2File(fn)
        ext = ext[:-4]
    else:
        fp = open(fn)
    if ext == 'js':
        return json.load(fp)
    elif ext == 'pkl':
        return pickle.load(fp)
    else:
        raise ValueError('File [%s] has unknown extension [%s]' % (fn, ext))


def dump(obj, fn):
    try:
        ext = _getext(fn)
    except ValueError:
        raise ValueError('File needs an extension')
    if ext[-3:] == '.gz':
        fp = gzip.GzipFile(fn, 'w')
        ext = ext[:-3]
    elif ext[-4:] == '.bz2':
        fp = bz2.BZ2File(fn, 'w')
        ext = ext[:-4]
    else:
        fp = open(fn, 'w')
    if ext == 'js':
        return json.dump(obj, fp)
    elif ext == 'pkl':
        return pickle.dump(obj, fp, -1)
    else:
        raise ValueError('File [%s] has unknown extension [%s]' % (fn, ext))
{'fraction_non_alphanumeric': 0.16978967495219885, 'fraction_numerical': 0.013766730401529637, 'mean_word_length': 2.038327526132404, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741994', 'n_tokens_mistral': 964, 'n_tokens_neox': 948, 'n_words': 267}