<commit_before>"""
Config.py contains configuration data like constants
Also contains notes on the data sets

author: chris @ sihrc
"""
import os
import pickle as p
from wrappers import debug


"""
Data sets
"""

codebook = "http://meps.ahrq.gov/mepsweb/data_stats/download_data/pufs/{0}/{0}su.txt"
download = "http://meps.ahrq.gov/data_files/pufs/%sdat.zip"
tables = "http://meps.ahrq.gov/mepsweb/data_stats/download_data/pufs/{0}/{0}doc.shtml"
def path(*path):
    """
    Replacement for os.path.join
    It performs makedirs on paths that don't exists
    Returns the os.path.join() result   
    author:chris
    """
    if len(path) == 0:
        return ""
    targetpath = os.path.join(*path)
    if "." in path[-1]:
        path = path[:-1]
        targetdirs = os.path.join(*path)
        if not os.path.exists(targetdirs):
            os.makedirs(targetdirs) 
    return targetpath

def get(fpath, func, **kwargs):
    print "Checking %s ..." % fpath
    if os.path.exists(fpath):
        return p.load(open(fpath, 'rb'))
    res = func(**kwargs)
    with open(fpath, 'wb') as f:
        print "Saved to %s" % fpath
        p.dump(res, f)
    return res

def load(fpath):
    if os.path.exists(fpath):
        print "Cache Loading from %s" % fpath
        return p.load(open(fpath, 'rb'))
    else:
        return None

def save(fpath, data):
    p.dump(data, open(fpath, 'wb'))

@debug
def clean(args, datafile):
    """
    Config - clean function
    Clean previously cached items
    """
    import os
    import shutil
    delPath = path("..","data",datafile)
    if not os.path.exists(delPath): return<commit_msg>Added documentation to config.py<commit_after>"""
Config.py contains configuration data like constants
Also contains notes on the data sets

author: chris @ sihrc
"""
import os
import pickle as p
from wrappers import debug


"""
Data sets
"""

codebook = "http://meps.ahrq.gov/mepsweb/data_stats/download_data/pufs/{0}/{0}su.txt"
download = "http://meps.ahrq.gov/data_files/pufs/%sdat.zip"
tables = "http://meps.ahrq.gov/mepsweb/data_stats/download_data/pufs/{0}/{0}doc.shtml"

def path(*path):
    """
    in config.py
    It performs makedirs on paths that don't exists
    Returns the path   
    """
    if len(path) == 0:
        return ""
    targetpath = os.path.join(*path)
    if "." in path[-1]:
        path = path[:-1]
        targetdirs = os.path.join(*path)
        if not os.path.exists(targetdirs):
            os.makedirs(targetdirs) 
    return targetpath

def get(fpath, func, **kwargs):
    """
    in config.py
    Caching function
    """
    print "Checking %s ..." % fpath
    if os.path.exists(fpath):
        return p.load(open(fpath, 'rb'))
    res = func(**kwargs)
    with open(fpath, 'wb') as f:
        print "Saved to %s" % fpath
        p.dump(res, f)
    return res

def load(fpath):
    """
    in config.py
    Pickle load that checks for path
    """
    if os.path.exists(fpath):
        print "Cache Loading from %s" % fpath
        return p.load(open(fpath, 'rb'))
    else:
        return None

def save(fpath, data):
    """
    in config.py
    Pickle save
    """
    p.dump(data, open(fpath, 'wb'))

@debug
def clean(args, datafile):
    """
    in config.py
    Config - clean function
    Clean previously cached items
    """
    import os
    import shutil
    delPath = path("..","data",datafile)
    if not os.path.exists(delPath): return
{'fraction_non_alphanumeric': 0.14189584552369808, 'fraction_numerical': 0.004095962551199532, 'mean_word_length': 2.6065400843881856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491737', 'n_tokens_mistral': 1196, 'n_tokens_neox': 1175, 'n_words': 305}