<commit_before>#!/usr/bin/env python
# <NAME> <<EMAIL>>
import os, sys
import errno
import argparse
import tarfile
import gzip

try:
   import cPickle as pickle
except:
   import pickle

##__________________________________________________________________||
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs = argparse.REMAINDER, help = 'paths to task packages')
args = parser.parse_args()

##__________________________________________________________________||
def main():

    setup()

    for package_path in args.paths:

        # e.g., package_path = 'task_00003.p.gz',
        # a relative to the directory in which this file is stored

        thisdir = os.path.dirname(__file__)
        package_path = os.path.join(thisdir, package_path)
        # e.g., package_path = 'c/d/task_00003.p.gz'

        result = run(package_path)

        result_path = compose_result_path(package_path)
        # e.g., '/a/b/c/d/results/task_00003/result.p.gz'

        store_result(result, result_path)

##__________________________________________________________________||
def setup():
    dirname = 'python_modules'
    tarname = dirname + '.tar.gz'
    if os.path.exists(tarname) and not os.path.exists(dirname):
        tar = tarfile.open(tarname)
        tar.extractall()
        tar.close()

    if not os.path.exists(dirname): return

    sys.path.insert(0, dirname)

##__________________________________________________________________||
def run(package_path):
    f = gzip.open(package_path, 'rb')
    package = pickle.load(f)
    result = package.task(*package.args, **package.kwargs)
    return result

##__________________________________________________________________||
def compose_result_path(package_path):

    # e.g., package_path = 'c/d/task_00003.p.gz'

    taskdir = os.path.dirname(os.path.abspath(package_path))
    # e.g., '/a/b/c/d'

    result_topdir = os.path.join(taskdir, 'results')
    # e.g., '/a/b/c/d/results'

    package_basename =  os.path.basename(package_path)
    # e.g., 'task_00003.p.gz'

    resultdir_basename = os.path.splitext(package_basename)[0]
    resultdir_basename = os.path.splitext(resultdir_basename)[0]
    # e.g., 'task_00003'

    resultdir = os.path.join(result_topdir, resultdir_basename)
    # e.g., '/a/b/c/d/results/task_00003'

    result_path = os.path.join(resultdir, 'result.p.gz')
    # e.g., '/a/b/c/d/results/task_00003/result.p.gz'

    return result_path

##__________________________________________________________________||
def store_result(result, result_path):
    mkdir_p(os.path.dirname(result_path))
    f = gzip.open(result_path, 'wb')
    pickle.dump(result, f, protocol = pickle.HIGHEST_PROTOCOL)

##__________________________________________________________________||
def mkdir_p(path):
    # http://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
    try:
        os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

##__________________________________________________________________||
if __name__ == '__main__':
    main()
<commit_msg>check if untarring is in progress<commit_after>#!/usr/bin/env python
# <NAME> <<EMAIL>>
import os, sys
import errno
import argparse
import tarfile
import gzip

try:
   import cPickle as pickle
except:
   import pickle

##__________________________________________________________________||
parser = argparse.ArgumentParser()
parser.add_argument('paths', nargs = argparse.REMAINDER, help = 'paths to task packages')
args = parser.parse_args()

##__________________________________________________________________||
def main():

    setup()

    for package_path in args.paths:

        # e.g., package_path = 'task_00003.p.gz',
        # a relative to the directory in which this file is stored

        thisdir = os.path.dirname(__file__)
        package_path = os.path.join(thisdir, package_path)
        # e.g., package_path = 'c/d/task_00003.p.gz'

        result = run(package_path)

        result_path = compose_result_path(package_path)
        # e.g., '/a/b/c/d/results/task_00003/result.p.gz'

        store_result(result, result_path)

##__________________________________________________________________||
def setup():
    dirname = 'python_modules'
    tarname = dirname + '.tar.gz'
    print os.getcwd()
    print os.listdir(os.getcwd())

    if os.path.exists(tarname) and not os.path.exists(dirname):
        if try_make_file('.untarring'):
            tar = tarfile.open(tarname)
            tar.extractall()
            tar.close()
            os.remove('.untarring')

    while os.path.isfile('.untarring'):
       pass

    if not os.path.exists(dirname): return

    sys.path.insert(0, dirname)

##_______________________________________________________________||
# http://stackoverflow.com/questions/33223564/atomically-creating-a-file-if-it-doesnt-exist-in-python
def try_make_file(filename):
    try:
        os.open(filename,  os.O_CREAT | os.O_EXCL)
        return True
    except OSError:
        # FileExistsError can be used for Python 3
        return False

##__________________________________________________________________||
def run(package_path):
    f = gzip.open(package_path, 'rb')
    package = pickle.load(f)
    result = package.task(*package.args, **package.kwargs)
    return result

##__________________________________________________________________||
def compose_result_path(package_path):

    # e.g., package_path = 'c/d/task_00003.p.gz'

    taskdir = os.path.dirname(os.path.abspath(package_path))
    # e.g., '/a/b/c/d'

    result_topdir = os.path.join(taskdir, 'results')
    # e.g., '/a/b/c/d/results'

    package_basename =  os.path.basename(package_path)
    # e.g., 'task_00003.p.gz'

    resultdir_basename = os.path.splitext(package_basename)[0]
    resultdir_basename = os.path.splitext(resultdir_basename)[0]
    # e.g., 'task_00003'

    resultdir = os.path.join(result_topdir, resultdir_basename)
    # e.g., '/a/b/c/d/results/task_00003'

    result_path = os.path.join(resultdir, 'result.p.gz')
    # e.g., '/a/b/c/d/results/task_00003/result.p.gz'

    return result_path

##__________________________________________________________________||
def store_result(result, result_path):
    mkdir_p(os.path.dirname(result_path))
    f = gzip.open(result_path, 'wb')
    pickle.dump(result, f, protocol = pickle.HIGHEST_PROTOCOL)

##__________________________________________________________________||
def mkdir_p(path):
    # http://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
    try:
        os.makedirs(path)
    except OSError as exc: # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else: raise

##__________________________________________________________________||
if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.12232194557035321, 'fraction_numerical': 0.016068326577880716, 'mean_word_length': 3.674560216508796, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5806197', 'n_tokens_mistral': 2245, 'n_tokens_neox': 2124, 'n_words': 387}