<commit_before>import os
import subprocess
from multiprocessing import Process

import util as u

def _internal_download_trim_clean():
    p = ''
    devnull = open(os.devnull, 'w')
    try:
        p = 'podget'
        subprocess.check_call([p], stderr=devnull, stdout=devnull)

        p = path.join(os.getcwd(), 'trim.py')
        p = path.normpath(p)
        subprocess.check_call([p], stdout=devnull)

    except subprocess.CalledProcessError, ex:
        printWarning('Failed to run %s (%s)' % (p, ex))
    except OSError, ex:
        printWarning('Cannot find application %s (%s)' % (p, ex))
    devnull.close()

def download_trim_clean():
    ###
    #printStep('Start download, trim, and cleanup in new thread')

    p = Process(target=_internal_download_trim_clean, args=())
    p.start()
    return p



def wait_for_download(p):
    # TODO: consider turning output back on (how?)
    print
    print
    print 'Waiting for download to complete...'
    print
    p.join()


def main():
    u.ensure_folders()
    dl.wait_for_download(dl.download_trim_clean())




def _test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    main()
<commit_msg>Add missing includes. Allow output in download.py<commit_after>#! /usr/bin/env python

import sys
import os
import os.path as path
import subprocess
from multiprocessing import Process

import util as u
import config as cfg

def _internal_download_trim_clean(silence=True):
    p = ''
    if silence:
        devnull = open(os.devnull, 'w')
    else:
        devnull = sys.stdout

    try:
        p = 'podget'
        subprocess.check_call([p], stderr=devnull, stdout=devnull)

        p = path.join(os.getcwd(), 'trim.py')
        p = path.normpath(p)
        subprocess.check_call([p], stdout=devnull)

    except subprocess.CalledProcessError, ex:
        printWarning('Failed to run %s (%s)' % (p, ex))
    except OSError, ex:
        printWarning('Cannot find application %s (%s)' % (p, ex))

    # don't close stdio
    if not silence:
        devnull.close()

def download_trim_clean():
    ###
    #printStep('Start download, trim, and cleanup in new thread')

    p = Process(target=_internal_download_trim_clean, args=())
    p.start()
    return p



def wait_for_download(p):
    # TODO: consider turning output back on (how?)
    print
    print
    print 'Waiting for download to complete...'
    print
    p.join()


def main():
    u.printStep('Initializing')
    u.ensure_folders()
    u.printStep('Beginning download')
    _internal_download_trim_clean(False)
    u.printStep('Done')




def _test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.12052238805970149, 'fraction_numerical': 0.0, 'mean_word_length': 2.9895833333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7264708', 'n_tokens_mistral': 892, 'n_tokens_neox': 870, 'n_words': 234}