<commit_before>#!/usr/bin/env python
import os, sys
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
import doctest

pyversion = sys.version[:3]
lib_py = 'lib/python%s/' % pyversion
here = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.join(here, 'test-scratch')
download_cache = os.path.join(here, 'test-cache')
if not os.path.exists(download_cache):
    os.makedirs(download_cache)

from scripttest import TestFileEnvironment

if 'PYTHONPATH' in os.environ:
    del os.environ['PYTHONPATH']

try:
    any
except NameError:
    def any(seq):
        for item in seq:
            if item:
                return True
        return False

def clear_environ(environ):
    return dict(((k, v) for k, v in environ.iteritems()
                if not k.lower().startswith('pip_')))

env = None
def reset_env(environ=None):
    global env
    if not environ:
        environ = os.environ.copy()
        environ = clear_environ(environ)
        environ['PIP_DOWNLOAD_CACHE'] = download_cache
    environ['PIP_NO_INPUT'] = '1'
    environ['PYTHONPATH'] = os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir))
    env = TestFileEnvironment(base_path, ignore_hidden=False, environ=environ)
    env.run(sys.executable, '-m', 'virtualenv', '--no-site-packages', env.base_path)
    # make sure we have current setuptools to avoid svn incompatibilities
    env.run('%s/bin/easy_install' % env.base_path, 'setuptools==0.6c11')
    # Uninstall (kind of) pip, so PYTHONPATH can take effect:
    env.run('%s/bin/easy_install' % env.base_path, '-m', 'pip')
    env.run('mkdir', 'src')

def run_pip(*args, **kw):
    args = (sys.executable, '-c', 'import pip; pip.main()', '-E', env.base_path) + args
    #print >> sys.__stdout__, 'running', ' '.join(args)
    if getattr(options, 'show_error', False):
        kw['expect_error'] = True
    result = env.run(*args, **kw)
    if getattr(options, 'show_error', False) and result.returncode:
        print result
    return result

def write_file(filename, text):
    f = open(os.path.join(base_path, filename), 'w')
    f.write(text)
    f.close()

def get_env():
    return env

# FIXME ScriptTest does something similar, but only within a single
# ProcResult; this generalizes it so states can be compared across
# multiple commands.  Maybe should be rolled into ScriptTest?<commit_msg>complete removal of doctest harness<commit_after>#!/usr/bin/env python
import os, sys
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
import doctest

pyversion = sys.version[:3]
lib_py = 'lib/python%s/' % pyversion
here = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.join(here, 'test-scratch')
download_cache = os.path.join(here, 'test-cache')
if not os.path.exists(download_cache):
    os.makedirs(download_cache)

from scripttest import TestFileEnvironment

if 'PYTHONPATH' in os.environ:
    del os.environ['PYTHONPATH']

try:
    any
except NameError:
    def any(seq):
        for item in seq:
            if item:
                return True
        return False

def clear_environ(environ):
    return dict(((k, v) for k, v in environ.iteritems()
                if not k.lower().startswith('pip_')))

env = None
def reset_env(environ=None):
    global env
    if not environ:
        environ = os.environ.copy()
        environ = clear_environ(environ)
        environ['PIP_DOWNLOAD_CACHE'] = download_cache
    environ['PIP_NO_INPUT'] = '1'
    environ['PYTHONPATH'] = os.path.abspath(os.path.join(__file__, os.path.pardir, os.path.pardir))
    env = TestFileEnvironment(base_path, ignore_hidden=False, environ=environ)
    env.run(sys.executable, '-m', 'virtualenv', '--no-site-packages', env.base_path)
    # make sure we have current setuptools to avoid svn incompatibilities
    env.run('%s/bin/easy_install' % env.base_path, 'setuptools==0.6c11')
    # Uninstall (kind of) pip, so PYTHONPATH can take effect:
    env.run('%s/bin/easy_install' % env.base_path, '-m', 'pip')
    env.run('mkdir', 'src')

def run_pip(*args, **kw):
    args = (sys.executable, '-c', 'import pip; pip.main()', '-E', env.base_path) + args
    #print >> sys.__stdout__, 'running', ' '.join(args)
    result = env.run(*args, **kw)
    return result

def write_file(filename, text):
    f = open(os.path.join(base_path, filename), 'w')
    f.write(text)
    f.close()

def get_env():
    return env

# FIXME ScriptTest does something similar, but only within a single
# ProcResult; this generalizes it so states can be compared across
# multiple commands.  Maybe should be rolled into ScriptTest?
{'fraction_non_alphanumeric': 0.13839772975332898, 'fraction_numerical': 0.0030561012879283997, 'mean_word_length': 3.623612512613522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3553761', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1565, 'n_words': 387}