<commit_before>from lettuce import *
import os
import re
import shutil
import tempfile
import itertools
from glob import glob
from subprocess import Popen, PIPE, call
import lettuce

VENV_DIR = os.getenv('KEEPENV', None) or tempfile.mkdtemp(prefix='pyg_env_')

ENVIRONMENTS = { 'standard': os.path.join(VENV_DIR, 'standard') }

@before.each_feature
def remove_std_packages(*a):
    for env, path in ENVIRONMENTS.iteritems():
        try:
            call([os.path.join(path, 'bin', 'pyg'),
                'remove',
                '-y',
                'bottle',
                'mercurial',
                'lk',
                'grin',
                'hg-git',
                'gevent'])
        except OSError:
            pass

@before.all
def init_env(*a):
    """ Ensure VENV_DIR folder exists and is empty (unless KEEPENV is used) """

    if 'KEEPENV' in os.environ and os.path.exists(VENV_DIR):
        return

    try:
        shutil.rmtree(VENV_DIR)
    except OSError:
        pass
    os.mkdir(VENV_DIR)

@after.all
def destroy_env(*a):
    """ Ensure VENV_DIR folder is destroyed, set KEEPENV=/some/folder to disable. """

    if 'KEEPENV' in os.environ:
        print "Env. path: %s" % VENV_DIR
    else:
        shutil.rmtree(VENV_DIR)

@step('I use "(.*)" environment')
def given_i_use_venv(step, env_name):
    world.env_path = os.path.join(VENV_DIR, env_name)

    # build python version from environment name
    py_version = re.compile('.*?(\d+\.?\d*)$').match(env_name)

    if py_version:
        py_version = py_version.groups()[0]

    # Create Venv if it can't be found
    if not os.path.exists(world.env_path):
        ENVIRONMENTS[env_name] = world.env_path
        if py_version:
            args = ['--python', 'python'+py_version]
        else:
            args = []

        call(['virtualenv'] + args + ['--no-site-packages', world.env_path])

    # Install pyg if not found
    if not os.path.exists(os.path.join(world.env_path, 'bin', 'pyg')):
        dn = os.path.dirname
        pyg_dir = dn(dn(dn(__file__))) + '/'
        os.chdir(pyg_dir)

        call('. %s ; python %s install' % (
         os.path.join(world.env_path, 'bin', 'activate'),
         os.path.join(pyg_dir, 'setup.py'),
        ), shell=True)

    os.chdir(world.env_path)

def wait_and_set():
    if world.proc:
        world.stdout, world.stderr = world.proc.communicate()
        world.returncode = world.proc.returncode
        world.proc = None
    return (world.returncode, world.stdout, world.stderr)

@step('I execute +(.*)')
def i_execute(step, cmd):
    prefixed_cmd = os.path.join(world.env_path, 'bin', cmd)
    world.proc = Popen(prefixed_cmd,
     shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)

@step('no trace of \s*(\w+).*')
def no_trace_of(step, what):
    matches = glob(os.path.join(world.env_path, 'lib', 'python*', 'site-packages', what))
    if 0 != len(matches):<commit_msg>tests: "setup.py develop" instead of install (useful with KEEPENV)<commit_after>from lettuce import *
import os
import re
import shutil
import tempfile
import itertools
from glob import glob
from subprocess import Popen, PIPE, call
import lettuce

VENV_DIR = os.getenv('KEEPENV', None) or tempfile.mkdtemp(prefix='pyg_env_')

ENVIRONMENTS = { 'standard': os.path.join(VENV_DIR, 'standard') }

@before.each_feature
def remove_std_packages(*a):
    for env, path in ENVIRONMENTS.iteritems():
        try:
            call([os.path.join(path, 'bin', 'pyg'),
                'remove',
                '-y',
                'bottle',
                'mercurial',
                'lk',
                'grin',
                'hg-git',
                'gevent'])
        except OSError:
            pass

@before.all
def init_env(*a):
    """ Ensure VENV_DIR folder exists and is empty (unless KEEPENV is used) """

    if 'KEEPENV' in os.environ and os.path.exists(VENV_DIR):
        return

    try:
        shutil.rmtree(VENV_DIR)
    except OSError:
        pass
    os.mkdir(VENV_DIR)

@after.all
def destroy_env(*a):
    """ Ensure VENV_DIR folder is destroyed, set KEEPENV=/some/folder to disable. """

    if 'KEEPENV' in os.environ:
        print "Env. path: %s" % VENV_DIR
    else:
        shutil.rmtree(VENV_DIR)

@step('I use "(.*)" environment')
def given_i_use_venv(step, env_name):
    world.env_path = os.path.join(VENV_DIR, env_name)

    # build python version from environment name
    py_version = re.compile('.*?(\d+\.?\d*)$').match(env_name)

    if py_version:
        py_version = py_version.groups()[0]

    # Create Venv if it can't be found
    if not os.path.exists(world.env_path):
        ENVIRONMENTS[env_name] = world.env_path
        if py_version:
            args = ['--python', 'python'+py_version]
        else:
            args = []

        call(['virtualenv'] + args + ['--no-site-packages', world.env_path])

    # Install pyg if not found
    if not os.path.exists(os.path.join(world.env_path, 'bin', 'pyg')):
        dn = os.path.dirname
        pyg_dir = dn(dn(dn(__file__))) + '/'
        os.chdir(pyg_dir)

        call('. %s ; python %s develop' % (
         os.path.join(world.env_path, 'bin', 'activate'),
         os.path.join(pyg_dir, 'setup.py'),
        ), shell=True)

    os.chdir(world.env_path)

def wait_and_set():
    if world.proc:
        world.stdout, world.stderr = world.proc.communicate()
        world.returncode = world.proc.returncode
        world.proc = None
    return (world.returncode, world.stdout, world.stderr)

@step('I execute +(.*)')
def i_execute(step, cmd):
    prefixed_cmd = os.path.join(world.env_path, 'bin', cmd)
    world.proc = Popen(prefixed_cmd,
     shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)

@step('no trace of \s*(\w+).*')
def no_trace_of(step, what):
    matches = glob(os.path.join(world.env_path, 'lib', 'python*', 'site-packages', what))
    if 0 != len(matches):
{'fraction_non_alphanumeric': 0.14404903797037288, 'fraction_numerical': 0.0006810829218457347, 'mean_word_length': 2.542822677925211, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516462', 'n_tokens_mistral': 2086, 'n_tokens_neox': 2017, 'n_words': 455}