<commit_before># Load in core dependencies
import os
import shutil
import subprocess
import time
import tempfile
import unittest

# Load in 3rd party dependencies
from jinja2 import Template

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))
SUBLIME_TEXT_VERSION = os.environ.get('SUBLIME_TEXT_VERSION', None)


# Set up helper fn
def template(tmpl_path):
    """ Decorator that templates the returned content. """
    # Pre-emptively read in the template
    tmpl = None
    with open(tmpl_path) as f:
        tmpl = Template(f.read())

    # Define our templating wrapper fn
    def decorator_fn(fn):
        def templator_fn(*args, **kwargs):
            # Run the normal function
            data = fn(*args, **kwargs)

            # Render the info
            return tmpl.render(**data)
        return templator_fn
    return decorator_fn


class Base(object):
    # Determine the plugins directory
    # TODO: Programmatically sniff for this (https://github.com/twolfson/sublime-plugin-tests/issues/4)
    _plugin_test_dir = os.path.expanduser('~/.config/sublime-text-2/Packages/sublime-plugin-tests-tmp')
    if SUBLIME_TEXT_VERSION == '3.0':
        _plugin_test_dir = os.path.expanduser('~/.config/sublime-text-3/Packages/sublime-plugin-tests-tmp')

    @classmethod
    def _ensure_plugin_test_dir(cls):
        # If the plugin test directory does not exist, create it
        if not os.path.exists(cls._plugin_test_dir):
            os.makedirs(cls._plugin_test_dir)

    @classmethod
    def _ensure_utils(cls):
        # Ensure the plugin test directory exists
        cls._ensure_plugin_test_dir()

        # TODO: Use similar copy model minus the exception
        # TODO: If we overwrite utils, be sure to wait so that changes for import get picked up
        if not os.path.exists(cls._plugin_test_dir + '/utils'):
            shutil.copytree(__dir__ + '/utils', cls._plugin_test_dir + '/utils')

    @classmethod
    def _install_command_launcher(cls):
        # Guarantee the plugin test dir exists
        cls._ensure_plugin_test_dir()

        # If command launcher doesn't exist, copy it
        orig_command_path = __dir__ + '/launchers/command.py'
        dest_command_path = cls._plugin_test_dir + '/command_launcher.py'
        if not os.path.exists(dest_command_path):
            shutil.copyfile(orig_command_path, dest_command_path)
        else:
        # Otherwise...
            # If there are updates for command launcher
            expected_command = None
            with open(orig_command_path) as f:
                expected_command = f.read()
            actual_command = None
            with open(dest_command_path) as f:
                actual_command = f.read()
            if expected_command != actual_command:
                # Update the file
                shutil.copyfile(orig_command_path, dest_command_path)

                # and notify the user we must restart Sublime
                raise Exception('We had to update the test launcher plugin. You must close or restart Sublime to continue testing.')

    _init_launcher_path = _plugin_test_dir + '/init_launcher.py'

    @classmethod
    def _remove_init_launcher(cls):
        # If the init launcher exists, delete it
        if os.path.exists(cls._init_launcher_path):
            os.unlink(cls._init_launcher_path)

    @classmethod
    def _install_init_launcher(cls):
        # Guarantee the plugin test dir exists
        cls._ensure_plugin_test_dir()

        # Clean up any past instances of init launcher
        cls._remove_init_launcher()

        # Install a new one
        # TODO: Verify this doesn't have any double invocation consequences
        orig_command_path = __dir__ + '/launchers/init.py'
        shutil.copyfile(orig_command_path, cls._init_launcher_path)

    @classmethod
    def _run_test(cls, test_str, auto_kill_sublime=False):
        # Guarantee there is an output directory and launcher
        cls._ensure_utils()

        # Reserve an output file
        output_file = tempfile.mkstemp()[1]

        # Template plugin
        plugin_runner = None
        f = open(__dir__ + '/templates/plugin_runner.py')
        runner_template = Template(f.read())
        plugin_runner = runner_template.render(output_file=output_file,
                                               auto_kill_sublime=auto_kill_sublime)
        f.close()

        # Output plugin_runner to directory
        f = open(cls._plugin_test_dir + '/plugin_runner.py', 'w')
        f.write(plugin_runner)
        f.close()

        # Output test to directory
        f = open(cls._plugin_test_dir + '/plugin.py', 'w')
        f.write(test_str)
        f.close()

        # TODO: These commands should go in a launching harness
        # If we are running Sublime Text 3 and it has not yet started, use `init`
        running_via_init = False
        if SUBLIME_TEXT_VERSION == '3.0':
            # TODO: Use tasklist for Windows
            # Get process list
            child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
            ps_list = child.stdout.read()

            # Kill the child
            child.kill()

            # Determine if Sublime Text is running
            # TODO: This could be subl, sublime_text, or other
            sublime_is_running = False
            for process in ps_list.split('\n'):
                # TODO: <defunct> is a bit of a hack and not sure how Windows will react. These are processes that are in the process of terminating
                if 'sublime_text' in process and '<defunct>' not in process:
                    sublime_is_running = True
                    break

            # If sublime isn't running, use our init trigger
            print(ps_list, sublime_is_running)
            if not sublime_is_running:
                # Install the init trigger
                cls._install_init_launcher()

                # and launch sublime_text
                subprocess.call(['sublime_text'])

                # Mark the init to prevent double launch
                running_via_init = True

        # Otherwise, use `--command` trigger
        # TODO: Can we consolidate these? `init` might work in *all* cases and allow us to move around the plugin locking in as with `--command`
        if not running_via_init:
            # Install the command launcher
            cls._install_command_launcher()

            # Start a subprocess to run the plugin
            # TODO: We might want a development mode (runs commands inside local sublime window) and a testing mode (calls out to Vagrant box)
            # TODO: or at least 2 plugin hooks, one for CLI based testing and one for internal dev
            subprocess.call(['sublime_text', '--command', 'sublime_plugin_test_tmp'])

        # Wait for the output file to exist
        print('waiting', output_file)

        child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
        ps_list = child.stdout.read()
        child.kill()
        print(ps_list)

        # Wait a second
        time.sleep(1)


        child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
        ps_list = child.stdout.read()
        child.kill()
        print(ps_list)

        child = subprocess.Popen(["ls", "/tmp/hi"], stdout=subprocess.PIPE)
        tmp_list = child.stdout.read()
        child.kill()
        print(tmp_list)

        while (not os.path.exists(output_file) or os.stat(output_file).st_size == 0):<commit_msg>Outputting environment<commit_after># Load in core dependencies
import os
import shutil
import subprocess
import time
import tempfile
import unittest

# Load in 3rd party dependencies
from jinja2 import Template

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))
SUBLIME_TEXT_VERSION = os.environ.get('SUBLIME_TEXT_VERSION', None)


# Set up helper fn
def template(tmpl_path):
    """ Decorator that templates the returned content. """
    # Pre-emptively read in the template
    tmpl = None
    with open(tmpl_path) as f:
        tmpl = Template(f.read())

    # Define our templating wrapper fn
    def decorator_fn(fn):
        def templator_fn(*args, **kwargs):
            # Run the normal function
            data = fn(*args, **kwargs)

            # Render the info
            return tmpl.render(**data)
        return templator_fn
    return decorator_fn


class Base(object):
    # Determine the plugins directory
    # TODO: Programmatically sniff for this (https://github.com/twolfson/sublime-plugin-tests/issues/4)
    _plugin_test_dir = os.path.expanduser('~/.config/sublime-text-2/Packages/sublime-plugin-tests-tmp')
    if SUBLIME_TEXT_VERSION == '3.0':
        _plugin_test_dir = os.path.expanduser('~/.config/sublime-text-3/Packages/sublime-plugin-tests-tmp')

    @classmethod
    def _ensure_plugin_test_dir(cls):
        # If the plugin test directory does not exist, create it
        if not os.path.exists(cls._plugin_test_dir):
            os.makedirs(cls._plugin_test_dir)

    @classmethod
    def _ensure_utils(cls):
        # Ensure the plugin test directory exists
        cls._ensure_plugin_test_dir()

        # TODO: Use similar copy model minus the exception
        # TODO: If we overwrite utils, be sure to wait so that changes for import get picked up
        if not os.path.exists(cls._plugin_test_dir + '/utils'):
            shutil.copytree(__dir__ + '/utils', cls._plugin_test_dir + '/utils')

    @classmethod
    def _install_command_launcher(cls):
        # Guarantee the plugin test dir exists
        cls._ensure_plugin_test_dir()

        # If command launcher doesn't exist, copy it
        orig_command_path = __dir__ + '/launchers/command.py'
        dest_command_path = cls._plugin_test_dir + '/command_launcher.py'
        if not os.path.exists(dest_command_path):
            shutil.copyfile(orig_command_path, dest_command_path)
        else:
        # Otherwise...
            # If there are updates for command launcher
            expected_command = None
            with open(orig_command_path) as f:
                expected_command = f.read()
            actual_command = None
            with open(dest_command_path) as f:
                actual_command = f.read()
            if expected_command != actual_command:
                # Update the file
                shutil.copyfile(orig_command_path, dest_command_path)

                # and notify the user we must restart Sublime
                raise Exception('We had to update the test launcher plugin. You must close or restart Sublime to continue testing.')

    _init_launcher_path = _plugin_test_dir + '/init_launcher.py'

    @classmethod
    def _remove_init_launcher(cls):
        # If the init launcher exists, delete it
        if os.path.exists(cls._init_launcher_path):
            os.unlink(cls._init_launcher_path)

    @classmethod
    def _install_init_launcher(cls):
        # Guarantee the plugin test dir exists
        cls._ensure_plugin_test_dir()

        # Clean up any past instances of init launcher
        cls._remove_init_launcher()

        # Install a new one
        # TODO: Verify this doesn't have any double invocation consequences
        orig_command_path = __dir__ + '/launchers/init.py'
        shutil.copyfile(orig_command_path, cls._init_launcher_path)

    @classmethod
    def _run_test(cls, test_str, auto_kill_sublime=False):
        # Guarantee there is an output directory and launcher
        cls._ensure_utils()

        # Reserve an output file
        output_file = tempfile.mkstemp()[1]

        # Template plugin
        plugin_runner = None
        f = open(__dir__ + '/templates/plugin_runner.py')
        runner_template = Template(f.read())
        plugin_runner = runner_template.render(output_file=output_file,
                                               auto_kill_sublime=auto_kill_sublime)
        f.close()

        # Output plugin_runner to directory
        f = open(cls._plugin_test_dir + '/plugin_runner.py', 'w')
        f.write(plugin_runner)
        f.close()

        # Output test to directory
        f = open(cls._plugin_test_dir + '/plugin.py', 'w')
        f.write(test_str)
        f.close()

        # TODO: These commands should go in a launching harness
        # If we are running Sublime Text 3 and it has not yet started, use `init`
        running_via_init = False
        if SUBLIME_TEXT_VERSION == '3.0':
            # TODO: Use tasklist for Windows
            # Get process list
            child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
            ps_list = child.stdout.read()

            # Kill the child
            child.kill()

            # Determine if Sublime Text is running
            # TODO: This could be subl, sublime_text, or other
            sublime_is_running = False
            for process in ps_list.split('\n'):
                # TODO: <defunct> is a bit of a hack and not sure how Windows will react. These are processes that are in the process of terminating
                if 'sublime_text' in process and '<defunct>' not in process:
                    sublime_is_running = True
                    break

            # If sublime isn't running, use our init trigger
            print(ps_list, sublime_is_running)
            if not sublime_is_running:
                # Install the init trigger
                cls._install_init_launcher()

                # and launch sublime_text
                subprocess.call(['sublime_text'])

                # Mark the init to prevent double launch
                running_via_init = True

        # Otherwise, use `--command` trigger
        # TODO: Can we consolidate these? `init` might work in *all* cases and allow us to move around the plugin locking in as with `--command`
        if not running_via_init:
            # Install the command launcher
            cls._install_command_launcher()

            # Start a subprocess to run the plugin
            # TODO: We might want a development mode (runs commands inside local sublime window) and a testing mode (calls out to Vagrant box)
            # TODO: or at least 2 plugin hooks, one for CLI based testing and one for internal dev
            subprocess.call(['sublime_text', '--command', 'sublime_plugin_test_tmp'])

        # Wait for the output file to exist
        print('waiting', output_file)

        import os
        print(os.environ)

        child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
        ps_list = child.stdout.read()
        child.kill()
        print(ps_list)

        # Wait a second
        time.sleep(1)


        child = subprocess.Popen(["ps", "ax"], stdout=subprocess.PIPE)
        ps_list = child.stdout.read()
        child.kill()
        print(ps_list)

        child = subprocess.Popen(["ls", "/tmp/hi"], stdout=subprocess.PIPE)
        tmp_list = child.stdout.read()
        child.kill()
        print(tmp_list)

        while (not os.path.exists(output_file) or os.stat(output_file).st_size == 0):
{'fraction_non_alphanumeric': 0.08245567257698974, 'fraction_numerical': 0.0018664178109585388, 'mean_word_length': 2.2779112956084773, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '908694', 'n_tokens_mistral': 4364, 'n_tokens_neox': 4205, 'n_words': 1243}