<commit_before>import sublime
import sublime_plugin
import sys
import os
import logging
from unittest import TextTestRunner, TestSuite
from .core import TestLoader, DeferringTextTestRunner, DeferrableTestCase
from .mixin import UnitTestingMixin
from .const import DONE_MESSAGE
from .utils import ProgressBar
import threading

version = sublime.version()


class UnitTestingCommand(sublime_plugin.ApplicationCommand, UnitTestingMixin):

    def run(self, package=None, **kargs):
        if not package:
            self.prompt_package(lambda x: self.run(x, **kargs))
            return

        package, pattern = self.input_parser(package)
        settings = self.load_unittesting_settings(package, pattern=pattern, **kargs)
        stream = self.load_stream(package, settings)

        if settings["async"]:
            threading.Thread(target=lambda: self.unit_testing(stream, package, settings)).start()
        else:
            self.unit_testing(stream, package, settings)

    def verify_testsuite(self, tests):
        for t in tests:
            if isinstance(t, TestSuite):
                self.verify_testsuite(t)
            if isinstance(t, DeferrableTestCase):
                raise Exception("DeferrableTestCase is used but `deferred` is `false`.")

    def unit_testing(self, stream, package, settings, cleanup_hooks=[]):
        if settings["capture_console"]:
            stdout = sys.stdout
            stderr = sys.stderr
            handler = logging.StreamHandler(stream)
            logging.root.addHandler(handler)

            class StdOutForward(object):
                def write(self, data):
                    stdout.write(data)
                    stream.write(data)
                def flush(self):
                    pass

            class StdErrForward(object):
                def write(self, data):
                    stderr.write(data)
                    stream.write(data)
                def flush(self):
                    pass

            sys.stdout = StdOutForward()
            sys.stderr = StdErrForward()
        testRunner = None
        progress_bar = ProgressBar("Testing %s" % package)
        progress_bar.start()

        try:
            # use custom loader which supports reloading modules
            self.remove_test_modules(package, settings["tests_dir"])
            loader = TestLoader(settings["deferred"])
            tests = loader.discover(os.path.join(
                sublime.packages_path(), package, settings["tests_dir"]), settings["pattern"]
            )
            # use deferred test runner or default test runner
            if settings["deferred"]:
                testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"])
            else:
                self.verify_testsuite(tests)
                testRunner = TextTestRunner(stream, verbosity=settings["verbosity"])

            testRunner.run(tests)

        except Exception as e:
            if not stream.closed:
                stream.write("ERROR: %s\n" % e)
            # force clean up
            testRunner = None
        finally:
            def cleanup(status=0):
                if not settings["deferred"] or not testRunner or \<commit_msg>Also forwarded the flush call their respective streams.<commit_after>import sublime
import sublime_plugin
import sys
import os
import logging
from unittest import TextTestRunner, TestSuite
from .core import TestLoader, DeferringTextTestRunner, DeferrableTestCase
from .mixin import UnitTestingMixin
from .const import DONE_MESSAGE
from .utils import ProgressBar
import threading

version = sublime.version()


class UnitTestingCommand(sublime_plugin.ApplicationCommand, UnitTestingMixin):

    def run(self, package=None, **kargs):
        if not package:
            self.prompt_package(lambda x: self.run(x, **kargs))
            return

        package, pattern = self.input_parser(package)
        settings = self.load_unittesting_settings(package, pattern=pattern, **kargs)
        stream = self.load_stream(package, settings)

        if settings["async"]:
            threading.Thread(target=lambda: self.unit_testing(stream, package, settings)).start()
        else:
            self.unit_testing(stream, package, settings)

    def verify_testsuite(self, tests):
        for t in tests:
            if isinstance(t, TestSuite):
                self.verify_testsuite(t)
            if isinstance(t, DeferrableTestCase):
                raise Exception("DeferrableTestCase is used but `deferred` is `false`.")

    def unit_testing(self, stream, package, settings, cleanup_hooks=[]):
        if settings["capture_console"]:
            stdout = sys.stdout
            stderr = sys.stderr
            handler = logging.StreamHandler(stream)
            logging.root.addHandler(handler)

            class StdOutForward(object):
                def write(self, data):
                    stdout.write(data)
                    stream.write(data)
                def flush(self):
                    stderr.flush()
                    stream.flush()

            class StdErrForward(object):
                def write(self, data):
                    stderr.write(data)
                    stream.write(data)
                def flush(self):
                    stderr.flush()
                    stream.flush()

            sys.stdout = StdOutForward()
            sys.stderr = StdErrForward()
        testRunner = None
        progress_bar = ProgressBar("Testing %s" % package)
        progress_bar.start()

        try:
            # use custom loader which supports reloading modules
            self.remove_test_modules(package, settings["tests_dir"])
            loader = TestLoader(settings["deferred"])
            tests = loader.discover(os.path.join(
                sublime.packages_path(), package, settings["tests_dir"]), settings["pattern"]
            )
            # use deferred test runner or default test runner
            if settings["deferred"]:
                testRunner = DeferringTextTestRunner(stream, verbosity=settings["verbosity"])
            else:
                self.verify_testsuite(tests)
                testRunner = TextTestRunner(stream, verbosity=settings["verbosity"])

            testRunner.run(tests)

        except Exception as e:
            if not stream.closed:
                stream.write("ERROR: %s\n" % e)
            # force clean up
            testRunner = None
        finally:
            def cleanup(status=0):
                if not settings["deferred"] or not testRunner or \
{'fraction_non_alphanumeric': 0.08689676237018937, 'fraction_numerical': 0.00030543677458766036, 'mean_word_length': 2.073205068043172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5420944', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1600, 'n_words': 413}