<commit_before>"""Raw data collector for Coverage."""

import os, sys, threading

try:
    # Use the C extension code when we can, for speed.
    from coverage.tracer import CTracer         # pylint: disable=F0401,E0611
except ImportError:
    # Couldn't import the C extension, maybe it isn't built.
    if os.getenv('COVERAGE_TEST_TRACER') == 'c':
        # During testing, we use the COVERAGE_TEST_TRACER env var to indicate
        # that we've fiddled with the environment to test this fallback code.
        # If we thought we had a C tracer, but couldn't import it, then exit
        # quickly and clearly instead of dribbling confusing errors. I'm using
        # sys.exit here instead of an exception because an exception here
        # causes all sorts of other noise in unittest.
        sys.stderr.write(
            "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n"
            )
        sys.exit(1)
    CTracer = None


class PyTracer(object):
    """Python implementation of the raw data tracer."""

    # Because of poor implementations of trace-function-manipulating tools,
    # the Python trace function must be kept very simple.  In particular, there
    # must be only one function ever set as the trace function, both through
    # sys.settrace, and as the return value from the trace function.  Put
    # another way, the trace function must always return itself.  It cannot
    # swap in other functions, or return None to avoid tracing a particular
    # frame.
    #
    # The trace manipulator that introduced this restriction is DecoratorTools,
    # which sets a trace function, and then later restores the pre-existing one
    # by calling sys.settrace with a function it found in the current frame.
    #
    # Systems that use DecoratorTools (or similar trace manipulations) must use
    # PyTracer to get accurate results.  The command-line --timid argument is
    # used to force the use of this tracer.

    def __init__(self):
        self.data = None
        self.should_trace = None
        self.should_trace_cache = None
        self.warn = None
        self.cur_file_data = None
        self.last_line = 0
        self.data_stack = []
        self.last_exc_back = None
        self.last_exc_firstlineno = 0
        self.arcs = False

    def _trace(self, frame, event, arg_unused):
        """The trace function passed to sys.settrace."""

        #print("trace event: %s %r @%d" % (
        #           event, frame.f_code.co_filename, frame.f_lineno),
        #      file=sys.stderr)

        if self.last_exc_back:
            if frame == self.last_exc_back:
                # Someone forgot a return event.
                if self.arcs and self.cur_file_data:
                    pair = (self.last_line, -self.last_exc_firstlineno)
                    self.cur_file_data[pair] = None
                self.cur_file_data, self.last_line = self.data_stack.pop()
            self.last_exc_back = None

        if event == 'call':
            # Entering a new function context.  Decide if we should trace
            # in this file.
            self.data_stack.append((self.cur_file_data, self.last_line))
            filename = frame.f_code.co_filename
            tracename = self.should_trace_cache.get(filename)
            if tracename is None:
                tracename = self.should_trace(filename, frame)
                self.should_trace_cache[filename] = tracename
            #print("called, stack is %d deep, tracename is %r" % (<commit_msg>A more convenient way to check on the trace function.<commit_after>"""Raw data collector for Coverage."""

import os, sys, threading

try:
    # Use the C extension code when we can, for speed.
    from coverage.tracer import CTracer         # pylint: disable=F0401,E0611
except ImportError:
    # Couldn't import the C extension, maybe it isn't built.
    if os.getenv('COVERAGE_TEST_TRACER') == 'c':
        # During testing, we use the COVERAGE_TEST_TRACER env var to indicate
        # that we've fiddled with the environment to test this fallback code.
        # If we thought we had a C tracer, but couldn't import it, then exit
        # quickly and clearly instead of dribbling confusing errors. I'm using
        # sys.exit here instead of an exception because an exception here
        # causes all sorts of other noise in unittest.
        sys.stderr.write(
            "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n"
            )
        sys.exit(1)
    CTracer = None


class PyTracer(object):
    """Python implementation of the raw data tracer."""

    # Because of poor implementations of trace-function-manipulating tools,
    # the Python trace function must be kept very simple.  In particular, there
    # must be only one function ever set as the trace function, both through
    # sys.settrace, and as the return value from the trace function.  Put
    # another way, the trace function must always return itself.  It cannot
    # swap in other functions, or return None to avoid tracing a particular
    # frame.
    #
    # The trace manipulator that introduced this restriction is DecoratorTools,
    # which sets a trace function, and then later restores the pre-existing one
    # by calling sys.settrace with a function it found in the current frame.
    #
    # Systems that use DecoratorTools (or similar trace manipulations) must use
    # PyTracer to get accurate results.  The command-line --timid argument is
    # used to force the use of this tracer.

    def __init__(self):
        self.data = None
        self.should_trace = None
        self.should_trace_cache = None
        self.warn = None
        self.cur_file_data = None
        self.last_line = 0
        self.data_stack = []
        self.last_exc_back = None
        self.last_exc_firstlineno = 0
        self.arcs = False

    def _trace(self, frame, event, arg_unused):
        """The trace function passed to sys.settrace."""

        if 0:
            sys.stderr.write("trace event: %s %r @%d\n" % (
                event, frame.f_code.co_filename, frame.f_lineno
            ))

        if self.last_exc_back:
            if frame == self.last_exc_back:
                # Someone forgot a return event.
                if self.arcs and self.cur_file_data:
                    pair = (self.last_line, -self.last_exc_firstlineno)
                    self.cur_file_data[pair] = None
                self.cur_file_data, self.last_line = self.data_stack.pop()
            self.last_exc_back = None

        if event == 'call':
            # Entering a new function context.  Decide if we should trace
            # in this file.
            self.data_stack.append((self.cur_file_data, self.last_line))
            filename = frame.f_code.co_filename
            tracename = self.should_trace_cache.get(filename)
            if tracename is None:
                tracename = self.should_trace(filename, frame)
                self.should_trace_cache[filename] = tracename
            #print("called, stack is %d deep, tracename is %r" % (
{'fraction_non_alphanumeric': 0.0738483345145287, 'fraction_numerical': 0.003260099220411056, 'mean_word_length': 2.303370786516854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7447552', 'n_tokens_mistral': 2014, 'n_tokens_neox': 1938, 'n_words': 722}