<commit_before>"""The Django coverage plugin."""

from __future__ import print_function, unicode_literals

import os.path
from six.moves import range

import coverage.plugin

import django
from django.template import Lexer, TextNode
from django.template.base import TOKEN_MAPPING
from django.template import TOKEN_BLOCK, TOKEN_TEXT, TOKEN_VAR

if 0:
    from blessed import Terminal
    t = Terminal()

# TODO: Add a check for TEMPLATE_DEBUG, and make noise if it is false.


class Plugin(coverage.plugin.CoveragePlugin, coverage.plugin.FileTracer):

    def __init__(self, options):
        super(Plugin, self).__init__(options)
        self.django_dir = os.path.dirname(django.__file__)
        self.django_template_dir = os.path.join(self.django_dir, "template")

        self.source_map = {}

    # --- CoveragePlugin methods

    def file_tracer(self, filename):
        if filename.startswith(self.django_template_dir):
            if "templatetags" not in filename:
                return self
        return None

    def file_reporter(self, filename):
        return FileReporter(filename)

    # --- FileTracer methods

    def has_dynamic_source_filename(self):
        return True

    def dynamic_source_filename(self, filename, frame):
        if frame.f_code.co_name != 'render':
            return None

        locals = frame.f_locals
        render_self = locals['self']
        if 0:
            dump_frame(frame)
        try:
            source = render_self.source
            origin = source[0]
            filename = origin.name
            return filename
        except (AttributeError, IndexError):
            pass
        return None

    def line_number_range(self, frame):
        assert frame.f_code.co_name == 'render'
        render_self = frame.f_locals['self']
        source = render_self.source
        s_start, s_end = source[1]
        if isinstance(render_self, TextNode):
            text = render_self.s
            first_line = text.splitlines(True)[0]
            if first_line.isspace():
                s_start += len(first_line)
        line_map = self.get_line_map(source[0].name)
        start = get_line_number(line_map, s_start)
        end = get_line_number(line_map, s_end-1)
        if start < 0 or end < 0:
            return -1, -1
        return start, end

    # --- FileTracer helpers

    def get_line_map(self, filename):
        """The line map for `filename`.

        A line map is a list of character offsets, indicating where each line
        in the text begins.  For example, a line map like this::

            [13, 19, 30]

        means that line 2 starts at character 13, line 3 starts at 19, etc.
        Line 1 always starts at character 0.

        """
        if filename not in self.source_map:
            with open(filename) as template_file:
                template_source = template_file.read()
                if 0:   # change to see the template text
                    for i in range(0, len(template_source), 10):
                        print("%3d: %r" % (i, template_source[i:i+10]))
            self.source_map[filename] = make_line_map(template_source)
        return self.source_map[filename]


class FileReporter(coverage.plugin.FileReporter):
    def __init__(self, filename):
        # TODO: do we want the .filename attribute to be part of the public
        # API of the coverage plugin?
        self.filename = filename

    def statements(self):
        source_lines = set()

        with open(self.filename) as f:
            text = f.read()

        tokens = Lexer(text, "<string>").tokenize()

        comment = False
        for token in tokens:
            if 0:   # change to see the tokens from the template
                print(
                    "%10s %2d: %r" % (
                        TOKEN_MAPPING[token.token_type],
                        token.lineno,
                        token.contents,<commit_msg>This lets us finish coverage html, but wonky<commit_after>"""The Django coverage plugin."""

from __future__ import print_function, unicode_literals

import os.path
from six.moves import range

import coverage.plugin

import django
from django.template import Lexer, TextNode
from django.template.base import TOKEN_MAPPING
from django.template import TOKEN_BLOCK, TOKEN_TEXT, TOKEN_VAR

if 0:
    from blessed import Terminal
    t = Terminal()

# TODO: Add a check for TEMPLATE_DEBUG, and make noise if it is false.


class Plugin(coverage.plugin.CoveragePlugin, coverage.plugin.FileTracer):

    def __init__(self, options):
        super(Plugin, self).__init__(options)
        self.django_dir = os.path.dirname(django.__file__)
        self.django_template_dir = os.path.join(self.django_dir, "template")

        self.source_map = {}

    # --- CoveragePlugin methods

    def file_tracer(self, filename):
        if filename.startswith(self.django_template_dir):
            if "templatetags" not in filename:
                return self
        return None

    def file_reporter(self, filename):
        return FileReporter(filename)

    # --- FileTracer methods

    def has_dynamic_source_filename(self):
        return True

    def dynamic_source_filename(self, filename, frame):
        if frame.f_code.co_name != 'render':
            return None

        locals = frame.f_locals
        render_self = locals['self']
        if 0:
            dump_frame(frame)
        try:
            source = render_self.source
            origin = source[0]
            filename = origin.name
            return filename
        except (AttributeError, IndexError):
            pass
        return None

    def line_number_range(self, frame):
        assert frame.f_code.co_name == 'render'
        render_self = frame.f_locals['self']
        source = render_self.source
        s_start, s_end = source[1]
        if isinstance(render_self, TextNode):
            text = render_self.s
            first_line = text.splitlines(True)[0]
            if first_line.isspace():
                s_start += len(first_line)
        line_map = self.get_line_map(source[0].name)
        start = get_line_number(line_map, s_start)
        end = get_line_number(line_map, s_end-1)
        if start < 0 or end < 0:
            return -1, -1
        return start, end

    # --- FileTracer helpers

    def get_line_map(self, filename):
        """The line map for `filename`.

        A line map is a list of character offsets, indicating where each line
        in the text begins.  For example, a line map like this::

            [13, 19, 30]

        means that line 2 starts at character 13, line 3 starts at 19, etc.
        Line 1 always starts at character 0.

        """
        if filename not in self.source_map:
            with open(filename) as template_file:
                template_source = template_file.read()
                if 0:   # change to see the template text
                    for i in range(0, len(template_source), 10):
                        print("%3d: %r" % (i, template_source[i:i+10]))
            self.source_map[filename] = make_line_map(template_source)
        return self.source_map[filename]


class FileReporter(coverage.plugin.FileReporter):
    def __init__(self, filename):
        # TODO: do we want the .filename attribute to be part of the public
        # API of the coverage plugin?
        self.filename = filename
        # TODO: is self.name required? Can the base class provide it somehow?
        self.name = os.path.basename(filename)

    def statements(self):
        source_lines = set()

        with open(self.filename) as f:
            text = f.read()

        tokens = Lexer(text, "<string>").tokenize()

        comment = False
        for token in tokens:
            if 0:   # change to see the tokens from the template
                print(
                    "%10s %2d: %r" % (
                        TOKEN_MAPPING[token.token_type],
                        token.lineno,
                        token.contents,
{'fraction_non_alphanumeric': 0.08402100525131283, 'fraction_numerical': 0.00900225056264066, 'mean_word_length': 2.084843810258388, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4768926', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2197, 'n_words': 665}