<commit_before>import sublime
import sublime_plugin
from os import path
import re
from collections import namedtuple


AssertionLineDetails = namedtuple(
    'AssertionLineDetails', ['comment_marker_match', 'assertion_colrange', 'line_region']
)
SyntaxTestHeader = namedtuple(
    'SyntaxTestHeader', ['comment_start', 'comment_end', 'syntax_file']
)


def get_syntax_test_tokens(view):
    """Parse the first line of the given view into a SyntaxTestHeader.

    Returns `None` if the file doesn't contain syntax tests.
    """

    line = view.line(0)
    match = None
    if line.size() < 1000:  # no point checking longer lines as they are unlikely to match
        first_line = view.substr(line)
        match = re.match(r'^(?P<comment_start>\s*\S+)'
                         r'\s+SYNTAX TEST\s+'
                         r'"(?P<syntax_file>[^"]+)"'
                         r'\s*(?P<comment_end>\S+)?$', first_line)
    if not match:
        return None
    else:
        return SyntaxTestHeader(**match.groupdict())


class SyntaxTestHighlighterListener(sublime_plugin.ViewEventListener):
    header = None

    def __init__(self, view):
        super().__init__(view)
        self.on_modified_async()
        self.on_selection_modified_async()

    def on_modified_async(self):
        """If the view has a filename, and that file name starts with
        the syntax test file prefix, or the view has no filename yet,
        read the first line to determine the syntax test header.
        """

        name = self.view.file_name()
        if name:
            name = path.basename(name)
            if not name.startswith('syntax_test_'):
                self.header = None
                return
        self.header = get_syntax_test_tokens(self.view)
        # if there is no comment start token, clear the header
        if self.header and not self.header.comment_start:
            self.header = None

    def get_details_of_test_assertion_line(self, pos):
        """Parse details of a test assertion at a given position in the view.

        Always returns a AssertionLineDetails instance, whose fields may be `None`.
        """

        tokens = self.header
        if not tokens:
            return AssertionLineDetails(None, None, None)
        line_region = self.view.line(pos)
        line_text = self.view.substr(line_region)
        test_start_token = re.match(r'^\s*(' + re.escape(tokens.comment_start) + r')', line_text)
        assertion_colrange = None
        if test_start_token:
            assertion = re.match(r'\s*(?:(<-)|(\^+))', line_text[test_start_token.end():])
            if assertion:
                if assertion.group(1):
                    assertion_colrange = (test_start_token.start(1),
                                          test_start_token.start(1))
                elif assertion.group(2):
                    assertion_colrange = (test_start_token.end() + assertion.start(2),
                                          test_start_token.end() + assertion.end(2))

        return AssertionLineDetails(test_start_token, assertion_colrange, line_region)

    def is_syntax_test_line(self, pos, must_contain_assertion):
        """Determine whether the line at the given character position is a syntax test line.

        It can optionally treat lines with comment markers but no assertion as a syntax test,
        useful for while the line is being written.
        """

        details = self.get_details_of_test_assertion_line(pos)
        if details.comment_marker_match:
            return not must_contain_assertion or details.assertion_colrange is not None
        return False

    def get_details_of_line_being_tested(self):
        """Starting from the cursor position, work upwards to find all syntax
         test lines that occur before the line that is being tested.
        Return a tuple containing a list of assertion line details,
        along with the region of the line being tested.
        """

        if not self.header:
            return (None, None)

        lines = []
        pos = self.view.sel()[0].begin()
        first_line = True
        while pos >= 0:
            details = self.get_details_of_test_assertion_line(pos)
            pos = details.line_region.begin() - 1
            if details.assertion_colrange:
                lines.append(details)
            elif not first_line or not details.comment_marker_match:
                break
            elif details.comment_marker_match:
                lines.append(details)
            first_line = False

        return (lines, details.line_region)

    def on_selection_modified_async(self):
        """Update highlighting of what the current line's test assertions point at."""

        if len(self.view.sel()) == 0:
            return

        lines, line = self.get_details_of_line_being_tested()

        if not lines or not lines[0].assertion_colrange:
            self.view.erase_regions('current_syntax_test')
            return

        cursor = self.view.sel()[0]
        highlight_only_cursor = False
        if cursor.empty():
            cursor = sublime.Region(cursor.begin(), cursor.end() + 1)
        else:
            highlight_only_cursor = re.match(r'^\^+$', self.view.substr(cursor)) is not None

        col_start, col_end = lines[0].assertion_colrange
        if highlight_only_cursor:
            col_start = self.view.rowcol(cursor.begin())[1]
            col_end = self.view.rowcol(cursor.end())[1]
        elif col_end == col_start:
            col_end += 1

        # if the tests extend past the newline character, stop highlighting at the \n
        # as this is what these tests will assert against
        pos_end = min(line.begin() + col_end, line.end() + 1)
        region = sublime.Region(line.begin() + col_start, pos_end)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        scope = prefs.get('syntax_test_highlight_scope', 'text')
        styles = prefs.get('syntax_test_highlight_styles', ['DRAW_NO_FILL'])
        style_flags = 0<commit_msg>Also test start of region for line bounds<commit_after>import sublime
import sublime_plugin
from os import path
import re
from collections import namedtuple


AssertionLineDetails = namedtuple(
    'AssertionLineDetails', ['comment_marker_match', 'assertion_colrange', 'line_region']
)
SyntaxTestHeader = namedtuple(
    'SyntaxTestHeader', ['comment_start', 'comment_end', 'syntax_file']
)


def get_syntax_test_tokens(view):
    """Parse the first line of the given view into a SyntaxTestHeader.

    Returns `None` if the file doesn't contain syntax tests.
    """

    line = view.line(0)
    match = None
    if line.size() < 1000:  # no point checking longer lines as they are unlikely to match
        first_line = view.substr(line)
        match = re.match(r'^(?P<comment_start>\s*\S+)'
                         r'\s+SYNTAX TEST\s+'
                         r'"(?P<syntax_file>[^"]+)"'
                         r'\s*(?P<comment_end>\S+)?$', first_line)
    if not match:
        return None
    else:
        return SyntaxTestHeader(**match.groupdict())


class SyntaxTestHighlighterListener(sublime_plugin.ViewEventListener):
    header = None

    def __init__(self, view):
        super().__init__(view)
        self.on_modified_async()
        self.on_selection_modified_async()

    def on_modified_async(self):
        """If the view has a filename, and that file name starts with
        the syntax test file prefix, or the view has no filename yet,
        read the first line to determine the syntax test header.
        """

        name = self.view.file_name()
        if name:
            name = path.basename(name)
            if not name.startswith('syntax_test_'):
                self.header = None
                return
        self.header = get_syntax_test_tokens(self.view)
        # if there is no comment start token, clear the header
        if self.header and not self.header.comment_start:
            self.header = None

    def get_details_of_test_assertion_line(self, pos):
        """Parse details of a test assertion at a given position in the view.

        Always returns a AssertionLineDetails instance, whose fields may be `None`.
        """

        tokens = self.header
        if not tokens:
            return AssertionLineDetails(None, None, None)
        line_region = self.view.line(pos)
        line_text = self.view.substr(line_region)
        test_start_token = re.match(r'^\s*(' + re.escape(tokens.comment_start) + r')', line_text)
        assertion_colrange = None
        if test_start_token:
            assertion = re.match(r'\s*(?:(<-)|(\^+))', line_text[test_start_token.end():])
            if assertion:
                if assertion.group(1):
                    assertion_colrange = (test_start_token.start(1),
                                          test_start_token.start(1))
                elif assertion.group(2):
                    assertion_colrange = (test_start_token.end() + assertion.start(2),
                                          test_start_token.end() + assertion.end(2))

        return AssertionLineDetails(test_start_token, assertion_colrange, line_region)

    def is_syntax_test_line(self, pos, must_contain_assertion):
        """Determine whether the line at the given character position is a syntax test line.

        It can optionally treat lines with comment markers but no assertion as a syntax test,
        useful for while the line is being written.
        """

        details = self.get_details_of_test_assertion_line(pos)
        if details.comment_marker_match:
            return not must_contain_assertion or details.assertion_colrange is not None
        return False

    def get_details_of_line_being_tested(self):
        """Starting from the cursor position, work upwards to find all syntax
         test lines that occur before the line that is being tested.
        Return a tuple containing a list of assertion line details,
        along with the region of the line being tested.
        """

        if not self.header:
            return (None, None)

        lines = []
        pos = self.view.sel()[0].begin()
        first_line = True
        while pos >= 0:
            details = self.get_details_of_test_assertion_line(pos)
            pos = details.line_region.begin() - 1
            if details.assertion_colrange:
                lines.append(details)
            elif not first_line or not details.comment_marker_match:
                break
            elif details.comment_marker_match:
                lines.append(details)
            first_line = False

        return (lines, details.line_region)

    def on_selection_modified_async(self):
        """Update highlighting of what the current line's test assertions point at."""

        if len(self.view.sel()) == 0:
            return

        lines, line = self.get_details_of_line_being_tested()

        if not lines or not lines[0].assertion_colrange:
            self.view.erase_regions('current_syntax_test')
            return

        cursor = self.view.sel()[0]
        highlight_only_cursor = False
        if cursor.empty():
            cursor = sublime.Region(cursor.begin(), cursor.end() + 1)
        else:
            highlight_only_cursor = re.match(r'^\^+$', self.view.substr(cursor)) is not None

        col_start, col_end = lines[0].assertion_colrange
        if highlight_only_cursor:
            col_start = self.view.rowcol(cursor.begin())[1]
            col_end = self.view.rowcol(cursor.end())[1]
        elif col_end == col_start:
            col_end += 1

        # if the tests extend past the newline character, stop highlighting at the \n
        # as this is what these tests will assert against
        pos_start = min(line.begin() + col_start, line.end())
        pos_end = min(line.begin() + col_end, line.end() + 1)
        region = sublime.Region(pos_start, pos_end)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        scope = prefs.get('syntax_test_highlight_scope', 'text')
        styles = prefs.get('syntax_test_highlight_styles', ['DRAW_NO_FILL'])
        style_flags = 0
{'fraction_non_alphanumeric': 0.09182079736950267, 'fraction_numerical': 0.003945745992601726, 'mean_word_length': 2.314083356033778, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4582086', 'n_tokens_mistral': 3495, 'n_tokens_neox': 3309, 'n_words': 937}