<commit_before>import re
import sublime

from lint.highlight import Highlight
import lint.persist as persist
import lint.util as util

syntax_re = re.compile(r'/([^/]+)\.tmLanguage$')

class Tracker(type):
    def __init__(cls, name, bases, attrs):
        persist.add_language(cls, name, attrs)

class Linter(metaclass=Tracker):
    language = ''
    cmd = ()
    regex = ''
    multiline = False
    flags = 0
    tab_size = 1

    scope = 'keyword'
    selector = None
    outline = True
    needs_api = False

    errors = None
    highlight = None
    defaults = None
    lint_settings = None

    def __init__(self, view, syntax, filename=None):
        self.view = view
        self.syntax = syntax
        self.filename = filename

        if self.regex:
            if self.multiline:
                self.flags |= re.MULTILINE

            try:
                self.regex = re.compile(self.regex, self.flags)
            except:
                persist.debug('Error compiling regex for {}'.format(self.language))

        self.highlight = Highlight(scope=self.scope)

    @classmethod
    def get_settings(cls):
        plugins = persist.settings.get('plugins', {})
        settings = cls.defaults or {}
        settings.update(plugins.get(cls.__name__, {}))
        return settings

    @property
    def settings(self):
        return self.get_settings()

    @classmethod
    def assign(cls, view):
        '''
        find a linter for a specified view if possible, then add it to our mapping of view <--> lint class and return it
        each view has its own linter to make it feasible for linters to store persistent data about a view
        '''
        try:
            vid = view.id()
        except RuntimeError:
            pass

        settings = view.settings()
        syn = settings.get('syntax')
        if not syn:
            cls.remove(vid)
            return

        match = syntax_re.search(syn)

        if match:
            syntax, = match.groups()
        else:
            syntax = syn

        if syntax:
            if vid in persist.linters and persist.linters[vid]:
                if tuple(persist.linters[vid])[0].syntax == syntax:
                    return

            linters = set()
            for name, entry in persist.languages.items():
                if entry.can_lint(syntax):
                    linter = entry(view, syntax, view.file_name())
                    linters.add(linter)

            if linters:
                persist.linters[vid] = linters
                return linters

        cls.remove(vid)

    @classmethod
    def remove(cls, vid):
        if vid in persist.linters:
            for linter in persist.linters[vid]:
                linter.clear()

            del persist.linters[vid]

    @classmethod
    def reload(cls, mod=None):
        '''
        reload all linters, optionally filtering by module
        '''
        plugins = persist.settings.get('plugins', {})
        for name, linter in persist.languages.items():
            settings = plugins.get(name, {})
            defaults = (linter.defaults or {}).copy()
            defaults.update(settings)
            linter.lint_settings = defaults

        for id, linters in persist.linters.items():
            for linter in linters:
                if mod and linter.__module__ != mod:
                    continue

                linter.clear()
                persist.linters[id].remove(linter)
                linter = persist.languages[linter.name](linter.view, linter.syntax, linter.filename)
                persist.linters[id].add(linter)
                linter.draw()

        return

    @classmethod
    def text(cls, view):
        return view.substr(sublime.Region(0, view.size()))

    @classmethod
    def get_view(cls, view_id):
        if view_id in persist.linters:
            return tuple(persist.linters[view_id])[0].view

    @classmethod
    def get_linters(cls, view_id):
        if view_id in persist.linters:
            return tuple(persist.linters[view_id])

        return ()

    @classmethod
    def get_selectors(cls, view_id):
        return [
            (linter.selector, linter)
            for linter in cls.get_linters(view_id)
            if linter.selector
        ]

    @classmethod
    def lint_view(cls, view_id, filename, code, sections, callback):
        if not code:
            return

        filename = filename or 'untitled'
        if view_id in persist.linters:
            selectors = Linter.get_selectors(view_id)

            linters = tuple(persist.linters[view_id])
            linter_text = (', '.join(l.name for l in linters))
            persist.debug('SublimeLint: `%s` as %s' % (filename, linter_text))
            for linter in linters:
                if linter.settings.get('disable'):
                    continue

                if not linter.selector:
                    linter.reset(code, filename=filename)
                    linter.lint()

            for sel, linter in selectors:
                if sel in sections:
                    linter.reset(code, filename=filename)

                    errors = {}
                    for line_offset, left, right in sections[sel]:
                        highlight.shift(line_offset, left)
                        linter.lint(code[left:right])

                        for line, error in linter.errors.items():
                            errors[line+line_offset] = error

                    linter.errors = errors

            # merge our result back to the main thread
            callback(linters[0].view, linters)

    def reset(self, code, filename=None, highlight=None):
        self.errors = {}
        self.code = code
        self.filename = filename or self.filename
        self.highlight = highlight or Highlight(
            self.code, scope=self.scope, outline=self.outline)<commit_msg>fix regression in scope linters (TODO linter)<commit_after>import re
import sublime

from lint.highlight import Highlight
import lint.persist as persist
import lint.util as util

syntax_re = re.compile(r'/([^/]+)\.tmLanguage$')

class Tracker(type):
    def __init__(cls, name, bases, attrs):
        persist.add_language(cls, name, attrs)

class Linter(metaclass=Tracker):
    language = ''
    cmd = ()
    regex = ''
    multiline = False
    flags = 0
    tab_size = 1

    scope = 'keyword'
    selector = None
    outline = True
    needs_api = False

    errors = None
    highlight = None
    defaults = None
    lint_settings = None

    def __init__(self, view, syntax, filename=None):
        self.view = view
        self.syntax = syntax
        self.filename = filename

        if self.regex:
            if self.multiline:
                self.flags |= re.MULTILINE

            try:
                self.regex = re.compile(self.regex, self.flags)
            except:
                persist.debug('Error compiling regex for {}'.format(self.language))

        self.highlight = Highlight(scope=self.scope)

    @classmethod
    def get_settings(cls):
        plugins = persist.settings.get('plugins', {})
        settings = cls.defaults or {}
        settings.update(plugins.get(cls.__name__, {}))
        return settings

    @property
    def settings(self):
        return self.get_settings()

    @classmethod
    def assign(cls, view):
        '''
        find a linter for a specified view if possible, then add it to our mapping of view <--> lint class and return it
        each view has its own linter to make it feasible for linters to store persistent data about a view
        '''
        try:
            vid = view.id()
        except RuntimeError:
            pass

        settings = view.settings()
        syn = settings.get('syntax')
        if not syn:
            cls.remove(vid)
            return

        match = syntax_re.search(syn)

        if match:
            syntax, = match.groups()
        else:
            syntax = syn

        if syntax:
            if vid in persist.linters and persist.linters[vid]:
                if tuple(persist.linters[vid])[0].syntax == syntax:
                    return

            linters = set()
            for name, entry in persist.languages.items():
                if entry.can_lint(syntax):
                    linter = entry(view, syntax, view.file_name())
                    linters.add(linter)

            if linters:
                persist.linters[vid] = linters
                return linters

        cls.remove(vid)

    @classmethod
    def remove(cls, vid):
        if vid in persist.linters:
            for linter in persist.linters[vid]:
                linter.clear()

            del persist.linters[vid]

    @classmethod
    def reload(cls, mod=None):
        '''
        reload all linters, optionally filtering by module
        '''
        plugins = persist.settings.get('plugins', {})
        for name, linter in persist.languages.items():
            settings = plugins.get(name, {})
            defaults = (linter.defaults or {}).copy()
            defaults.update(settings)
            linter.lint_settings = defaults

        for id, linters in persist.linters.items():
            for linter in linters:
                if mod and linter.__module__ != mod:
                    continue

                linter.clear()
                persist.linters[id].remove(linter)
                linter = persist.languages[linter.name](linter.view, linter.syntax, linter.filename)
                persist.linters[id].add(linter)
                linter.draw()

        return

    @classmethod
    def text(cls, view):
        return view.substr(sublime.Region(0, view.size()))

    @classmethod
    def get_view(cls, view_id):
        if view_id in persist.linters:
            return tuple(persist.linters[view_id])[0].view

    @classmethod
    def get_linters(cls, view_id):
        if view_id in persist.linters:
            return tuple(persist.linters[view_id])

        return ()

    @classmethod
    def get_selectors(cls, view_id):
        return [
            (linter.selector, linter)
            for linter in cls.get_linters(view_id)
            if linter.selector
        ]

    @classmethod
    def lint_view(cls, view_id, filename, code, sections, callback):
        if not code:
            return

        filename = filename or 'untitled'
        if view_id in persist.linters:
            selectors = Linter.get_selectors(view_id)

            linters = tuple(persist.linters[view_id])
            linter_text = (', '.join(l.name for l in linters))
            persist.debug('SublimeLint: `%s` as %s' % (filename, linter_text))
            for linter in linters:
                if linter.settings.get('disable'):
                    continue

                if not linter.selector:
                    linter.reset(code, filename=filename)
                    linter.lint()

            for sel, linter in selectors:
                if sel in sections:
                    linter.reset(code, filename=filename)

                    errors = {}
                    for line_offset, left, right in sections[sel]:
                        linter.highlight.shift(line_offset, left)
                        linter.lint(code[left:right])

                        for line, error in linter.errors.items():
                            errors[line+line_offset] = error

                    linter.errors = errors

            # merge our result back to the main thread
            callback(linters[0].view, linters)

    def reset(self, code, filename=None, highlight=None):
        self.errors = {}
        self.code = code
        self.filename = filename or self.filename
        self.highlight = highlight or Highlight(
            self.code, scope=self.scope, outline=self.outline)
{'fraction_non_alphanumeric': 0.09373672133933883, 'fraction_numerical': 0.0010198011387779382, 'mean_word_length': 1.7052873563218391, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1147099', 'n_tokens_mistral': 3351, 'n_tokens_neox': 3152, 'n_words': 839}