<commit_before>import shlex
import subprocess
import re
from os.path import relpath

from denite import util, process
from .base import Base

# GREP_HEADER_SYNTAX = (
#     'syntax match deniteSource_grepHeader '
#     r'/\v[^:]*:\d+(:\d+)? / '
#     'contained keepend')

# GREP_FILE_SYNTAX = (
#     'syntax match deniteSource_grepFile '
#     r'/[^:]*:/ '
#     'contained containedin=deniteSource_grepHeader '
#     'nextgroup=deniteSource_grepLineNR')
# GREP_FILE_HIGHLIGHT = 'highlight default link deniteSource_grepFile Comment'

# GREP_LINE_SYNTAX = (
#     'syntax match deniteSource_grepLineNR '
#     r'/\d\+\(:\d\+\)\?/ '
#     'contained containedin=deniteSource_grepHeader')
# GREP_LINE_HIGHLIGHT = 'highlight default link deniteSource_grepLineNR LineNR'

# GREP_PATTERNS_HIGHLIGHT = 'highlight default link deniteGrepPatterns Function'

def _candidate(result, context):
    m = re.match(r'^(\S+) \[([^\]]+)\] \(([^\):]+):(\d+)\)', result)
    if not m:
        return None

    path = context['__config_base_path'] + '/' + m[3]
    path = relpath(path, start=context['path'])

    return {
        'word': result,
        'action__path': path,
        'action__line': m[4],
        'action__col': 1,
    }

class Source(Base):
    def __init__(self, vim):
        super().__init__(vim)

        self.name = 'i18n_flow'
        self.kind = 'file'
        self.matchers = ['matcher/ignore_globs', 'matcher/regexp']

    def on_init(self, context):
        context['__proc'] = None

        args = dict(enumerate(context['args']))

        context['__paths'] = self._init_paths(context, args)
        context['__arguments'] = self._init_arguments(context, args)
        context['__patterns'] = self._init_patterns(context, args)

    def on_close(self, context):
        if context['__proc']:
            context['__proc'].kill()
            context['__proc'] = None

    # def highlight(self):
    #     self.vim.command(GREP_HEADER_SYNTAX)
    #     self.vim.command(GREP_FILE_SYNTAX)
    #     self.vim.command(GREP_FILE_HIGHLIGHT)
    #     self.vim.command(GREP_LINE_SYNTAX)
    #     self.vim.command(GREP_LINE_HIGHLIGHT)
    #     self.vim.command(GREP_PATTERNS_HIGHLIGHT)

    # def define_syntax(self):
    #     self.vim.command(
    #         'syntax region ' + self.syntax_name + ' start=// end=/$/ '
    #         'contains=deniteSource_grepHeader,deniteMatchedRange contained')
    #     if self.context['__patterns']:
    #         self.vim.command(
    #             'syntax match deniteGrepPatterns ' +
    #             r'/%s/ ' % r'\|'.join(
    #                 util.regex_convert_str_vim(pattern)
    #                 for pattern in self.context['__patterns']) +
    #             'contained containedin=' + self.syntax_name)

    def gather_candidates(self, context):
        if context['event'] == 'interactive':
            # Update input
            self.on_close(context)

            if not context['input']:
                return []<commit_msg>Add highlights<commit_after>import shlex
import subprocess
import re
from os.path import relpath

from denite import util, process
from .base import Base

_HEADER_SYNTAX = (
    'syntax match deniteSource_i18nflowHeader '
    r'/\v(\S+) \[([^\]]+)\] \(([^\):]+):(\d+)\)/ '
    'contained keepend'
)
_HEADER_KEY_SYNTAX = (
    'syntax match deniteSource_i18nflowHeaderKey '
    r'/\v(\S+)\ze \[/ '
    'contained containedin=deniteSource_i18nflowHeader'
)
_HEADER_LOCALE_SYNTAX = (
    'syntax match deniteSource_i18nflowHeaderLocale '
    r'/\v\[\zs[^\]]+\ze\]/ '
    'contained containedin=deniteSource_i18nflowHeader'
)
_HEADER_LOCATION_SYNTAX = (
    'syntax match deniteSource_i18nflowHeaderLocation '
    r'/\v\(\zs[^\)]+\ze\)/ '
    'contained containedin=deniteSource_i18nflowHeader'
)

_HEADER_KEY_HIGHLIGHT = 'highlight default link deniteSource_i18nflowHeaderKey String'
_HEADER_LOCALE_HIGHLIGHT = 'highlight default link deniteSource_i18nflowHeaderLocale Function'
_HEADER_LOCATION_HIGHLIGHT = 'highlight default link deniteSource_i18nflowHeaderLocation Comment'

def _candidate(result, context):
    m = re.match(r'^(\S+) \[([^\]]+)\] \(([^\):]+):(\d+)\)', result)
    if not m:
        return None

    path = context['__config_base_path'] + '/' + m[3]
    path = relpath(path, start=context['path'])

    return {
        'word': result,
        'action__path': path,
        'action__line': m[4],
        'action__col': 1,
    }

class Source(Base):
    def __init__(self, vim):
        super().__init__(vim)

        self.name = 'i18n_flow'
        self.kind = 'file'
        self.matchers = ['matcher/ignore_globs', 'matcher/regexp']

    def on_init(self, context):
        context['__proc'] = None

        args = dict(enumerate(context['args']))

        context['__paths'] = self._init_paths(context, args)
        context['__arguments'] = self._init_arguments(context, args)
        context['__patterns'] = self._init_patterns(context, args)

    def on_close(self, context):
        if context['__proc']:
            context['__proc'].kill()
            context['__proc'] = None

    def highlight(self):
        self.vim.command(_HEADER_SYNTAX)
        self.vim.command(_HEADER_KEY_SYNTAX)
        self.vim.command(_HEADER_LOCALE_SYNTAX)
        self.vim.command(_HEADER_LOCATION_SYNTAX)

        self.vim.command(_HEADER_KEY_HIGHLIGHT)
        self.vim.command(_HEADER_LOCALE_HIGHLIGHT)
        self.vim.command(_HEADER_LOCATION_HIGHLIGHT)

    def define_syntax(self):
        self.vim.command(
            'syntax region ' + self.syntax_name + ' start=// end=/$/ '
            'contains=deniteSource_i18nflowHeader,deniteMatchedRange contained')
        if self.context['__patterns']:
            self.vim.command(
                'syntax match deniteI18nflowPatterns ' +
                r'/%s/ ' % r'\|'.join(
                    util.regex_convert_str_vim(pattern)
                    for pattern in self.context['__patterns']) +
                'contained containedin=' + self.syntax_name)

    def gather_candidates(self, context):
        if context['event'] == 'interactive':
            # Update input
            self.on_close(context)

            if not context['input']:
                return []
{'fraction_non_alphanumeric': 0.13791994856132456, 'fraction_numerical': 0.005465359266998875, 'mean_word_length': 2.7617896009673517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4547005', 'n_tokens_mistral': 2164, 'n_tokens_neox': 2039, 'n_words': 349}