<commit_before># -*- coding: utf-8 -*-

import re
import vim

from completor import Completor


LIMIT = 50
word = re.compile('\w+$')


def gen_tokens(base):
    for buffer in vim.buffers:
        if not buffer.valid:
            continue

        data = '\n'.join(buffer[:])
        for match in re.finditer('{}\w+'.format(base), data, re.M | re.I):
            yield match.group()
<commit_msg>Refined buffer completion<commit_after># -*- coding: utf-8 -*-

import re
import vim

from completor import Completor


LIMIT = 50
word = re.compile('\w+$')


def gen_tokens(base):
    current = vim.current.buffer
    line, _ = vim.current.window.cursor

    pat = re.compile('{}\w+'.format(base), re.M | re.I)

    for buffer in vim.buffers:
        if not buffer.valid or \
                (buffer.number != current.number and len(buffer) > 10000):
            continue

        if buffer.number == current.number:
            start = line - 1000
            end = line + 1000
            if start < 0:
                start = 0
            data = '\n'.join(buffer[start:end])
        else:
            data = '\n'.join(buffer[:])
        for match in pat.finditer(data):
            yield match.group()

{'fraction_non_alphanumeric': 0.13666666666666666, 'fraction_numerical': 0.0175, 'mean_word_length': 1.9292682926829268, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '218039', 'n_tokens_mistral': 398, 'n_tokens_neox': 368, 'n_words': 102}