<commit_before>import os.path
from collections import defaultdict
from itertools import islice

from vial import vim, vfunc
from vial.fsearch import get_files
from vial.utils import focus_window, get_projects, buffer_with_file, mark, single
from vial.widgets import ListFormatter, ListView, SearchDialog


class MatchTree(object):
    def __init__(self):
        self.files = []
        self.names = defaultdict(lambda: defaultdict(set))
        self.parts = []

    def clear(self):
        self.files[:] = []
        self.names.clear()
        self.parts[:] = []

    def extend(self, items):
        idx = len(self.files)
        for item in items:
            self.files.append(item)
            parts = item[1].split('/')
            parts.reverse()
            self.parts.append(parts)
            for i, p in enumerate(parts):
                self.names[i][p].add(idx)
            idx += 1

    def match_name(self, query):
        for n, indices in self.names[0].iteritems():
            if n.startswith(query):
                yield indices

        for n, indices in self.names[0].iteritems():
            if query in n:
                yield indices

    def get_files(self, matcher):
        matched = set()
        files = self.files
        for indices in matcher:
            for idx in indices - matched:
                yield files[idx]
            matched.update(indices)

    def get_indexes(self, matcher):
        matched = set()
        files = self.files
        for indices in matcher:
            for idx in indices - matched:
                yield idx
            matched.update(indices)

    def match_path(self, query):
        d, _, n = query.rpartition('/')
        parts = self.parts
        for level in range(1, 5):
            for idx in self.get_indexes(self.match_name(n)):
                try:
                    part = parts[idx][level]
                except IndexError:
                    continue
                if part.startswith(d):
                    yield set((idx,))

        for level in range(1, 5):
            for idx in self.get_indexes(self.match_name(n)):
                try:
                    part = parts[idx][level]
                except IndexError:
                    continue
                if d in part:
                    yield set((idx,))

    def match(self, query):
        if '/' in query:
            return self.get_files(self.match_path(query))
        else:
            return self.get_files(self.match_name(query))


class QuickOpen(SearchDialog):
    def __init__(self):
        self.filelist = []
        SearchDialog.__init__(self, '__vial_quick_open__',
            ListView(self.filelist, ListFormatter(0, 0, 1, 1)), 'quick-open')

        self.matcher = MatchTree()
        self.file_iter_cache = {}

    def open(self):
        self.matcher.clear()
        self.file_iter_cache.clear()
        self.matcher.extend(self.get_buffer_paths())
        self.last_window = vfunc.winnr()
        self.roots = ['__buffer__'] + list(get_projects())
        self.list_view.clear()
        self.show(u'')
        self.loop.enter()
<commit_msg>fast sub path matching and keep sorted order of vial.fsearch<commit_after>import os.path
from collections import defaultdict
from itertools import islice

from vial import vim, vfunc
from vial.fsearch import get_files
from vial.utils import focus_window, get_projects, buffer_with_file, mark, single
from vial.widgets import ListFormatter, ListView, SearchDialog


class MatchTree(object):
    def __init__(self):
        self.files = []
        self.names = defaultdict(lambda: defaultdict(list))

    def clear(self):
        self.files[:] = []
        self.names.clear()

    def extend(self, items):
        idx = len(self.files)
        for item in items:
            self.files.append(item)
            parts = item[1].split('/')
            parts.reverse()
            for i, p in enumerate(parts):
                self.names[i][p].append(idx)
            idx += 1

    def match_name(self, query, level=0):
        if level not in self.names:
            return

        for n, indices in sorted(self.names[level].iteritems()):
            if n.startswith(query):
                yield indices

        for n, indices in sorted(self.names[level].iteritems()):
            if query in n:
                yield indices

    def get_files(self, matcher):
        matched = set()
        files = self.files
        for indices in matcher:
            for idx in (r for r in indices if r not in matched):
                yield files[idx]
            matched.update(indices)

    def match_path(self, query):
        d, _, n = query.rpartition('/')
        def di():
            for level in range(1, 5):
                for r in self.match_name(d, level):
                    yield r

        for item in self.get_files(di()):
            if item[0].startswith(n):
                yield item

        for item in self.get_files(di()):
            if n in item[0] and not item[0].startswith(n):
                yield item

    def match(self, query):
        if '/' in query:
            return self.match_path(query)
        else:
            return self.get_files(self.match_name(query))


class QuickOpen(SearchDialog):
    def __init__(self):
        self.filelist = []
        SearchDialog.__init__(self, '__vial_quick_open__',
            ListView(self.filelist, ListFormatter(0, 0, 1, 1)), 'quick-open')

        self.matcher = MatchTree()
        self.file_iter_cache = {}

    def open(self):
        self.matcher.clear()
        self.file_iter_cache.clear()
        self.matcher.extend(self.get_buffer_paths())
        self.last_window = vfunc.winnr()
        self.roots = ['__buffer__'] + list(get_projects())
        self.list_view.clear()
        self.show(u'')
        self.loop.enter()

{'fraction_non_alphanumeric': 0.10616732520185534, 'fraction_numerical': 0.004123002920460402, 'mean_word_length': 1.8344693281402142, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3607385', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1643, 'n_words': 431}