<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import vim
import re
import os
import os.path
import subprocess
import tempfile
import itertools
import multiprocessing
from .utils import *
from .explorer import *
from .manager import *
from .asyncExecutor import AsyncExecutor


#*****************************************************
# BufTagExplorer
#*****************************************************
class BufTagExplorer(Explorer):
    def __init__(self):
        self._ctags = lfEval("g:Lf_Ctags")
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))
        self._tag_list = {}        # a dict with (key, value) = (buffer number, taglist)
        self._buf_changedtick = {} # a dict with (key, value) = (buffer number, changedtick)
        self._executor = []

    def getContent(self, *args, **kwargs):
        if len(args) > 0: # all buffers
            cur_buffer = vim.current.buffer
            for b in vim.buffers:
                if b.options["buflisted"]:
                    if lfEval("bufloaded(%d)" % b.number) == '0':
                        vim.current.buffer = b
            if vim.current.buffer != cur_buffer:
                vim.current.buffer = cur_buffer

            for b in vim.buffers:
                if b.options["buflisted"] and b.name:
                    changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % b.number))
                    if changedtick != self._buf_changedtick.get(b.number, -1):
                        break
            else:
                return list(itertools.chain.from_iterable(self._tag_list.values()))

            return itertools.chain.from_iterable(self._getTagList())
        else:
            result = self._getTagResult(vim.current.buffer)
            if isinstance(result, list):
                return result
            else:
                return self._formatResult(*result)

    def _getTagList(self):
        buffers = [b for b in vim.buffers]
        n = multiprocessing.cpu_count()
        for i in range(0, len(vim.buffers), n):
            tag_list = []
            exe_result = []
            for b in buffers[i:i+n]:
                if b.options["buflisted"] and b.name:
                    result = self._getTagResult(b)
                    if isinstance(result, list):
                        tag_list.extend(result)
                    else:
                        exe_result.append(result)
            if not exe_result:
                yield tag_list
            else:
                exe_taglist = (self._formatResult(*r) for r in exe_result)
                # list can reduce the flash of screen
                yield list(itertools.chain(tag_list, itertools.chain.from_iterable(exe_taglist)))

    def _getTagResult(self, buffer):
        if not buffer.name:
            return []
        changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buffer.number))
        # there is no change since last call
        if changedtick == self._buf_changedtick.get(buffer.number, -1):
            if buffer.number in self._tag_list:
                return self._tag_list[buffer.number]
            else:
                return []
        else:
            self._buf_changedtick[buffer.number] = changedtick

        if lfEval("getbufvar(%d, '&filetype')" % buffer.number) == "cpp":
            extra_options = "--c++-kinds=+p"
        elif lfEval("getbufvar(%d, '&filetype')" % buffer.number) == "c":
            extra_options = "--c-kinds=+p"
        else:
            extra_options = ""

        executor = AsyncExecutor()
        self._executor.append(executor)
        if buffer.options["modified"] == True:
            with tempfile.NamedTemporaryFile(mode='w+',
                                             suffix='_'+os.path.basename(buffer.name),
                                             delete=False) as f:
                for line in buffer[:]:
                    f.write(line + '\n')
                file_name = f.name
            # {tagname}<Tab>{tagfile}<Tab>{tagaddress}[;"<Tab>{tagfield}..]
            # {tagname}<Tab>{tagfile}<Tab>{tagaddress};"<Tab>{kind}<Tab>{scope}
            cmd = '{} -n -u --fields=Ks {} -f- "{}"'.format(self._ctags, extra_options, lfDecode(file_name))
            result = executor.execute(cmd, cleanup=partial(os.remove, file_name))<commit_msg>add support for *.pyw file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import vim
import re
import os
import os.path
import subprocess
import tempfile
import itertools
import multiprocessing
from .utils import *
from .explorer import *
from .manager import *
from .asyncExecutor import AsyncExecutor


#*****************************************************
# BufTagExplorer
#*****************************************************
class BufTagExplorer(Explorer):
    def __init__(self):
        self._ctags = lfEval("g:Lf_Ctags")
        self._supports_preview = int(lfEval("g:Lf_PreviewCode"))
        self._tag_list = {}        # a dict with (key, value) = (buffer number, taglist)
        self._buf_changedtick = {} # a dict with (key, value) = (buffer number, changedtick)
        self._executor = []

    def getContent(self, *args, **kwargs):
        if len(args) > 0: # all buffers
            cur_buffer = vim.current.buffer
            for b in vim.buffers:
                if b.options["buflisted"]:
                    if lfEval("bufloaded(%d)" % b.number) == '0':
                        vim.current.buffer = b
            if vim.current.buffer != cur_buffer:
                vim.current.buffer = cur_buffer

            for b in vim.buffers:
                if b.options["buflisted"] and b.name:
                    changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % b.number))
                    if changedtick != self._buf_changedtick.get(b.number, -1):
                        break
            else:
                return list(itertools.chain.from_iterable(self._tag_list.values()))

            return itertools.chain.from_iterable(self._getTagList())
        else:
            result = self._getTagResult(vim.current.buffer)
            if isinstance(result, list):
                return result
            else:
                return self._formatResult(*result)

    def _getTagList(self):
        buffers = [b for b in vim.buffers]
        n = multiprocessing.cpu_count()
        for i in range(0, len(vim.buffers), n):
            tag_list = []
            exe_result = []
            for b in buffers[i:i+n]:
                if b.options["buflisted"] and b.name:
                    result = self._getTagResult(b)
                    if isinstance(result, list):
                        tag_list.extend(result)
                    else:
                        exe_result.append(result)
            if not exe_result:
                yield tag_list
            else:
                exe_taglist = (self._formatResult(*r) for r in exe_result)
                # list can reduce the flash of screen
                yield list(itertools.chain(tag_list, itertools.chain.from_iterable(exe_taglist)))

    def _getTagResult(self, buffer):
        if not buffer.name:
            return []
        changedtick = int(lfEval("getbufvar(%d, 'changedtick')" % buffer.number))
        # there is no change since last call
        if changedtick == self._buf_changedtick.get(buffer.number, -1):
            if buffer.number in self._tag_list:
                return self._tag_list[buffer.number]
            else:
                return []
        else:
            self._buf_changedtick[buffer.number] = changedtick

        if lfEval("getbufvar(%d, '&filetype')" % buffer.number) == "cpp":
            extra_options = "--c++-kinds=+p"
        elif lfEval("getbufvar(%d, '&filetype')" % buffer.number) == "c":
            extra_options = "--c-kinds=+p"
        elif lfEval("getbufvar(%d, '&filetype')" % buffer.number) == "python":
            extra_options = "--language-force=Python"
        else:
            extra_options = ""

        executor = AsyncExecutor()
        self._executor.append(executor)
        if buffer.options["modified"] == True:
            with tempfile.NamedTemporaryFile(mode='w+',
                                             suffix='_'+os.path.basename(buffer.name),
                                             delete=False) as f:
                for line in buffer[:]:
                    f.write(line + '\n')
                file_name = f.name
            # {tagname}<Tab>{tagfile}<Tab>{tagaddress}[;"<Tab>{tagfield}..]
            # {tagname}<Tab>{tagfile}<Tab>{tagaddress};"<Tab>{kind}<Tab>{scope}
            cmd = '{} -n -u --fields=Ks {} -f- "{}"'.format(self._ctags, extra_options, lfDecode(file_name))
            result = executor.execute(cmd, cleanup=partial(os.remove, file_name))
{'fraction_non_alphanumeric': 0.14150513112884835, 'fraction_numerical': 0.0013683010262257698, 'mean_word_length': 1.870091623036649, 'pattern_counts': {'":': 5, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075501', 'n_tokens_mistral': 2550, 'n_tokens_neox': 2441, 'n_words': 532}