<commit_before>import vim
import subprocess
import string
import os
import threading
from multiprocessing.pool import ThreadPool

# Decorators
def debounce(wait):
    """ Decorator that will postpone a functions
        execution until after wait seconds
        have elapsed since the last time it was invoked. """
    def decorator(fn):
        def debounced(*args, **kwargs):
            def call_it():
                fn(*args, **kwargs)
            try:
                debounced.t.cancel()
            except(AttributeError):
                pass
            debounced.t = threading.Timer(wait, call_it)
            debounced.t.start()
        return debounced
    return decorator


def one_at_a_time():
    def decorator(fn):
        lock = threading.Lock()

        def onced(*args, **kwargs):
            if lock.acquire(False):
                try:
                    fn(*args, **kwargs)
                finally:
                    lock.release()
        return onced
    return decorator


# Functions called from vim
@debounce(2)
def update_cache_eventually():
    update_cache()


cache = ''
@debounce(0)
@one_at_a_time()
def update_cache():
    return


pool = ThreadPool(processes=4)
def run_commands():
    commands = vim.eval("a:commands")

    outputs = _run_commands(commands)

    vim.command('return [%s]' % ','.join(outputs))


def run_command():
    command = vim.eval("a:command")
    output = get_output(command)
    vim.command('return %s' % vim_str(output))


# Support functions
def _run_commands(commands):
    results = pool.map(get_output, commands)
    return [vim_str(result) for result in results]


def get_output(command):
    return subprocess.check_output("%s" % (command), shell=True)


def git_diff(args, extra=""):
    return (" git diff --diff-filter=AM --no-color {} 2>/dev/null"
            " | grep \\^+\s*.. 2>/dev/null"
            " | grep -v '+++ [ab]/' 2>/dev/null"
            "{}"
            " || true"
            ).format(args, extra)


def untracked_keywords():
    return ('git ls-files --others --exclude-standard 2>/dev/null'
            ' | xargs -I % {}'
            ).format(git_diff('--no-index /dev/null %'))


def uncommitted_keywords():
    return git_diff("HEAD")


def recently_committed_keywords():
    return git_diff("@'{1.hour.ago}' HEAD", "| sed '1!G;h;$!d' 2>/dev/null")

commands = [
        untracked_keywords(),
        uncommitted_keywords(),
        recently_committed_keywords(),
        ]

# Vim helpers
def vim_str(text):
    return "'%s'" % text.replace("'", "''")


def echom(text):
    vim.command("echom %s" % vim_str(text))<commit_msg>Update cache in background<commit_after>import vim
import subprocess
import string
import os
import threading
from multiprocessing.pool import ThreadPool

# Decorators
def debounce(wait):
    """ Decorator that will postpone a functions
        execution until after wait seconds
        have elapsed since the last time it was invoked. """
    def decorator(fn):
        def debounced(*args, **kwargs):
            def call_it():
                fn(*args, **kwargs)
            try:
                debounced.t.cancel()
            except(AttributeError):
                pass
            debounced.t = threading.Timer(wait, call_it)
            debounced.t.start()
        return debounced
    return decorator


def one_at_a_time():
    def decorator(fn):
        lock = threading.Lock()

        def onced(*args, **kwargs):
            if lock.acquire(False):
                try:
                    fn(*args, **kwargs)
                finally:
                    lock.release()
        return onced
    return decorator


# Functions called from vim
@debounce(2)
def update_cache_eventually():
    update_cache()


cache = []
@debounce(0)
@one_at_a_time()
def update_cache():
    cache = _run_commands(cacheable_commands)


pool = ThreadPool(processes=4)
def run_commands():
    commands = vim.eval("a:commands")

    outputs = _run_commands(commands)

    vim.command('return [%s]' % ','.join(outputs))


def run_command():
    command = vim.eval("a:command")
    output = get_output(command)
    vim.command('return %s' % vim_str(output))


# Support functions
def _run_commands(commands):
    results = pool.map(get_output, commands)
    return [vim_str(result) for result in results]


def get_output(command):
    return subprocess.check_output("%s" % (command), shell=True)


def git_diff(args, extra=""):
    return (" git diff --diff-filter=AM --no-color {} 2>/dev/null"
            " | grep \\^+\s*.. 2>/dev/null"
            " | grep -v '+++ [ab]/' 2>/dev/null"
            " | sed 's/^+//' 2>/dev/null"
            "{}"
            " || true"
            ).format(args, extra)


def untracked_keywords():
    return ('git ls-files --others --exclude-standard 2>/dev/null'
            ' | xargs -I % {}'
            ).format(git_diff('--no-index /dev/null %'))


def uncommitted_keywords():
    return git_diff("HEAD")


def recently_committed_keywords():
    return git_diff("@'{1.hour.ago}' HEAD", "| sed '1!G;h;$!d' 2>/dev/null")

cacheable_commands = [
        untracked_keywords(),
        uncommitted_keywords(),
        recently_committed_keywords(),
        ]

# Vim helpers
def vim_str(text):
    return "'%s'" % text.replace("'", "''")


def echom(text):
    vim.command("echom %s" % vim_str(text))
{'fraction_non_alphanumeric': 0.13981308411214954, 'fraction_numerical': 0.003925233644859813, 'mean_word_length': 2.40828025477707, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1040506', 'n_tokens_mistral': 1752, 'n_tokens_neox': 1656, 'n_words': 386}