<commit_before>"""
Actions with musical files
"""

from functools import update_wrapper
from threading import Thread
try:
    # python3
    from queue import Queue
except ImportError:
    # python2
    from Queue import Queue
import click
import lyricstagger.log as log
import lyricstagger.misc as misc

class ActionThread(Thread):
    """Thread to perform action"""
    def __init__(self, logger, action, file_queue, result_queue=None):
        Thread.__init__(self)
        self.action = action
        self.logger = logger
        self.file_queue = file_queue
        self.result_queue = result_queue

    def run(self):
        while True:
            filepath = self.file_queue.get()
            self.logger.log_processing(filepath)
            self.action(self.logger, filepath)
            self.file_queue.task_done()
            if self.result_queue:
                self.result_queue.put(filepath)

def _massive_action_with_progress(logger, path_list, action, threads, label=""):
    """Run action function in threads for each file in path_list with progressbar"""
    file_queue = Queue()
    result_queue = Queue()
    for _ in range(threads):
        trd = ActionThread(logger, action, file_queue, result_queue)
        trd.setDaemon(True)
        trd.start()

    # get length of generator without converting it to list,
    # saves memory on large file lists, but can be a bit slower
    max_files = sum(1 for _ in misc.get_file_list(path_list))
    with click.progressbar(label=label,
                           length=max_files) as progressbar:
        for filepath in misc.get_file_list(path_list):
            file_queue.put(filepath)

        for progress in range(0, max_files):
            result_queue.get()
            progressbar.update(progress)

        file_queue.join()


def _massive_action(logger, path_list, action, threads, label=""):
    """Run action function in threads for each file in path_list"""
    file_queue = Queue()
    for _ in range(threads):
        trd = ActionThread(logger, action, file_queue)
        trd.setDaemon(True)
        trd.start()
    if label:
        click.echo(label)
    for filepath in misc.get_file_list(path_list):
        file_queue.put(filepath)
    file_queue.join()


def massive_action(logger, path_list, action, threads=1, progress=False, label=""):<commit_msg>Fixed progress bar division exception<commit_after>"""
Actions with musical files
"""

from functools import update_wrapper
from threading import Thread
try:
    # python3
    from queue import Queue
except ImportError:
    # python2
    from Queue import Queue
import click
import lyricstagger.log as log
import lyricstagger.misc as misc

class ActionThread(Thread):
    """Thread to perform action"""
    def __init__(self, logger, action, file_queue, result_queue=None):
        Thread.__init__(self)
        self.action = action
        self.logger = logger
        self.file_queue = file_queue
        self.result_queue = result_queue

    def run(self):
        while True:
            filepath = self.file_queue.get()
            self.logger.log_processing(filepath)
            self.action(self.logger, filepath)
            self.file_queue.task_done()
            if self.result_queue:
                self.result_queue.put(filepath)

def _massive_action_with_progress(logger, path_list, action, threads, label=""):
    """Run action function in threads for each file in path_list with progressbar"""
    file_queue = Queue()
    result_queue = Queue()
    for _ in range(threads):
        trd = ActionThread(logger, action, file_queue, result_queue)
        trd.setDaemon(True)
        trd.start()

    # get length of generator without converting it to list,
    # saves memory on large file lists, but can be a bit slower
    max_files = sum(1 for _ in misc.get_file_list(path_list))
    with click.progressbar(label=label,
                           length=max_files) as progressbar:
        for filepath in misc.get_file_list(path_list):
            file_queue.put(filepath)

        for progress in range(1, max_files):
            result_queue.get()
            progressbar.update(progress)

        file_queue.join()


def _massive_action(logger, path_list, action, threads, label=""):
    """Run action function in threads for each file in path_list"""
    file_queue = Queue()
    for _ in range(threads):
        trd = ActionThread(logger, action, file_queue)
        trd.setDaemon(True)
        trd.start()
    if label:
        click.echo(label)
    for filepath in misc.get_file_list(path_list):
        file_queue.put(filepath)
    file_queue.join()


def massive_action(logger, path_list, action, threads=1, progress=False, label=""):
{'fraction_non_alphanumeric': 0.08734554750745632, 'fraction_numerical': 0.0021303792074989347, 'mean_word_length': 2.83265306122449, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2723699', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1356, 'n_words': 381}