<commit_before>#!/usr/bin/env python3
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import glob
import multiprocessing
import os
import queue
import subprocess
import sys
import threading


def getIndentedPaths():
    ret = []

    for fileList in [".git/indented-files.cache", ".git/indented-files2.cache"]:
        sock = open(fileList)
        for line in sock.readlines():
            path = line.strip()
            if not path.endswith("cxx"):
                continue

            ret.append(path)

    return ret


def run_tool(assume, task_queue, failed_files):
    while True:
        invocation1, invocation2 = task_queue.get()
        if not len(failed_files):
            print(" ".join(invocation1) + " | " + " ".join(invocation2))
            p1 = subprocess.Popen(invocation1, stdout=subprocess.PIPE)
            p2 = subprocess.Popen(invocation2, stdin=p1.stdout, stdout=subprocess.PIPE)
            p1.stdout.close()
            sys.stdout.write(p2.communicate()[0].decode('utf-8'))
            if p2.returncode != 0:
                failed_files.append(invocation1)
        task_queue.task_done()


def tidy(paths, assume=None):
    return_code = 0
    try:
        max_task = multiprocessing.cpu_count()
        task_queue = queue.Queue(max_task)
        failed_files = []
        for _ in range(max_task):
            t = threading.Thread(target=run_tool, args=(assume, task_queue, failed_files))
            t.daemon = True
            t.start()

        for path in sorted(paths):
            iwyu_tool = os.path.join(os.environ["HOME"], "git/include-what-you-use/iwyu_tool.py")
            find_unneeded_includes = os.path.join(os.environ["HOME"], "git/vmexam/libreoffice/find-unneeded-includes")
<commit_msg>make this a bit more portable<commit_after>#!/usr/bin/env python3
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import glob
import multiprocessing
import os
import queue
import subprocess
import sys
import threading


def getIndentedPaths():
    ret = []

    for fileList in [".git/indented-files.cache", ".git/indented-files2.cache"]:
        sock = open(fileList)
        for line in sock.readlines():
            path = line.strip()
            if not path.endswith("cxx"):
                continue

            ret.append(path)

    return ret


def run_tool(assume, task_queue, failed_files):
    while True:
        invocation1, invocation2 = task_queue.get()
        if not len(failed_files):
            print(" ".join(invocation1) + " | " + " ".join(invocation2))
            p1 = subprocess.Popen(invocation1, stdout=subprocess.PIPE)
            p2 = subprocess.Popen(invocation2, stdin=p1.stdout, stdout=subprocess.PIPE)
            p1.stdout.close()
            sys.stdout.write(p2.communicate()[0].decode('utf-8'))
            if p2.returncode != 0:
                failed_files.append(invocation1)
        task_queue.task_done()


def tidy(paths, assume=None):
    return_code = 0
    try:
        max_task = multiprocessing.cpu_count()
        task_queue = queue.Queue(max_task)
        failed_files = []
        for _ in range(max_task):
            t = threading.Thread(target=run_tool, args=(assume, task_queue, failed_files))
            t.daemon = True
            t.start()

        for path in sorted(paths):
            iwyu_tool = os.path.join(os.environ["HOME"], "git/include-what-you-use/iwyu_tool.py")
            find_unneeded_includes = os.path.join(os.path.dirname(__file__), "find-unneeded-includes")

{'fraction_non_alphanumeric': 0.11686938127974617, 'fraction_numerical': 0.012162876784769964, 'mean_word_length': 2.4642857142857144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '92840', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1189, 'n_words': 273}