<commit_before>"""
cargo/io.py

Operations associated with the filesystem.

@author: <NAME> <<EMAIL>>
"""

import os
import os.path
import errno
import fnmatch
import hashlib
import threading
import mimetypes
import subprocess

from bz2 import BZ2File
from gzip import GzipFile
from cargo.errors import print_ignored_error

def files_under(path, pattern = "*"):
    """
    Iterate over the set of paths to files in the specified directory tree.

    @param path: Specified path.
    @param pattern: Optional filter pattern.
    """

    for (dirpath, dirnames, filenames) in os.walk(path):
        for filename in filenames:
            if fnmatch.fnmatch(filename, pattern):
                yield os.path.join(dirpath, filename)

def bq(arguments, cwd = None):
    """
    Spawn a process and return its output.<commit_msg>Support multiple patterns in files_under.<commit_after>"""
cargo/io.py

Operations associated with the filesystem.

@author: <NAME> <<EMAIL>>
"""

import os
import os.path
import errno
import hashlib
import threading
import mimetypes
import subprocess

from bz2 import BZ2File
from gzip import GzipFile
from fnmatch import fnmatch
from cargo.errors import print_ignored_error

def files_under(path, pattern = "*"):
    """
    Iterate over the set of paths to files in the specified directory tree.

    @param path: Specified path.
    @param pattern: Optional filter pattern(s).
    """

    if isinstance(pattern, str):
        pattern = (str,)

    for (dirpath, dirnames, filenames) in os.walk(path):
        for filename in filenames:
            if any(fnmatch(filename, p) for p in pattern):
                yield os.path.join(dirpath, filename)

def bq(arguments, cwd = None):
    """
    Spawn a process and return its output.
{'fraction_non_alphanumeric': 0.08745031232254401, 'fraction_numerical': 0.002271436683702442, 'mean_word_length': 3.405, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321707', 'n_tokens_mistral': 527, 'n_tokens_neox': 520, 'n_words': 194}