<commit_before># Copyright 2018 <NAME>.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl).

import hashlib
import os
from fnmatch import fnmatch


def _fnmatch(filename, patterns):
    for pattern in patterns:
        if fnmatch(filename, pattern):
            return True
    return False


def _walk(top, exclude_patterns, keep_langs):
    keep_langs = {l.split("_")[0] for l in keep_langs}
    for dirpath, dirnames, filenames in os.walk(top):
        dirnames.sort()
        reldir = os.path.relpath(dirpath, top)
        if reldir == ".":
            reldir = ""
        for filename in sorted(filenames):
            filepath = os.path.join(reldir, filename)
            if _fnmatch(filepath, exclude_patterns):
                continue
            if keep_langs and reldir in {"i18n", "i18n_extra"}:
                basename, ext = os.path.splitext(filename)
                if ext == ".po":<commit_msg>[MIG] module_auto_update: Migration to 14.0<commit_after># Copyright 2018 <NAME>.
# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl).

import hashlib
import os
from fnmatch import fnmatch


def _fnmatch(filename, patterns):
    for pattern in patterns:
        if fnmatch(filename, pattern):
            return True
    return False


def _walk(top, exclude_patterns, keep_langs):
    keep_langs = {language.split("_")[0] for language in keep_langs}
    for dirpath, dirnames, filenames in os.walk(top):
        dirnames.sort()
        reldir = os.path.relpath(dirpath, top)
        if reldir == ".":
            reldir = ""
        for filename in sorted(filenames):
            filepath = os.path.join(reldir, filename)
            if _fnmatch(filepath, exclude_patterns):
                continue
            if keep_langs and reldir in {"i18n", "i18n_extra"}:
                basename, ext = os.path.splitext(filename)
                if ext == ".po":
{'fraction_non_alphanumeric': 0.10887949260042283, 'fraction_numerical': 0.013213530655391121, 'mean_word_length': 2.380357142857143, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6518027', 'n_tokens_mistral': 620, 'n_tokens_neox': 589, 'n_words': 159}