<commit_before># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import contextlib
import os
import itertools
import re
import StringIO
import site
import subprocess
import sys
import json
from licenseck import licenses

filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".lock", ".py", ".toml", ".webidl", ".json"]

ignored_files = [
    # Generated and upstream code combined with our own. Could use cleanup
    os.path.join(".", "ports", "gonk", "src", "native_window_glue.cpp"),
    os.path.join(".", "ports", "geckolib", "bindings.rs"),
    os.path.join(".", "resources", "hsts_preload.json"),
    os.path.join(".", "tests", "wpt", "metadata", "MANIFEST.json"),
    os.path.join(".", "tests", "wpt", "metadata-css", "MANIFEST.json"),
    # MIT license
    os.path.join(".", "components", "util", "deque", "mod.rs"),
    # Hidden files
    os.path.join(".", "."),
]

ignored_dirs = [
    # Upstream
    os.path.join(".", "support", "android", "apk"),
    os.path.join(".", "support", "rust-task_info"),
    os.path.join(".", "tests", "wpt", "css-tests"),
    os.path.join(".", "tests", "wpt", "harness"),
    os.path.join(".", "tests", "wpt", "update"),
    os.path.join(".", "tests", "wpt", "web-platform-tests"),
    os.path.join(".", "python", "mach"),
    os.path.join(".", "components", "script", "dom", "bindings", "codegen", "parser"),
    os.path.join(".", "components", "script", "dom", "bindings", "codegen", "ply"),
    os.path.join(".", "python", "_virtualenv"),
    # Generated and upstream code combined with our own. Could use cleanup
    os.path.join(".", "target"),
    os.path.join(".", "ports", "cef"),
    # Hidden directories
    os.path.join(".", "."),
]


# A simple wrapper for iterators to show progress (note that it's inefficient for giant iterators)
def progress_wrapper(iterator):
    list_of_stuff = list(iterator)
    total_files, progress = len(list_of_stuff), 0
    for idx, thing in enumerate(list_of_stuff):
        progress = int(float(idx + 1) / total_files * 100)
        sys.stdout.write('\r  Progress: %s%% (%d/%d)' % (progress, idx + 1, total_files))
        sys.stdout.flush()
        yield thing


def filter_files(start_dir, faster, progress):
    file_iter = get_file_list(start_dir, faster, ignored_dirs)
    if progress:
        file_iter = progress_wrapper(file_iter)
    for file_name in file_iter:
        if os.path.basename(file_name) == "Cargo.lock":<commit_msg>Auto merge of #9941 - Ms2ger:tidy-ignore, r=jdm<commit_after># Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

import contextlib
import os
import itertools
import re
import StringIO
import site
import subprocess
import sys
import json
from licenseck import licenses

filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".lock", ".py", ".toml", ".webidl", ".json"]

ignored_files = [
    # Generated and upstream code combined with our own. Could use cleanup
    os.path.join(".", "ports", "gonk", "src", "native_window_glue.cpp"),
    os.path.join(".", "ports", "geckolib", "bindings.rs"),
    os.path.join(".", "resources", "hsts_preload.json"),
    os.path.join(".", "tests", "wpt", "metadata", "MANIFEST.json"),
    os.path.join(".", "tests", "wpt", "metadata-css", "MANIFEST.json"),
    # Hidden files
    os.path.join(".", "."),
]

ignored_dirs = [
    # Upstream
    os.path.join(".", "support", "android", "apk"),
    os.path.join(".", "support", "rust-task_info"),
    os.path.join(".", "tests", "wpt", "css-tests"),
    os.path.join(".", "tests", "wpt", "harness"),
    os.path.join(".", "tests", "wpt", "update"),
    os.path.join(".", "tests", "wpt", "web-platform-tests"),
    os.path.join(".", "tests", "wpt", "sync"),
    os.path.join(".", "tests", "wpt", "sync_css"),
    os.path.join(".", "python", "mach"),
    os.path.join(".", "components", "script", "dom", "bindings", "codegen", "parser"),
    os.path.join(".", "components", "script", "dom", "bindings", "codegen", "ply"),
    os.path.join(".", "python", "_virtualenv"),
    # Generated and upstream code combined with our own. Could use cleanup
    os.path.join(".", "target"),
    os.path.join(".", "ports", "cef"),
    # Hidden directories
    os.path.join(".", "."),
]


# A simple wrapper for iterators to show progress (note that it's inefficient for giant iterators)
def progress_wrapper(iterator):
    list_of_stuff = list(iterator)
    total_files, progress = len(list_of_stuff), 0
    for idx, thing in enumerate(list_of_stuff):
        progress = int(float(idx + 1) / total_files * 100)
        sys.stdout.write('\r  Progress: %s%% (%d/%d)' % (progress, idx + 1, total_files))
        sys.stdout.flush()
        yield thing


def filter_files(start_dir, faster, progress):
    file_iter = get_file_list(start_dir, faster, ignored_dirs)
    if progress:
        file_iter = progress_wrapper(file_iter)
    for file_name in file_iter:
        if os.path.basename(file_name) == "Cargo.lock":
{'fraction_non_alphanumeric': 0.18978361120964882, 'fraction_numerical': 0.0058531394111387015, 'mean_word_length': 3.7346767422334173, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1946741', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1856, 'n_words': 517}