<commit_before>import { FileUtils, StandardizedFilePath, TransactionalFileSystem } from "../fileSystem";
import { FileSystemEntries, getFileMatcherPatterns, matchFiles } from "../typescript";

export function readDirectory(
    fileSystemWrapper: TransactionalFileSystem,
    useCaseSensitiveFileNames: boolean,
    rootDir: string,
    extensions: ReadonlyArray<string>,
    excludes: ReadonlyArray<string> | undefined,
    includes: ReadonlyArray<string>,
    depth?: number,
) {
    const currentDir = fileSystemWrapper.getCurrentDirectory();
    const directories: StandardizedFilePath[] = [];

    // start: code from compiler api
    const regexFlag = useCaseSensitiveFileNames ? "" : "i";
    const patterns = getFileMatcherPatterns(rootDir, excludes || [], includes, useCaseSensitiveFileNames, currentDir);
    const includeDirectoryRegex = patterns.includeDirectoryPattern && new RegExp(patterns.includeDirectoryPattern, regexFlag);
    const excludeRegex = patterns.excludePattern && new RegExp(patterns.excludePattern, regexFlag);
    // end

    return {
        files: matchFiles(
            rootDir,
            extensions,
            excludes || [],
            includes,
            useCaseSensitiveFileNames,
            currentDir,
            undefined,
            path => {
                const includeDir = dirPathMatches(path);
                const standardizedPath = fileSystemWrapper.getStandardizedAbsolutePath(path);
                if (includeDir)
                    directories.push(standardizedPath);
                return getFileSystemEntries(standardizedPath, fileSystemWrapper);
            },
            path => fileSystemWrapper.realpathSync(fileSystemWrapper.getStandardizedAbsolutePath(path)),
        ),
        directories,
    };

    function dirPathMatches(absoluteName: string) {
        // needed for the regex to match
        if (absoluteName[absoluteName.length - 1] !== "/")
            absoluteName += "/";

        // condition is from compiler api
        return (!includeDirectoryRegex || includeDirectoryRegex.test(absoluteName))
            && (!excludeRegex || !excludeRegex.test(absoluteName));
    }
}
<commit_msg>chore: readDirectory - Ensure the depth is passed down.<commit_after>import { FileUtils, StandardizedFilePath, TransactionalFileSystem } from "../fileSystem";
import { FileSystemEntries, getFileMatcherPatterns, matchFiles } from "../typescript";

export function readDirectory(
    fileSystemWrapper: TransactionalFileSystem,
    useCaseSensitiveFileNames: boolean,
    rootDir: string,
    extensions: ReadonlyArray<string>,
    excludes: ReadonlyArray<string> | undefined,
    includes: ReadonlyArray<string>,
    depth?: number,
) {
    const currentDir = fileSystemWrapper.getCurrentDirectory();
    const directories: StandardizedFilePath[] = [];

    // start: code from TypeScript compiler source
    const regexFlag = useCaseSensitiveFileNames ? "" : "i";
    const patterns = getFileMatcherPatterns(rootDir, excludes || [], includes, useCaseSensitiveFileNames, currentDir);
    const includeDirectoryRegex = patterns.includeDirectoryPattern && new RegExp(patterns.includeDirectoryPattern, regexFlag);
    const excludeRegex = patterns.excludePattern && new RegExp(patterns.excludePattern, regexFlag);
    // end

    return {
        files: matchFiles(
            rootDir,
            extensions,
            excludes || [],
            includes,
            useCaseSensitiveFileNames,
            currentDir,
            depth,
            path => {
                const includeDir = dirPathMatches(path);
                const standardizedPath = fileSystemWrapper.getStandardizedAbsolutePath(path);
                if (includeDir)
                    directories.push(standardizedPath);
                return getFileSystemEntries(standardizedPath, fileSystemWrapper);
            },
            path => fileSystemWrapper.realpathSync(fileSystemWrapper.getStandardizedAbsolutePath(path)),
        ),
        directories,
    };

    function dirPathMatches(absoluteName: string) {
        // needed for the regex to match
        if (absoluteName[absoluteName.length - 1] !== "/")
            absoluteName += "/";

        // condition is from TypeScript compiler source
        return (!includeDirectoryRegex || includeDirectoryRegex.test(absoluteName))
            && (!excludeRegex || !excludeRegex.test(absoluteName));
    }
}

{'fraction_non_alphanumeric': 0.09249208502939846, 'fraction_numerical': 0.0004522840343735866, 'mean_word_length': 2.9385574354407837, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4617661', 'n_tokens_mistral': 1074, 'n_tokens_neox': 1042, 'n_words': 237}