<commit_before>/// This module contains a file pattern globbing implementation.
[<AutoOpen>]
module Fake.FileSystem
    
open System
open System.Collections.Generic
open System.IO
open Fake
open System.Text.RegularExpressions

type private SearchOption = 
| Directory of string
| Drive of string
| Recursive
| FilePattern of string
        
let private checkSubDirs absolute (dir:string) root =
    if dir.Contains "*" then
        Directory.EnumerateDirectories(root, dir, SearchOption.TopDirectoryOnly) |> Seq.toList
    else
        let di = 
            if absolute then new DirectoryInfo(dir) else 
            new DirectoryInfo(root + directorySeparator + dir)
        if di.Exists then [di.FullName] else []
        
let rec private buildPaths acc (input : SearchOption list) =
    match input with
    | [] -> acc
    | Directory(name) :: t -> 
        let subDirs = 
            acc
            |> List.map (checkSubDirs false name) 
            |> List.concat
        buildPaths subDirs t
    | Drive(name) :: t ->
        let subDirs = 
            acc
            |> List.map (checkSubDirs true name)
            |> List.concat
        buildPaths subDirs t
    | Recursive :: [] ->
        let dirs = 
            Seq.collect (fun dir -> Directory.EnumerateFileSystemEntries(dir, "*", SearchOption.AllDirectories)) acc
            |> Seq.toList
        buildPaths (acc @ dirs) []
    | Recursive :: t ->
        let dirs = 
            Seq.collect (fun dir -> Directory.EnumerateDirectories(dir, "*", SearchOption.AllDirectories)) acc
            |> Seq.toList
        buildPaths (acc @ dirs) t
    | FilePattern(pattern) :: t ->
        Seq.collect (fun dir -> Directory.EnumerateFiles(dir, pattern)) acc
        |> Seq.toList
         
let private isDrive =
    let regex = Regex(@"^[A-Za-z]:$", RegexOptions.Compiled)<commit_msg>Trace temporary<commit_after>/// This module contains a file pattern globbing implementation.
[<AutoOpen>]
module Fake.FileSystem
    
open System
open System.Collections.Generic
open System.IO
open Fake
open System.Text.RegularExpressions

type private SearchOption = 
| Directory of string
| Drive of string
| Recursive
| FilePattern of string
        
let private checkSubDirs absolute (dir:string) root =
    if dir.Contains "*" then
        Directory.EnumerateDirectories(root, dir, SearchOption.TopDirectoryOnly) |> Seq.toList
    else
        let di = 
            if absolute then new DirectoryInfo(dir) else 
            new DirectoryInfo(root + directorySeparator + dir)
        if di.Exists then [di.FullName] else []
        
let rec private buildPaths acc (input : SearchOption list) =
    match input with
    | [] -> acc
    | Directory(name) :: t -> 
        let subDirs = 
            acc
            |> List.map (checkSubDirs false name) 
            |> List.concat
        tracefn "All subdir  %s" name
        buildPaths subDirs t
    | Drive(name) :: t ->
        let subDirs = 
            acc
            |> List.map (checkSubDirs true name)
            |> List.concat
        buildPaths subDirs t
    | Recursive :: [] ->
        let dirs = 
            Seq.collect (fun dir ->        tracefn "All subdirs and files %s" dir; Directory.EnumerateFileSystemEntries(dir, "*", SearchOption.AllDirectories)) acc
            |> Seq.toList
        buildPaths (acc @ dirs) []
    | Recursive :: t ->
 
        let dirs = 
            Seq.collect (fun dir ->       tracefn "All subdirs %s" dir; Directory.EnumerateDirectories(dir, "*", SearchOption.AllDirectories)) acc
            |> Seq.toList
        buildPaths (acc @ dirs) t
    | FilePattern(pattern) :: t ->
        tracefn "All files %s" pattern
        Seq.collect (fun dir -> Directory.EnumerateFiles(dir, pattern)) acc
        |> Seq.toList
         
let private isDrive =
    let regex = Regex(@"^[A-Za-z]:$", RegexOptions.Compiled)
{'fraction_non_alphanumeric': 0.09916926272066459, 'fraction_numerical': 0.0, 'mean_word_length': 2.1948590381426203, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '510976', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1078, 'n_words': 318}