<commit_before>﻿[<AutoOpen>]
/// Contains methods for IO.
module Paket.Utils

open System
open System.IO
open System.Net

/// [omit]
let monitor = new Object()

/// [omit]
let trace (s : string) = lock monitor (fun () -> printfn "%s" s)

/// [omit]
let tracefn fmt = Printf.ksprintf trace fmt


let writeText toStdErr color newLine text = 
    
    
    if toStdErr then 
        if newLine then eprintfn "%s" text
        else eprintf "%s" text
    else if newLine then printfn "%s" text
    else printf "%s" text

/// [omit]
let traceError (s : string) = 
    lock monitor 
        (fun () ->
            let color = ConsoleColor.Red
            let curColor = Console.ForegroundColor
            if curColor <> color then Console.ForegroundColor <- color
            printfn "%s" s
            if curColor <> color then Console.ForegroundColor <- curColor)

/// [omit]
let traceErrorfn fmt = Printf.ksprintf traceError fmt

/// Creates a directory if it does not exist.
let CreateDir path = 
    let dir = DirectoryInfo path
    if not dir.Exists then dir.Create()

/// Cleans a directory by removing all files and sub-directories.
let CleanDir path = 
    let di = DirectoryInfo path
    if di.Exists then 
        // delete all files
        let files = Directory.GetFiles(path, "*.*", SearchOption.AllDirectories)
        files |> Seq.iter (fun file -> 
                     let fi = FileInfo file
                     fi.IsReadOnly <- false
                     fi.Delete())
        // deletes all subdirectories
        let rec deleteDirs actDir = 
            Directory.GetDirectories(actDir) |> Seq.iter deleteDirs
            Directory.Delete(actDir, true)
        Directory.GetDirectories path |> Seq.iter deleteDirs
    else CreateDir path
    // set writeable
    File.SetAttributes(path, FileAttributes.Normal)

/// [omit]
let getFromUrl (url : string) = 
    async { 
        use client = new WebClient()
        try 
            return! client.AsyncDownloadString(Uri(url))
        with exn -> 
            // TODO: Handle HTTP 404 errors gracefully and return an empty string to indicate there is no content.
            return ""
    }<commit_msg>Do not silently catch HTTP error<commit_after>﻿[<AutoOpen>]
/// Contains methods for IO.
module Paket.Utils

open System
open System.IO
open System.Net

/// [omit]
let monitor = new Object()

/// [omit]
let trace (s : string) = lock monitor (fun () -> printfn "%s" s)

/// [omit]
let tracefn fmt = Printf.ksprintf trace fmt


let writeText toStdErr color newLine text = 
    
    
    if toStdErr then 
        if newLine then eprintfn "%s" text
        else eprintf "%s" text
    else if newLine then printfn "%s" text
    else printf "%s" text

/// [omit]
let traceError (s : string) = 
    lock monitor 
        (fun () ->
            let color = ConsoleColor.Red
            let curColor = Console.ForegroundColor
            if curColor <> color then Console.ForegroundColor <- color
            printfn "%s" s
            if curColor <> color then Console.ForegroundColor <- curColor)

/// [omit]
let traceErrorfn fmt = Printf.ksprintf traceError fmt

/// Creates a directory if it does not exist.
let CreateDir path = 
    let dir = DirectoryInfo path
    if not dir.Exists then dir.Create()

/// Cleans a directory by removing all files and sub-directories.
let CleanDir path = 
    let di = DirectoryInfo path
    if di.Exists then 
        // delete all files
        let files = Directory.GetFiles(path, "*.*", SearchOption.AllDirectories)
        files |> Seq.iter (fun file -> 
                     let fi = FileInfo file
                     fi.IsReadOnly <- false
                     fi.Delete())
        // deletes all subdirectories
        let rec deleteDirs actDir = 
            Directory.GetDirectories(actDir) |> Seq.iter deleteDirs
            Directory.Delete(actDir, true)
        Directory.GetDirectories path |> Seq.iter deleteDirs
    else CreateDir path
    // set writeable
    File.SetAttributes(path, FileAttributes.Normal)

/// [omit]
let getFromUrl (url : string) = 
    async { 
        use client = new WebClient()
        return! client.AsyncDownloadString(Uri(url))        
    }
{'fraction_non_alphanumeric': 0.09273422562141491, 'fraction_numerical': 0.0007170172084130019, 'mean_word_length': 2.109212481426449, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6467390', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1153, 'n_words': 414}