<commit_before>﻿[<AutoOpen>]
/// Contains methods for IO.
module Paket.Utils

open System
open System.IO
open System.Net
open System.Xml
open System.Text
open Paket
open Paket.Logging
open Paket.Constants
open Chessie.ErrorHandling
open Paket.Domain
open FSharp.Polyfill

open System.Net.Http
open System.Threading
open Microsoft.FSharp.Core.Printf
open System.Threading.Tasks
open System.Collections.Concurrent

#if !NETSTANDARD1_6
// TODO: Activate this in .NETCore 2.0
ServicePointManager.SecurityProtocol <- unbox 192 ||| unbox 768 ||| unbox 3072 ||| unbox 48
                                        ///SecurityProtocolType.Tls ||| SecurityProtocolType.Tls11 ||| SecurityProtocolType.Tls12 ||| SecurityProtocolType.Ssl3
#endif

let sndOf3 (_,v,_) = v
let thirdOf3 (_,_,v) = v

let rethrowf f inner fmt =
    ksprintf (fun msg -> raise (f(msg,inner))) fmt

/// Adds quotes around the string
/// [omit]
let quote (str:string) = "\"" + str.Replace("\"","\\\"") + "\""


let inline isNotNull x = not (isNull x)

let acceptXml = "application/atom+xml,application/xml"
let acceptJson = "application/atom+json,application/json"

let notNullOrEmpty = not << System.String.IsNullOrEmpty

let inline force (lz: 'a Lazy)  = lz.Force()
let inline endsWith text x = (^a:(member EndsWith:string->bool)x, text) 
let inline toLower str = (^a:(member ToLower:unit->string)str)


let inline tryGet (key:^k) this =
    let mutable v = Unchecked.defaultof<'v>
    let scc = ( ^a : (member TryGetValue : 'k * ('v byref) -> bool) this, key, &v)
    if scc then Some v else None

let internal removeInvalidChars (str : string) = RegularExpressions.Regex.Replace(str, "[:@\,]", "_")

let inline internal memoizeByExt (getKey : 'a -> 'key) (f: 'a -> 'b) : ('a -> 'b) * ('key * 'b -> unit) =
    let cache = System.Collections.Concurrent.ConcurrentDictionary<'key, 'b>()
    (fun (x: 'a) ->
        cache.GetOrAdd(getKey x, fun _ -> f x)),
    (fun (key, c) ->
        cache.TryAdd(key, c) |> ignore)

let inline internal memoizeBy (getKey : 'a -> 'key) (f: 'a -> 'b) : ('a -> 'b) =
    memoizeByExt getKey f |> fst

let inline internal memoize (f: 'a -> 'b) : 'a -> 'b = memoizeBy id f

type MemoizeAsyncExResult<'TResult, 'TCached> =
    | FirstCall of ( 'TCached * 'TResult ) Task
    | SubsequentCall of 'TCached Task

let internal memoizeAsyncEx (f: 'iext -> 'i -> Async<'o * 'oext>) =
    let cache = ConcurrentDictionary<'i, Task<'o>>()
    let handle (ex:'iext) (x:'i) : MemoizeAsyncExResult<'oext, 'o> =<commit_msg>cleanup<commit_after>﻿[<AutoOpen>]
/// Contains methods for IO.
module Paket.Utils

open System
open System.IO
open System.Net
open System.Xml
open System.Text
open Paket
open Paket.Logging
open Paket.Constants
open Chessie.ErrorHandling
open Paket.Domain
open FSharp.Polyfill

open System.Net.Http
open System.Threading
open Microsoft.FSharp.Core.Printf
open System.Threading.Tasks
open System.Collections.Concurrent

#if !NETSTANDARD1_6
// TODO: Activate this in .NETCore 2.0
ServicePointManager.SecurityProtocol <- unbox 192 ||| unbox 768 ||| unbox 3072 ||| unbox 48
                                        ///SecurityProtocolType.Tls ||| SecurityProtocolType.Tls11 ||| SecurityProtocolType.Tls12 ||| SecurityProtocolType.Ssl3
#endif

let sndOf3 (_,v,_) = v
let thirdOf3 (_,_,v) = v

let rethrowf f inner fmt =
    ksprintf (fun msg -> raise (f(msg,inner))) fmt

/// Adds quotes around the string
/// [omit]
let quote (str:string) = "\"" + str.Replace("\"","\\\"") + "\""


let inline isNotNull x = not (isNull x)

let acceptXml = "application/atom+xml,application/xml"
let acceptJson = "application/atom+json,application/json"

let notNullOrEmpty = not << System.String.IsNullOrEmpty

let inline force (lz: 'a Lazy)  = lz.Force()
let inline endsWith text x = (^a:(member EndsWith:string->bool)x, text) 
let inline toLower str = (^a:(member ToLower:unit->string)str)

let internal removeInvalidChars (str : string) = RegularExpressions.Regex.Replace(str, "[:@\,]", "_")


let inline tryGet (key:^k) this =
    let mutable v = Unchecked.defaultof<'v>
    let scc = ( ^a : (member TryGetValue : 'k * ('v byref) -> bool) this, key, &v)
    if scc then Some v else None

let inline internal memoizeByExt (getKey : 'a -> 'key) (f: 'a -> 'b) : ('a -> 'b) * ('key * 'b -> unit) =
    let cache = System.Collections.Concurrent.ConcurrentDictionary<'key, 'b>()
    (fun (x: 'a) ->
        cache.GetOrAdd(getKey x, fun _ -> f x)),
    (fun (key, c) ->
        cache.TryAdd(key, c) |> ignore)

let inline internal memoizeBy (getKey : 'a -> 'key) (f: 'a -> 'b) : ('a -> 'b) =
    memoizeByExt getKey f |> fst

let inline internal memoize (f: 'a -> 'b) : 'a -> 'b = memoizeBy id f

type MemoizeAsyncExResult<'TResult, 'TCached> =
    | FirstCall of ( 'TCached * 'TResult ) Task
    | SubsequentCall of 'TCached Task

let internal memoizeAsyncEx (f: 'iext -> 'i -> Async<'o * 'oext>) =
    let cache = ConcurrentDictionary<'i, Task<'o>>()
    let handle (ex:'iext) (x:'i) : MemoizeAsyncExResult<'oext, 'o> =
{'fraction_non_alphanumeric': 0.15326130452180872, 'fraction_numerical': 0.009203681472589036, 'mean_word_length': 3.6244218316373726, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4914359', 'n_tokens_mistral': 1759, 'n_tokens_neox': 1738, 'n_words': 506}