<commit_before>/// Contains NuGet support.
module Paket.NuGetV2

open System
open System.IO
open System.Net
open Newtonsoft.Json
open Ionic.Zip
open System.Xml
open System.Text.RegularExpressions
open Paket.Logging
open System.Text

open Paket.Domain
open Paket.Utils
open Paket.Xml
open Paket.PackageSources
open Paket.NuGetV3

/// Represents type of NuGet packages.config file
type NugetPackagesConfigType = ProjectLevel | SolutionLevel

/// Represents NuGet packages.config file
type NugetPackagesConfig = {
    File: FileInfo;
    Packages: (string*SemVerInfo) list
    Type: NugetPackagesConfigType
}

type NugetPackageCache =
    { Dependencies : (string * VersionRequirement * (FrameworkIdentifier option)) list
      Name : string
      SourceUrl: string
      Unlisted : bool
      DownloadUrl : string}

let rec private followODataLink auth url = 
    async { 
        let! raw = getFromUrl (auth, url)
        let doc = XmlDocument()
        doc.LoadXml raw
        let feed = 
            match doc |> getNode "feed" with
            | Some node -> node
            | None -> failwithf "unable to parse data from %s" url

        let readEntryVersion = optGetNode "properties" >> optGetNode "Version"

        let entriesVersions = 
            feed
            |> getNodes "entry"
            |> List.choose (fun entry -> Some entry |> readEntryVersion |> Option.map (fun node -> node.InnerText))

        let linksVersions = 
            feed 
            |> getNodes "link"
            |> List.filter (fun node -> node |> getAttribute "rel" = Some "next")
            |> List.choose (getAttribute "href")
            |> List.map (followODataLink auth)
            |> Async.Parallel
            |> Async.RunSynchronously
            |> Seq.concat

        return
            entriesVersions
            |> Seq.append linksVersions
    }

/// Gets versions of the given package via OData via /Packages?$filter=Id eq 'packageId'
let getAllVersionsFromNugetODataWithFilter (auth, nugetURL, package) = 
    // we cannot cache this
    followODataLink auth (sprintf "%s/Packages?$filter=Id eq '%s'" nugetURL package)

/// Gets versions of the given package via OData via /FindPackagesById()?id='packageId'.
let getAllVersionsFromNugetOData (auth, nugetURL, package) = 
    async { 
        // we cannot cache this
        try 
            let url = sprintf "%s/FindPackagesById()?id='%s'" nugetURL package
            return! followODataLink auth url
        with _ -> return! getAllVersionsFromNugetODataWithFilter (auth, nugetURL, package)
    }

/// Gets all versions no. of the given package.
let getAllVersionsFromNuGet2(auth,nugetURL,package) = 
    // we cannot cache this
    async { 
        let! raw = safeGetFromUrl(auth,sprintf "%s/package-versions/%s?includePrerelease=true" nugetURL package)
        match raw with
        | None -> let! result = getAllVersionsFromNugetOData(auth,nugetURL, package)
                  return result
        | Some data -> 
            try 
                try 
                    let result = JsonConvert.DeserializeObject<string []>(data) |> Array.toSeq
                    return result
                with _ -> let! result = getAllVersionsFromNugetOData(auth,nugetURL, package)
                          return result
            with exn -> 
                return! failwithf "Could not get data from %s for package %s.%s Message: %s" nugetURL package 
                    Environment.NewLine exn.Message
    }


let getAllVersions(auth, nugetURL, package) = 
    async { 
        let! raw = getViaNuGet3(auth, nugetURL, package)

        match raw with
        | None -> let! result = getAllVersionsFromNuGet2(auth,nugetURL,package)
                  return result
        | Some data -> 
            try 
                try 
                    let result = getJSONLDDetails data
                    return (Array.toSeq result)
                with _ -> let! result = getAllVersionsFromNuGet2(auth,nugetURL, package)
                          return result
            with exn ->
                return! failwithf "Could not get data from %s for package %s.%s Message: %s" nugetURL package
                    Environment.NewLine exn.Message
    }

/// Gets versions of the given package from local Nuget feed.
let getAllVersionsFromLocalPath (localNugetPath, package) =<commit_msg>Refactor NuGetV2.followODataLink to take a function to get the url contents so that a stub can be injected for testing.<commit_after>/// Contains NuGet support.
module Paket.NuGetV2

open System
open System.IO
open System.Net
open Newtonsoft.Json
open Ionic.Zip
open System.Xml
open System.Text.RegularExpressions
open Paket.Logging
open System.Text

open Paket.Domain
open Paket.Utils
open Paket.Xml
open Paket.PackageSources
open Paket.NuGetV3

/// Represents type of NuGet packages.config file
type NugetPackagesConfigType = ProjectLevel | SolutionLevel

/// Represents NuGet packages.config file
type NugetPackagesConfig = {
    File: FileInfo;
    Packages: (string*SemVerInfo) list
    Type: NugetPackagesConfigType
}

type NugetPackageCache =
    { Dependencies : (string * VersionRequirement * (FrameworkIdentifier option)) list
      Name : string
      SourceUrl: string
      Unlisted : bool
      DownloadUrl : string}

let rec private followODataLink getUrlContents url = 
    async { 
        let! raw = getUrlContents url
        let doc = XmlDocument()
        doc.LoadXml raw
        let feed = 
            match doc |> getNode "feed" with
            | Some node -> node
            | None -> failwithf "unable to parse data from %s" url

        let readEntryVersion = optGetNode "properties" >> optGetNode "Version"

        let entriesVersions = 
            feed
            |> getNodes "entry"
            |> List.choose (fun entry -> Some entry |> readEntryVersion |> Option.map (fun node -> node.InnerText))

        let linksVersions = 
            feed 
            |> getNodes "link"
            |> List.filter (fun node -> node |> getAttribute "rel" = Some "next")
            |> List.choose (getAttribute "href")
            |> List.map (followODataLink getUrlContents)
            |> Async.Parallel
            |> Async.RunSynchronously
            |> Seq.concat

        return
            entriesVersions
            |> Seq.append linksVersions
    }

/// Gets versions of the given package via OData via /Packages?$filter=Id eq 'packageId'
let getAllVersionsFromNugetODataWithFilter (getUrlContents, nugetURL, package) = 
    // we cannot cache this
    let url = sprintf "%s/Packages?$filter=Id eq '%s'" nugetURL package
    followODataLink getUrlContents url

/// Gets versions of the given package via OData via /FindPackagesById()?id='packageId'.
let getAllVersionsFromNugetOData (auth, nugetURL, package) = 
    async { 
        // we cannot cache this
        let getUrlContents url = getFromUrl(auth, url)
        try 
            let url = sprintf "%s/FindPackagesById()?id='%s'" nugetURL package
            return! followODataLink getUrlContents url
        with _ -> return! getAllVersionsFromNugetODataWithFilter (getUrlContents, nugetURL, package)
    }

/// Gets all versions no. of the given package.
let getAllVersionsFromNuGet2(auth,nugetURL,package) = 
    // we cannot cache this
    async { 
        let! raw = safeGetFromUrl(auth,sprintf "%s/package-versions/%s?includePrerelease=true" nugetURL package)
        match raw with
        | None -> let! result = getAllVersionsFromNugetOData(auth, nugetURL, package)
                  return result
        | Some data -> 
            try 
                try 
                    let result = JsonConvert.DeserializeObject<string []>(data) |> Array.toSeq
                    return result
                with _ -> let! result = getAllVersionsFromNugetOData(auth, nugetURL, package)
                          return result
            with exn -> 
                return! failwithf "Could not get data from %s for package %s.%s Message: %s" nugetURL package 
                    Environment.NewLine exn.Message
    }


let getAllVersions(auth, nugetURL, package) = 
    async { 
        let! raw = getViaNuGet3(auth, nugetURL, package)

        match raw with
        | None -> let! result = getAllVersionsFromNuGet2(auth,nugetURL,package)
                  return result
        | Some data -> 
            try 
                try 
                    let result = getJSONLDDetails data
                    return (Array.toSeq result)
                with _ -> let! result = getAllVersionsFromNuGet2(auth,nugetURL, package)
                          return result
            with exn ->
                return! failwithf "Could not get data from %s for package %s.%s Message: %s" nugetURL package
                    Environment.NewLine exn.Message
    }

/// Gets versions of the given package from local Nuget feed.
let getAllVersionsFromLocalPath (localNugetPath, package) =
{'fraction_non_alphanumeric': 0.07905049826447207, 'fraction_numerical': 0.001455604075691412, 'mean_word_length': 2.177516897901103, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5598337', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2406, 'n_words': 754}