<commit_before>﻿/// Contains NuGet support.
module Paket.NuGetV2

open System
open System.IO
open System.Net
open Newtonsoft.Json
open System.IO.Compression
open System.Xml
open System.Text.RegularExpressions
open Paket.Logging
open System.Text

open Paket.Domain
open Paket.NuGet
open Paket.Utils
open Paket.Xml
open Paket.PackageSources
open Paket.Requirements
open FSharp.Polyfill

let rec private followODataLink auth url =
    async {
        let! raw = getFromUrl(auth, url, acceptXml)
        if String.IsNullOrWhiteSpace raw then return [||] else
        let doc = XmlDocument()
        doc.LoadXml raw
        let feed =
            match doc |> getNode "feed" with
            | Some node -> node
            | None -> failwithf "unable to parse data from %s" url

        let readEntryVersion = Some
                               >> optGetNode "properties"
                               >> optGetNode "Version"
                               >> Option.map (fun node -> node.InnerText)

        let entriesVersions = feed |> getNodes "entry" |> List.choose readEntryVersion

        let! linksVersions =
            feed
            |> getNodes "link"
            |> List.filter (fun node -> node |> getAttribute "rel" = Some "next")
            |> List.choose (getAttribute "href")
            |> List.map (followODataLink auth)
            |> Async.Parallel

        return
            linksVersions
            |> Seq.collect id
            |> Seq.append entriesVersions
            |> Seq.toArray
    }


let tryGetAllVersionsFromNugetODataWithFilter (auth, nugetURL, package:PackageName) =
    async {
        try
            let url = sprintf "%s/Packages?$filter=tolower(Id) eq '%s'" nugetURL (package.GetCompareString())
            verbosefn "getAllVersionsFromNugetODataWithFilter from url '%s'" url
            let! result = followODataLink auth url
            return Some result
        with _ -> return None
    }

let tryGetAllVersionsFromNugetODataFindById (auth, nugetURL, package:PackageName) =
    async {
        try
            let url = sprintf "%s/FindPackagesById()?id='%O'" nugetURL package
            verbosefn "getAllVersionsFromNugetODataFindById from url '%s'" url
            let! result = followODataLink auth url
            return Some result
        with _ -> return None
    }
<commit_msg>avoid endlessly querying the same url<commit_after>﻿/// Contains NuGet support.
module Paket.NuGetV2

open System
open System.IO
open System.Net
open Newtonsoft.Json
open System.IO.Compression
open System.Xml
open System.Text.RegularExpressions
open Paket.Logging
open System.Text

open Paket.Domain
open Paket.NuGet
open Paket.Utils
open Paket.Xml
open Paket.PackageSources
open Paket.Requirements
open FSharp.Polyfill

let rec private followODataLink auth url =
    async {
        let! raw = getFromUrl(auth, url, acceptXml)
        if String.IsNullOrWhiteSpace raw then return [||] else
        let doc = XmlDocument()
        doc.LoadXml raw
        let feed =
            match doc |> getNode "feed" with
            | Some node -> node
            | None -> failwithf "unable to parse data from %s" url

        let readEntryVersion = Some
                               >> optGetNode "properties"
                               >> optGetNode "Version"
                               >> Option.map (fun node -> node.InnerText)

        let entriesVersions = feed |> getNodes "entry" |> List.choose readEntryVersion

        let! linksVersions =
            feed
            |> getNodes "link"
            |> List.filter (fun node -> node |> getAttribute "rel" = Some "next")
            |> List.choose (getAttribute "href")
            |> List.filter (fun x -> x <> url)
            |> List.map (followODataLink auth)
            |> Async.Parallel

        return
            linksVersions
            |> Seq.collect id
            |> Seq.append entriesVersions
            |> Seq.toArray
    }


let tryGetAllVersionsFromNugetODataWithFilter (auth, nugetURL, package:PackageName) =
    async {
        try
            let url = sprintf "%s/Packages?$filter=tolower(Id) eq '%s'" nugetURL (package.GetCompareString())
            verbosefn "getAllVersionsFromNugetODataWithFilter from url '%s'" url
            let! result = followODataLink auth url
            return Some result
        with _ -> return None
    }

let tryGetAllVersionsFromNugetODataFindById (auth, nugetURL, package:PackageName) =
    async {
        try
            let url = sprintf "%s/FindPackagesById()?id='%O'" nugetURL package
            verbosefn "getAllVersionsFromNugetODataFindById from url '%s'" url
            let! result = followODataLink auth url
            return Some result
        with _ -> return None
    }

{'fraction_non_alphanumeric': 0.08240252897787144, 'fraction_numerical': 0.00042149631190727084, 'mean_word_length': 2.069857697283312, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5738291', 'n_tokens_mistral': 1349, 'n_tokens_neox': 1303, 'n_words': 384}