<commit_before>﻿module Paket.RemoteDownload

open Paket
open Newtonsoft.Json.Linq
open System.IO
open Paket.Logging
open Paket.ModuleResolver
open System.IO.Compression
open Paket.Domain

// Gets the sha1 of a branch
let getSHA1OfBranch origin owner project branch = 
    async { 
        match origin with
        | ModuleResolver.SingleSourceFileOrigin.GitHubLink -> 
            let url = sprintf "https://api.github.com/repos/%s/%s/commits/%s" owner project branch
            let! document = getFromUrl(None, url, null)
            let json = JObject.Parse(document)
            return json.["sha"].ToString()
        | ModuleResolver.SingleSourceFileOrigin.GistLink ->  
            let url = sprintf "https://api.github.com/gists/%s/%s" project branch
            let! document = getFromUrl(None, url, null)
            let json = JObject.Parse(document)
            let latest = json.["history"].First.["version"]
            return latest.ToString()
        | ModuleResolver.SingleSourceFileOrigin.HttpLink _ -> return ""
    }

let private rawFileUrl owner project branch fileName =
    sprintf "https://github.com/%s/%s/raw/%s/%s" owner project branch fileName

let private rawGistFileUrl owner project fileName =
    sprintf "https://gist.githubusercontent.com/%s/%s/raw/%s" owner project fileName

/// Gets a dependencies file from the remote source and tries to parse it.
let downloadDependenciesFile(rootPath,groupName,parserF,remoteFile:ModuleResolver.ResolvedSourceFile) = async {
    let fi = FileInfo(remoteFile.Name)

    let dependenciesFileName = remoteFile.Name.Replace(fi.Name,Constants.DependenciesFileName)

    let auth, url = 
        match remoteFile.Origin with
        | ModuleResolver.GitHubLink -> 
            None, rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName
        | ModuleResolver.GistLink -> 
            None, rawGistFileUrl remoteFile.Owner remoteFile.Project dependenciesFileName
        | ModuleResolver.HttpLink url -> 
            let url = url.Replace(remoteFile.Name,Constants.DependenciesFileName)
            let auth = 
                ConfigFile.GetCredentialsForUrl remoteFile.Project url
                |> Option.map (fun (un, pwd) -> { Username = un; Password = <PASSWORD> })
            auth, url

    let! result = safeGetFromUrl(auth,url,null)

    match result with
    | Some text when parserF text ->        
        let destination = remoteFile.ComputeFilePath(rootPath,groupName,dependenciesFileName)

        Directory.CreateDirectory(destination |> Path.GetDirectoryName) |> ignore
        File.WriteAllText(destination, text)
        return text
    | _ -> return "" }


let ExtractZip(fileName : string, targetFolder) = 
    Directory.CreateDirectory(targetFolder) |> ignore
    ZipFile.ExtractToDirectory(fileName,targetFolder)

let rec DirectoryCopy(sourceDirName, destDirName, copySubDirs) =
    let dir = new DirectoryInfo(sourceDirName)
    let dirs = dir.GetDirectories()

    if not <| Directory.Exists(destDirName) then
        Directory.CreateDirectory(destDirName) |> ignore

    for file in dir.GetFiles() do
        file.CopyTo(Path.Combine(destDirName, file.Name), true) |> ignore

    // If copying subdirectories, copy them and their contents to new location. 
    if copySubDirs then
        for subdir in dirs do
            DirectoryCopy(subdir.FullName, Path.Combine(destDirName, subdir.Name), copySubDirs)

/// Gets a single file from github.
let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destination) = async {
    match remoteFile.Origin, remoteFile.Name with
    | SingleSourceFileOrigin.GistLink, Constants.FullProjectSourceFileName ->
        let fi = FileInfo(destination)
        let projectPath = fi.Directory.FullName

        let url = sprintf "https://api.github.com/gists/%s" remoteFile.Project
        let! document = getFromUrl(None, url, null)
        let json = JObject.Parse(document)
        let files = json.["files"] |> Seq.map (fun i -> i.First.["filename"].ToString(), i.First.["raw_url"].ToString())

        let task = 
            files |> Seq.map (fun (filename, url) -> 
                async {
                    let path = Path.Combine(projectPath,filename)
                    do! downloadFromUrl(None, url) path
                } 
            ) |> Async.Parallel
        task |> Async.RunSynchronously |> ignore

        // GIST currently does not support zip-packages, so now this fetches all files separately.
        // let downloadUrl = sprintf "https://gist.github.com/%s/%s/download" remoteFile.Owner remoteFile.Project //is a tar.gz
<commit_msg>Use commit w/gist download in RemoteDownload.downloadRemoteFiles<commit_after>﻿module Paket.RemoteDownload

open Paket
open Newtonsoft.Json.Linq
open System.IO
open Paket.Logging
open Paket.ModuleResolver
open System.IO.Compression
open Paket.Domain

// Gets the sha1 of a branch
let getSHA1OfBranch origin owner project branch = 
    async { 
        match origin with
        | ModuleResolver.SingleSourceFileOrigin.GitHubLink -> 
            let url = sprintf "https://api.github.com/repos/%s/%s/commits/%s" owner project branch
            let! document = getFromUrl(None, url, null)
            let json = JObject.Parse(document)
            return json.["sha"].ToString()
        | ModuleResolver.SingleSourceFileOrigin.GistLink ->  
            let url = sprintf "https://api.github.com/gists/%s/%s" project branch
            let! document = getFromUrl(None, url, null)
            let json = JObject.Parse(document)
            let latest = json.["history"].First.["version"]
            return latest.ToString()
        | ModuleResolver.SingleSourceFileOrigin.HttpLink _ -> return ""
    }

let private rawFileUrl owner project branch fileName =
    sprintf "https://github.com/%s/%s/raw/%s/%s" owner project branch fileName

let private rawGistFileUrl owner project fileName =
    sprintf "https://gist.githubusercontent.com/%s/%s/raw/%s" owner project fileName

/// Gets a dependencies file from the remote source and tries to parse it.
let downloadDependenciesFile(rootPath,groupName,parserF,remoteFile:ModuleResolver.ResolvedSourceFile) = async {
    let fi = FileInfo(remoteFile.Name)

    let dependenciesFileName = remoteFile.Name.Replace(fi.Name,Constants.DependenciesFileName)

    let auth, url = 
        match remoteFile.Origin with
        | ModuleResolver.GitHubLink -> 
            None, rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName
        | ModuleResolver.GistLink -> 
            None, rawGistFileUrl remoteFile.Owner remoteFile.Project dependenciesFileName
        | ModuleResolver.HttpLink url -> 
            let url = url.Replace(remoteFile.Name,Constants.DependenciesFileName)
            let auth = 
                ConfigFile.GetCredentialsForUrl remoteFile.Project url
                |> Option.map (fun (un, pwd) -> { Username = un; Password = <PASSWORD> })
            auth, url

    let! result = safeGetFromUrl(auth,url,null)

    match result with
    | Some text when parserF text ->        
        let destination = remoteFile.ComputeFilePath(rootPath,groupName,dependenciesFileName)

        Directory.CreateDirectory(destination |> Path.GetDirectoryName) |> ignore
        File.WriteAllText(destination, text)
        return text
    | _ -> return "" }


let ExtractZip(fileName : string, targetFolder) = 
    Directory.CreateDirectory(targetFolder) |> ignore
    ZipFile.ExtractToDirectory(fileName,targetFolder)

let rec DirectoryCopy(sourceDirName, destDirName, copySubDirs) =
    let dir = new DirectoryInfo(sourceDirName)
    let dirs = dir.GetDirectories()

    if not <| Directory.Exists(destDirName) then
        Directory.CreateDirectory(destDirName) |> ignore

    for file in dir.GetFiles() do
        file.CopyTo(Path.Combine(destDirName, file.Name), true) |> ignore

    // If copying subdirectories, copy them and their contents to new location. 
    if copySubDirs then
        for subdir in dirs do
            DirectoryCopy(subdir.FullName, Path.Combine(destDirName, subdir.Name), copySubDirs)

/// Gets a single file from github.
let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destination) = async {
    match remoteFile.Origin, remoteFile.Name with
    | SingleSourceFileOrigin.GistLink, Constants.FullProjectSourceFileName ->
        let fi = FileInfo(destination)
        let projectPath = fi.Directory.FullName

        let url = sprintf "https://api.github.com/gists/%s/%s" remoteFile.Project remoteFile.Commit
        let! document = getFromUrl(None, url, null)
        let json = JObject.Parse(document)
        let files = json.["files"] |> Seq.map (fun i -> i.First.["filename"].ToString(), i.First.["raw_url"].ToString())

        let task = 
            files |> Seq.map (fun (filename, url) -> 
                async {
                    let path = Path.Combine(projectPath,filename)
                    do! downloadFromUrl(None, url) path
                } 
            ) |> Async.Parallel
        task |> Async.RunSynchronously |> ignore

        // GIST currently does not support zip-packages, so now this fetches all files separately.
        // let downloadUrl = sprintf "https://gist.github.com/%s/%s/download" remoteFile.Owner remoteFile.Project //is a tar.gz

{'fraction_non_alphanumeric': 0.09587717414644621, 'fraction_numerical': 0.00042946102641185313, 'mean_word_length': 2.8539511791477037, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 51, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1057864', 'n_tokens_mistral': 2574, 'n_tokens_neox': 2508, 'n_words': 644}