<commit_before><#
TODO
 - add a github release autoupdate type
 - tests (single arch, without hashes etc.)
 - clean up
#>
. "$psscriptroot\..\lib\json.ps1"

function substitute([String] $str, [Hashtable] $params) {
    $params.GetEnumerator() | % {
        $str = $str.Replace($_.Name, $_.Value)
    }

    return $str
}

function check_url([String] $url) {
    if ($url.Contains("github.com") -or
        $url.Contains("nuget.org") -or
        $url.Contains("chocolatey.org") -or
        $url.Contains("bitbucket.org")) {
        # github does not allow HEAD requests
        warn "Unable to check github/nuget/chocolatey/bitbucket url (assuming it is ok)"
        return $true
    }

    try {
        $response = Invoke-WebRequest -Uri $url -Method HEAD
        return ($response -and $response.StatusCode.Equals(200)) # redirects might be ok
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
    }

    return $false
}

function find_hash_in_rdf([String] $url, [String] $filename) {
    Write-Host -f DarkYellow "RDF URL: $url"
    Write-Host -f DarkYellow "File: $filename"

    $data = ""
    try {
        # Download and parse RDF XML file
        [xml]$data = (new-object net.webclient).downloadstring($url)
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return $null
    }

    # Find file content
    $digest = $data.RDF.Content | ? { [String]$_.about -eq $filename }

    return $digest.sha256
}

function find_hash_in_textfile([String] $url, [String] $basename, [String] $type, [String] $regex) {
    $hashfile = $null

    try {
        $hashfile = (new-object net.webclient).downloadstring($url)
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return
    }

    if ($regex -eq $null) {
        $regex = "([a-z0-9]+)"
    }
    $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}

    if ($hashfile -match $regex) {
        $hash = $matches[1]

        if ($type -and !($type -eq "sha256")) {
            $hash = $type + ":$hash"
        }

        return $hash
    }
}

function find_hash_in_json([String] $url, [String] $basename, [String] $jsonpath) {
    $json = $null

    try {
        $json = (new-object net.webclient).downloadstring($url) | convertfrom-json -ea stop
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return
    }

    return json_path $json $jsonpath $basename
}

function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) {
    $hash = $null

    <#
    TODO implement more hashing types
    `extract` Should be able to extract from origin page source (checkver)
    `rdf` Find hash from a RDF Xml file
    `download` Last resort, download the real file and hash it
    #>
    $hashmode = $config.mode
    $basename = fname($url)

    $hashfile_url = substitute $config.url @{'$url' = $url}
    $hashfile_url = substitute $hashfile_url $substitutions<commit_msg>autoupdate extract hash with original filename<commit_after><#
TODO
 - add a github release autoupdate type
 - tests (single arch, without hashes etc.)
 - clean up
#>
. "$psscriptroot\..\lib\json.ps1"

function substitute([String] $str, [Hashtable] $params) {
    $params.GetEnumerator() | % {
        $str = $str.Replace($_.Name, $_.Value)
    }

    return $str
}

function check_url([String] $url) {
    if ($url.Contains("github.com") -or
        $url.Contains("nuget.org") -or
        $url.Contains("chocolatey.org") -or
        $url.Contains("bitbucket.org")) {
        # github does not allow HEAD requests
        warn "Unable to check github/nuget/chocolatey/bitbucket url (assuming it is ok)"
        return $true
    }

    try {
        $response = Invoke-WebRequest -Uri $url -Method HEAD
        return ($response -and $response.StatusCode.Equals(200)) # redirects might be ok
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
    }

    return $false
}

function find_hash_in_rdf([String] $url, [String] $filename) {
    Write-Host -f DarkYellow "RDF URL: $url"
    Write-Host -f DarkYellow "File: $filename"

    $data = ""
    try {
        # Download and parse RDF XML file
        [xml]$data = (new-object net.webclient).downloadstring($url)
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return $null
    }

    # Find file content
    $digest = $data.RDF.Content | ? { [String]$_.about -eq $filename }

    return $digest.sha256
}

function find_hash_in_textfile([String] $url, [String] $basename, [String] $type, [String] $regex) {
    $hashfile = $null

    try {
        $hashfile = (new-object net.webclient).downloadstring($url)
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return
    }

    if ($regex -eq $null) {
        $regex = "([a-z0-9]+)"
    }
    $regex = substitute $regex @{'$basename' = [regex]::Escape($basename)}

    if ($hashfile -match $regex) {
        $hash = $matches[1]

        if ($type -and !($type -eq "sha256")) {
            $hash = $type + ":$hash"
        }

        return $hash
    }
}

function find_hash_in_json([String] $url, [String] $basename, [String] $jsonpath) {
    $json = $null

    try {
        $json = (new-object net.webclient).downloadstring($url) | convertfrom-json -ea stop
    } catch [system.net.webexception] {
        write-host -f darkred $_
        write-host -f darkred "URL $url is not valid"
        return
    }

    return json_path $json $jsonpath $basename
}

function get_hash_for_app([String] $app, $config, [String] $version, [String] $url, [Hashtable] $substitutions) {
    $hash = $null

    <#
    TODO implement more hashing types
    `extract` Should be able to extract from origin page source (checkver)
    `rdf` Find hash from a RDF Xml file
    `download` Last resort, download the real file and hash it
    #>
    $hashmode = $config.mode
    if ($url.Contains("#")) {
        <#
        The download url can end with a hash to specify the local download name.
        We need to original filename to extract the hash from the file
        Example: julia.json
        #>
        $basename = fname($url.Substring(0, $url.IndexOf("#")))
    } else {
        $basename = fname($url)
    }

    $hashfile_url = substitute $config.url @{'$url' = $url}
    $hashfile_url = substitute $hashfile_url $substitutions
{'fraction_non_alphanumeric': 0.13581340419250074, 'fraction_numerical': 0.003985828166519044, 'mean_word_length': 2.5471204188481678, 'pattern_counts': {'":': 2, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4395719', 'n_tokens_mistral': 2249, 'n_tokens_neox': 2188, 'n_words': 618}