<commit_before>struct DownloadsBackend <: AWS.AbstractBackend
    downloader::Union{Nothing, Downloads.Downloader}
end

DownloadsBackend() = DownloadsBackend(nothing)

const AWS_DOWNLOADER = Ref{Union{Nothing, Downloader}}(nothing)
const AWS_DOWNLOAD_LOCK = ReentrantLock()

# Here we mimic Download.jl's own setup for using a global downloader.
# We do this to have our own downloader (separate from Downloads.jl's global downloader)
# because we add a hook to avoid redirects in order to try to match the HTTPBackend's
# implementation, and we don't want to mutate the global downloader from Downloads.jl.
# https://github.com/JuliaLang/Downloads.jl/blob/84e948c02b8a0625552a764bf90f7d2ee97c949c/src/Downloads.jl#L293-L301
function get_downloader(downloader=nothing)
    lock(AWS_DOWNLOAD_LOCK) do
        yield() # let other downloads finish
        downloader isa Downloader && return
        while true
            downloader = AWS_DOWNLOADER[]
            downloader isa Downloader && return
            D = Downloader()
            D.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false)
            AWS_DOWNLOADER[] = D
        end
    end
    return downloader
end


function AWS._http_request(backend::DownloadsBackend, request)
    # If we pass `output`, Downloads.jl will expect a message
    # body in the response. Specifically, it sets
    # <https://curl.se/libcurl/c/CURLOPT_NOBODY.html>
    # only when we do not pass the `output` argument.
    #
    # When the method is `HEAD`, the response may have a Content-Length
    # but not send any content back (which appears to be correct,
    # <https://stackoverflow.com/a/18925736/12486544>).
    #
    # Thus, if we did not set `CURLOPT_NOBODY`, and it gets a Content-Length
    # back, it will hang waiting for that body.
    #
    # Therefore, we do not pass an `output` when the `request_method` is `HEAD`.
    if request.request_method != "HEAD"
        output = IOBuffer()
        output_arg = (; output=output)

        # We set a callback so later on we know how to get the `body` back.
        body_arg = () -> (; body = take!(output))
    else
        output_arg = NamedTuple()
        body_arg = () -> NamedTuple()
    end

    # HTTP.jl sets this header automatically.
    request.headers["Content-Length"] = string(length(request.content))

    # We pass an `input` only when we have content we wish to send.
    input = IOBuffer()
    if !isempty(request.content)
        write(input, request.content)
        input_arg = (; input=input)
    else<commit_msg>More correct Content-Length<commit_after>struct DownloadsBackend <: AWS.AbstractBackend
    downloader::Union{Nothing, Downloads.Downloader}
end

DownloadsBackend() = DownloadsBackend(nothing)

const AWS_DOWNLOADER = Ref{Union{Nothing, Downloader}}(nothing)
const AWS_DOWNLOAD_LOCK = ReentrantLock()

# Here we mimic Download.jl's own setup for using a global downloader.
# We do this to have our own downloader (separate from Downloads.jl's global downloader)
# because we add a hook to avoid redirects in order to try to match the HTTPBackend's
# implementation, and we don't want to mutate the global downloader from Downloads.jl.
# https://github.com/JuliaLang/Downloads.jl/blob/84e948c02b8a0625552a764bf90f7d2ee97c949c/src/Downloads.jl#L293-L301
function get_downloader(downloader=nothing)
    lock(AWS_DOWNLOAD_LOCK) do
        yield() # let other downloads finish
        downloader isa Downloader && return
        while true
            downloader = AWS_DOWNLOADER[]
            downloader isa Downloader && return
            D = Downloader()
            D.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false)
            AWS_DOWNLOADER[] = D
        end
    end
    return downloader
end

# https://github.com/JuliaWeb/HTTP.jl/blob/2a03ca76376162ffc3423ba7f15bd6d966edff9b/src/MessageRequest.jl#L84-L85
body_length(x::Vector{UInt8}) = length(x)
body_length(x::String) = sizeof(x)


function AWS._http_request(backend::DownloadsBackend, request)
    # If we pass `output`, Downloads.jl will expect a message
    # body in the response. Specifically, it sets
    # <https://curl.se/libcurl/c/CURLOPT_NOBODY.html>
    # only when we do not pass the `output` argument.
    #
    # When the method is `HEAD`, the response may have a Content-Length
    # but not send any content back (which appears to be correct,
    # <https://stackoverflow.com/a/18925736/12486544>).
    #
    # Thus, if we did not set `CURLOPT_NOBODY`, and it gets a Content-Length
    # back, it will hang waiting for that body.
    #
    # Therefore, we do not pass an `output` when the `request_method` is `HEAD`.
    if request.request_method != "HEAD"
        output = IOBuffer()
        output_arg = (; output=output)

        # We set a callback so later on we know how to get the `body` back.
        body_arg = () -> (; body = take!(output))
    else
        output_arg = NamedTuple()
        body_arg = () -> NamedTuple()
    end

    # HTTP.jl sets this header automatically.
    request.headers["Content-Length"] = string(body_length(request.content))

    # We pass an `input` only when we have content we wish to send.
    input = IOBuffer()
    if !isempty(request.content)
        write(input, request.content)
        input_arg = (; input=input)
    else
{'fraction_non_alphanumeric': 0.09470335017780274, 'fraction_numerical': 0.023582257158899493, 'mean_word_length': 3.2111899133175728, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1989934', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1633, 'n_words': 502}