<commit_before>module BinDeps
    importall Base

    export @make_run, @build_steps, find_library, download_cmd, unpack_cmd,
            Choice, Choices, CCompile, FileDownloader, FileRule,
            ChangeDirectory, FileDownloader, FileUnpacker, prepare_src,
            autotools_install, CreateDirectory, MakeTargets, SystemLibInstall

    const dlext = isdefined(Base.Sys, :shlib_ext) ? Base.Sys.shlib_ext : Base.Sys.dlext # Julia 0.2/0.3 compatibility
    const shlib_ext = dlext # compatibility with older packages (e.g. ZMQ)

    function find_library(pkg,libname,files)
        Base.warn_once("BinDeps.find_library is deprecated, use Base.find_library instead.")
        dl = C_NULL
        for filename in files
            dl = dlopen_e(joinpath(Pkg.dir(),pkg,"deps","usr","lib",filename))
            if dl != C_NULL
                ccall(:add_library_mapping,Cint,(Ptr{Cchar},Ptr{Void}),libname,dl)
                return true
            end
               
            dl = dlopen_e(filename)
            if dl != C_NULL
                ccall(:add_library_mapping,Cint,(Ptr{Cchar},Ptr{Void}),libname,dl)
                return true
            end                
        end

        dl = dlopen_e(libname)
        dl != C_NULL ? true : false
    end

    macro make_rule(condition,command)
        quote
            if(!$(esc(condition)))
                $(esc(command))
                @assert $(esc(condition))
            end
        end
    end

    abstract BuildStep

    downloadcmd = nothing
    function download_cmd(url::String, filename::String)
        global downloadcmd
        if downloadcmd === nothing
            for checkcmd in (:curl, :wget, :fetch)
                try
                    if success(`$checkcmd --help`)
                        downloadcmd = checkcmd
                        break
                    end
                catch
                    continue # don't bail if one of these fails
                end
            end
        end
        if downloadcmd == :wget
            return `wget -O $filename $url`
        elseif downloadcmd == :curl
            return `curl -o $filename -L $url`
        elseif downloadcmd == :fetch
            return `fetch -f $filename $url`
        else
            error("No download agent available; install curl, wget, or fetch.")
        end
    end

    @unix_only begin
        function unpack_cmd(file,directory,extension,secondary_extension)
            if (extension == ".gz" && secondary_extension == ".tar") || extension == ".tgz"
                return (`tar xzf $file --directory=$directory`)
            elseif (extension == ".bz2" && secondary_extension == ".tar") || extension == ".tbz"
                return (`tar xjf $file --directory=$directory`)
            elseif extension == ".xz" && secondary_extension == ".tar"
                return (`unxz -c $file `|>`tar xv --directory=$directory`)
            elseif extension == ".tar"
                return (`tar xf $file --directory=$directory`)<commit_msg>Add Powershell option for download_cmd<commit_after>module BinDeps
    importall Base

    export @make_run, @build_steps, find_library, download_cmd, unpack_cmd,
            Choice, Choices, CCompile, FileDownloader, FileRule,
            ChangeDirectory, FileDownloader, FileUnpacker, prepare_src,
            autotools_install, CreateDirectory, MakeTargets, SystemLibInstall

    const dlext = isdefined(Base.Sys, :shlib_ext) ? Base.Sys.shlib_ext : Base.Sys.dlext # Julia 0.2/0.3 compatibility
    const shlib_ext = dlext # compatibility with older packages (e.g. ZMQ)

    function find_library(pkg,libname,files)
        Base.warn_once("BinDeps.find_library is deprecated, use Base.find_library instead.")
        dl = C_NULL
        for filename in files
            dl = dlopen_e(joinpath(Pkg.dir(),pkg,"deps","usr","lib",filename))
            if dl != C_NULL
                ccall(:add_library_mapping,Cint,(Ptr{Cchar},Ptr{Void}),libname,dl)
                return true
            end
               
            dl = dlopen_e(filename)
            if dl != C_NULL
                ccall(:add_library_mapping,Cint,(Ptr{Cchar},Ptr{Void}),libname,dl)
                return true
            end                
        end

        dl = dlopen_e(libname)
        dl != C_NULL ? true : false
    end

    macro make_rule(condition,command)
        quote
            if(!$(esc(condition)))
                $(esc(command))
                @assert $(esc(condition))
            end
        end
    end

    abstract BuildStep

    downloadcmd = nothing
    function download_cmd(url::String, filename::String)
        global downloadcmd
        if downloadcmd === nothing
            for checkcmd in @windows? (:powershell, :curl, :wget, :fetch) : (:curl, :wget, :fetch)
                try
                    if success(`$checkcmd --help`)
                        downloadcmd = checkcmd
                        break
                    end
                catch
                    continue # don't bail if one of these fails
                end
            end
        end
        if downloadcmd == :wget
            return `wget -O $filename $url`
        elseif downloadcmd == :curl
            return `curl -o $filename -L $url`
        elseif downloadcmd == :fetch
            return `fetch -f $filename $url`
        elseif downloadcmd == :powershell
            return `powershell -Command "(new-object net.webclient).DownloadFile(\"$url\", \"$filename\")"`
        else
            error("No download agent available; install curl, wget, or fetch.")
        end
    end

    @unix_only begin
        function unpack_cmd(file,directory,extension,secondary_extension)
            if (extension == ".gz" && secondary_extension == ".tar") || extension == ".tgz"
                return (`tar xzf $file --directory=$directory`)
            elseif (extension == ".bz2" && secondary_extension == ".tar") || extension == ".tbz"
                return (`tar xjf $file --directory=$directory`)
            elseif extension == ".xz" && secondary_extension == ".tar"
                return (`unxz -c $file `|>`tar xv --directory=$directory`)
            elseif extension == ".tar"
                return (`tar xf $file --directory=$directory`)
{'fraction_non_alphanumeric': 0.10438946528332003, 'fraction_numerical': 0.0015961691939345571, 'mean_word_length': 1.8237945020279405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3571874', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1641, 'n_words': 445}