<commit_before>local fs        = require "lemoon.fs"
local sys       = require "lemoon.sys"
local class     = require "lemoon.class"
local filepath  = require "lemoon.filepath"
local logger    = class.new("lemoon.log","gsmake")
local console   = class.new("lemoon.log","console")


task.sync_init = function(self)
    if sys.host() == "Windows" then
        curl = filepath.join(loader.Path,"tools/win32/curl.exe")
        unzip = filepath.join(loader.Path,"tools/win32/7z.exe")
    else
        local ok, curlpath = sys.lookup("curl")
        if not ok then
            throw("curl program not found !!!!!!!")
        end

        curl = curlpath

        local ok, unzippath = sys.lookup("7z")
        if not ok then
            throw("7z program not found !!!!!!!")
        end

        unzip = unzippath
    end


end

task.sync_remote = function(self,name,version,url)
    print(string.format("sync remote package:%s",url))

    local workdir = filepath.join(loader.Config.GlobalRepo,"curl",name)

    if not fs.exists(workdir) then
        fs.mkdir(workdir,true)
    end

    local exec = sys.exec(curl)
    exec:dir(workdir)
    -- --
    local tmpfile = string.format("%s.tmp",version)
    --
    exec:start("-o",tmpfile,"-k",url)

    if 0 ~= exec:wait() then
        console:E("download remote package error :%s",url)
        return true
    end

    local outputdir = filepath.join(workdir,version)

    -- if fs.exists(outputdir) then
    --     fs.rm(outputdir,true)
    -- end<commit_msg>fix windows curl lookup bug<commit_after>local fs        = require "lemoon.fs"
local sys       = require "lemoon.sys"
local class     = require "lemoon.class"
local filepath  = require "lemoon.filepath"
local logger    = class.new("lemoon.log","gsmake")
local console   = class.new("lemoon.log","console")


task.sync_init = function(self)
    if sys.host() == "Windows" then
        curl = filepath.join(self.Package.Path,"tools/win32/curl.exe")
        unzip = filepath.join(self.Package.Path,"tools/win32/7z.exe")
    else
        local ok, curlpath = sys.lookup("curl")
        if not ok then
            throw("curl program not found !!!!!!!")
        end

        curl = curlpath

        local ok, unzippath = sys.lookup("7z")
        if not ok then
            throw("7z program not found !!!!!!!")
        end

        unzip = unzippath
    end


end

task.sync_remote = function(self,name,version,url)
    print(string.format("sync remote package:%s",url))

    local workdir = filepath.join(loader.Config.GlobalRepo,"curl",name)

    if not fs.exists(workdir) then
        fs.mkdir(workdir,true)
    end

    print(curl)

    local exec = sys.exec(curl)
    exec:dir(workdir)
    -- --
    local tmpfile = string.format("%s.tmp",version)
    --
    exec:start("-o",tmpfile,"-k",url)

    if 0 ~= exec:wait() then
        console:E("download remote package error :%s",url)
        return true
    end

    local outputdir = filepath.join(workdir,version)

    -- if fs.exists(outputdir) then
    --     fs.rm(outputdir,true)
    -- end
{'fraction_non_alphanumeric': 0.13849688217919265, 'fraction_numerical': 0.005251066622907778, 'mean_word_length': 2.507479861910242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '15223', 'n_tokens_mistral': 1011, 'n_tokens_neox': 980, 'n_words': 218}