<commit_before>cURLImporter = ->

    # Create Paw request
    @createPawRequest = (context, curlArgs, rawString) ->

        if not curlArgs or curlArgs.length < 0
            console.log "Invalid curl command"

        curlObj = {
            method: "GET",
            headers: {},
            body: null,
            multipartBody: null,
            url: ""
        }

        @resolveURL curlArgs, rawString, curlObj
        @resolveMethod curlArgs, curlObj
        @resolveHeaders curlArgs, curlObj

        @resolveBody curlArgs, curlObj

        @resolveCompressed curlArgs, curlObj
        @resolveUserAgent curlArgs, curlObj
        @resolveCookie curlArgs, curlObj
        @resolveReferer curlArgs, curlObj
        @resolveAuth curlArgs, curlObj

        @cleanUrl curlObj
        # Create Paw request
        pawRequest = context.createRequest "cURL", curlObj.method, curlObj.url


        for header, value of curlObj.headers
            pawRequest.setHeader(@clean(header), @clean(value))

        if curlObj.body != null
            pawRequest.body = curlObj.body
        else if curlObj.multipartBody != null
            pawRequest.multipartBody = curlObj.multipartBody
        return pawRequest

    @clean = (string) ->
        if typeof stringValue isnt 'string'
            return string
        if string == null
            return ""
        re = /^(?:\"([^\"]+)\"$|\'([^\']+)\'|[\ ]*([^\ ]+[^]*[^\ ]+)[\ ]*|[\ ]*([^\ ])[\ ]*)$/;
        m = re.exec(string)
        if m == null
            throw new Error "Invalid curl file unable to clean string: #{string}"
        if typeof m[1] isnt 'undefined'
            return m[1]
        else if typeof m[2] isnt 'undefined'
            return m[2]
        else if typeof m[3] isnt 'undefined'
            return m[3]
        else if typeof m[4] isnt 'undefined'
            return m[4]
        else
            throw new Error "Invalid curl file unable to clean string: #{string}"


    @resolveURL = (curlArgs, rawString, curlObj) ->
        if "url" of curlArgs
            urls = curlArgs.url
            curlObj.url = @clean urls[urls.length-1]
        else
            re = /([^\ ]+)[\ ]*$/;
            m = re.exec(rawString)
            if m != null
                curlObj.url = @clean m[1]

    @resolveMethod =  (curlArgs, curlObj) ->
        methods = ["GET"]
        if "X" of curlArgs<commit_msg>Add support for URLs without http:// prefix<commit_after>cURLImporter = ->

    # Create Paw request
    @createPawRequest = (context, curlArgs, rawString) ->

        if not curlArgs or curlArgs.length < 0
            console.log "Invalid curl command"

        curlObj = {
            method: "GET",
            headers: {},
            body: null,
            multipartBody: null,
            url: ""
        }

        @resolveURL curlArgs, rawString, curlObj
        @resolveMethod curlArgs, curlObj
        @resolveHeaders curlArgs, curlObj

        @resolveBody curlArgs, curlObj

        @resolveCompressed curlArgs, curlObj
        @resolveUserAgent curlArgs, curlObj
        @resolveCookie curlArgs, curlObj
        @resolveReferer curlArgs, curlObj
        @resolveAuth curlArgs, curlObj

        @cleanUrl curlObj
        # Create Paw request
        pawRequest = context.createRequest "cURL", curlObj.method, curlObj.url


        for header, value of curlObj.headers
            pawRequest.setHeader(@clean(header), @clean(value))

        if curlObj.body != null
            pawRequest.body = curlObj.body
        else if curlObj.multipartBody != null
            pawRequest.multipartBody = curlObj.multipartBody
        return pawRequest

    @clean = (string) ->
        if typeof stringValue isnt 'string'
            return string
        if string == null
            return ""
        re = /^(?:\"([^\"]+)\"$|\'([^\']+)\'|[\ ]*([^\ ]+[^]*[^\ ]+)[\ ]*|[\ ]*([^\ ])[\ ]*)$/;
        m = re.exec(string)
        if m == null
            throw new Error "Invalid curl file unable to clean string: #{string}"
        if typeof m[1] isnt 'undefined'
            return m[1]
        else if typeof m[2] isnt 'undefined'
            return m[2]
        else if typeof m[3] isnt 'undefined'
            return m[3]
        else if typeof m[4] isnt 'undefined'
            return m[4]
        else
            throw new Error "Invalid curl file unable to clean string: #{string}"

    @cleanURL = (string) ->
      string = @clean string
      if not string.match(/^https?\:\/\//)
        string = 'http://' + string
      return string

    @resolveURL = (curlArgs, rawString, curlObj) ->
        if "url" of curlArgs
            urls = curlArgs.url
            curlObj.url = @cleanURL urls[urls.length-1]
        else
            re = /([^\ ]+)[\ ]*$/;
            m = re.exec(rawString)
            if m != null
                curlObj.url = @cleanURL m[1]

    @resolveMethod =  (curlArgs, curlObj) ->
        methods = ["GET"]
        if "X" of curlArgs
{'fraction_non_alphanumeric': 0.1198948645369996, 'fraction_numerical': 0.004448038819247877, 'mean_word_length': 1.772982062780269, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4500181', 'n_tokens_mistral': 1454, 'n_tokens_neox': 1408, 'n_words': 398}