<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"os"
	"strings"

	"github.com/levigross/grequests"
	"gopkg.in/urfave/cli.v2"
)

func action(ctx *cli.Context) error {
	var url string
	if len(ctx.String("url")) > 0 {
		url = ctx.String("url")
	} else if len(ctx.Args().Get(0)) > 0 {
		url = ctx.Args().Get(0)
	} else {
		return errors.New("No URL specified")
	}

	ro := &grequests.RequestOptions{}
	ro.InsecureSkipVerify = ctx.Bool("insecure")
	ro.UserAgent = ctx.String("user-agent")

	headers := ctx.StringSlice("header")
	if len(headers) > 0 {
		ro.Headers = make(map[string]string)
		for _, header := range headers {
			hs := strings.SplitN(header, ":", 2)
			if len(hs) == 2 {
				ro.Headers[hs[0]] = hs[1]
			}
		}
	}

	data := ctx.String("data")
	if len(data) > 0 {
		ro.RequestBody = bytes.NewReader([]byte(data))
	}

	method := ctx.String("request")
	if !ctx.IsSet("request") && ctx.IsSet("head") {
		method = "HEAD"
	}
	if !ctx.IsSet("request") && ctx.IsSet("data") {
		method = "POST"
	}

	if method == "POST" {
		if ro.Headers == nil {
			ro.Headers = make(map[string]string, 0)
		}
		if _, ok := ro.Headers["Content-Type"]; !ok {
			ro.Headers["Content-Type"] = "application/x-www-form-urlencoded"
		}
	}

	user := ctx.String("user")
	if len(user) > 0 {
		ro.Auth = strings.Split(user, ":")
	}

	ro.HTTPClient = grequests.BuildHTTPClient(*ro)
	if !ctx.Bool("location") {
		ro.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
	}

	var rsp *grequests.Response
	var err error
	switch method {
	case "GET":
		rsp, err = grequests.Get(url, ro)
	case "HEAD":
		rsp, err = grequests.Head(url, ro)
	case "POST":
		rsp, err = grequests.Post(url, ro)
	case "PUT":
		rsp, err = grequests.Put(url, ro)
	case "OPTIONS":
		rsp, err = grequests.Options(url, ro)
	case "PATCH":
		rsp, err = grequests.Patch(url, ro)
	case "DELETE":
		rsp, err = grequests.Delete(url, ro)
	default:
		return errors.New("Unsupported COMMAND")
	}
	if err == nil {
		if ctx.Bool("include") || ctx.Bool("head") {
			rawRsp := rsp.RawResponse
			fmt.Println(rawRsp.Proto + " " + rawRsp.Status)
			for k, v := range rsp.Header {
				fmt.Println(k + ": " + strings.Join(v, ";"))
			}
			fmt.Println("")
		}
		if !ctx.Bool("head") {
			fmt.Print(rsp)
		}
	}
	return nil
}

func main() {
	cli.AppHelpTemplate = `usage: {{.Name}} [options...] <url>
author: {{range .Authors}}{{ . }}{{end}}
options:
    {{range .VisibleFlags}}{{.}}
    {{end}}`

	fetch := &cli.App{
		Name:    "fetch",
		Version: "0.0.1",
		Authors: []*cli.Author{
			&cli.Author{Name: "qiyi", Email: "<EMAIL>"}},
		Description: "URL fetch application",
		Action:      action,
	}
	flags := []cli.Flag{
		&cli.StringFlag{
			Name:    "data",
			Aliases: []string{"d"},
			Usage:   "HTTP POST `DATA`",
		},
		&cli.StringSliceFlag{
			Name:    "header",
			Aliases: []string{"H"},
			Usage:   "Custom header `LINE` to pass to server",
		},<commit_msg>Change version to 0.0.2<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"net/http"
	"os"
	"strings"

	"github.com/levigross/grequests"
	"gopkg.in/urfave/cli.v2"
)

func action(ctx *cli.Context) error {
	var url string
	if len(ctx.String("url")) > 0 {
		url = ctx.String("url")
	} else if len(ctx.Args().Get(0)) > 0 {
		url = ctx.Args().Get(0)
	} else {
		return errors.New("No URL specified")
	}

	ro := &grequests.RequestOptions{}
	ro.InsecureSkipVerify = ctx.Bool("insecure")
	ro.UserAgent = ctx.String("user-agent")

	headers := ctx.StringSlice("header")
	if len(headers) > 0 {
		ro.Headers = make(map[string]string)
		for _, header := range headers {
			hs := strings.SplitN(header, ":", 2)
			if len(hs) == 2 {
				ro.Headers[hs[0]] = hs[1]
			}
		}
	}

	data := ctx.String("data")
	if len(data) > 0 {
		ro.RequestBody = bytes.NewReader([]byte(data))
	}

	method := ctx.String("request")
	if !ctx.IsSet("request") && ctx.IsSet("head") {
		method = "HEAD"
	}
	if !ctx.IsSet("request") && ctx.IsSet("data") {
		method = "POST"
	}

	if method == "POST" {
		if ro.Headers == nil {
			ro.Headers = make(map[string]string, 0)
		}
		if _, ok := ro.Headers["Content-Type"]; !ok {
			ro.Headers["Content-Type"] = "application/x-www-form-urlencoded"
		}
	}

	user := ctx.String("user")
	if len(user) > 0 {
		ro.Auth = strings.Split(user, ":")
	}

	ro.HTTPClient = grequests.BuildHTTPClient(*ro)
	if !ctx.Bool("location") {
		ro.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
	}

	var rsp *grequests.Response
	var err error
	switch method {
	case "GET":
		rsp, err = grequests.Get(url, ro)
	case "HEAD":
		rsp, err = grequests.Head(url, ro)
	case "POST":
		rsp, err = grequests.Post(url, ro)
	case "PUT":
		rsp, err = grequests.Put(url, ro)
	case "OPTIONS":
		rsp, err = grequests.Options(url, ro)
	case "PATCH":
		rsp, err = grequests.Patch(url, ro)
	case "DELETE":
		rsp, err = grequests.Delete(url, ro)
	default:
		return errors.New("Unsupported COMMAND")
	}
	if err == nil {
		if ctx.Bool("include") || ctx.Bool("head") {
			rawRsp := rsp.RawResponse
			fmt.Println(rawRsp.Proto + " " + rawRsp.Status)
			for k, v := range rsp.Header {
				fmt.Println(k + ": " + strings.Join(v, ";"))
			}
			fmt.Println("")
		}
		if !ctx.Bool("head") {
			fmt.Print(rsp)
		}
	}
	return nil
}

func main() {
	cli.AppHelpTemplate = `usage: {{.Name}} [options...] <url>
author: {{range .Authors}}{{ . }}{{end}}
options:
    {{range .VisibleFlags}}{{.}}
    {{end}}`

	fetch := &cli.App{
		Name:    "fetch",
		Version: "0.0.2",
		Authors: []*cli.Author{
			&cli.Author{Name: "qiyi", Email: "<EMAIL>"}},
		Description: "URL fetch application",
		Action:      action,
	}
	flags := []cli.Flag{
		&cli.StringFlag{
			Name:    "data",
			Aliases: []string{"d"},
			Usage:   "HTTP POST `DATA`",
		},
		&cli.StringSliceFlag{
			Name:    "header",
			Aliases: []string{"H"},
			Usage:   "Custom header `LINE` to pass to server",
		},
{'fraction_non_alphanumeric': 0.19698492462311556, 'fraction_numerical': 0.005862646566164154, 'mean_word_length': 3.3839941262848754, 'pattern_counts': {'":': 20, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3235861', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2349, 'n_words': 495}