<commit_before>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
)

// init injects our "http" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "http",
		Aliases:     []string{"https"},
		Usage:       "connect to host/IP through HTTP",
		Description: "send HTTP queries to an host/IP",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:        "X, method",
				Usage:       "Use this method instead of default GET",
				Destination: &fHTTPMethod,
			},
			cli.StringFlag{
				Name:        "U, user-agent",
				Usage:       "Override User-Agent.",
				Destination: &fUserAgent,
			},
			cli.StringFlag{
				Name:        "V, version",
				Usage:       "Set a specific HTTP version.",
				Destination: &fHTTPVersion,
			},
		},
		Action: cmdHTTP,
	})
}

func makeDefinition(ip int) (def *atlas.Definition) {
	def = &atlas.Definition{
		AF:     ip,
		Type:   "http",
		Method: "GET",
	}
	if fHTTPMethod != "" {
		def.Method = fHTTPMethod
	}
	if fUserAgent != "" {
		def.UserAgent = fUserAgent
	}
	if fHTTPVersion != "" {
		def.Version = fHTTPVersion
	}
	return
}

func cmdHTTP(c *cli.Context) error {
	// By default we want both
	if !fWant4 && !fWant6 {
		fWant6, fWant4 = true, true
	}

	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a hostname/site!")
	}

	// We expect target to be using [http|https]://<site>[:port]/path
	target := args[0]

	var (
		defs []atlas.Definition
	)

	_, site, path, port := analyzeTarget(target)

	if fWant4 {
		def := makeDefinition(4)
		def.Description = fmt.Sprintf("HTTP v4 - %s", target)
		def.Target = site
		def.Port = port
		def.Path = path

		defs = append(defs, *def)
	}

	if fWant6 {
		def := makeDefinition(6)
		def.Description = fmt.Sprintf("HTTP v6 - %s", target)
		def.Target = site
		def.Port = port
		def.Path = path

		defs = append(defs, *def)
	}

	req := atlas.MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
	}
	// Default set of probes
	probes := atlas.ProbeSet{
		{
			Requested: mycnf.PoolSize,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}

	req.Probes = probes
	log.Printf("req=%#v", req)<commit_msg>Simplify cmdHTTP by using the new helpers NewMeasurement() and AddDefintion().<commit_after>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
)

// init injects our "http" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "http",
		Aliases:     []string{"https"},
		Usage:       "connect to host/IP through HTTP",
		Description: "send HTTP queries to an host/IP",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:        "X, method",
				Usage:       "Use this method instead of default GET",
				Value:       "GET",
				Destination: &fHTTPMethod,
			},
			cli.StringFlag{
				Name:        "U, user-agent",
				Usage:       "Override User-Agent.",
				Destination: &fUserAgent,
			},
			cli.StringFlag{
				Name:        "V, version",
				Usage:       "Set a specific HTTP version.",
				Destination: &fHTTPVersion,
			},
		},
		Action: cmdHTTP,
	})
}

func prepareHTTP(target, path string, port int) (req *atlas.MeasurementRequest) {
	opts := map[string]string{
		"Type":        "http",
		"Method":      fHTTPMethod,
		"UserAgent":   fUserAgent,
		"Version":     fHTTPVersion,
		"Description": fmt.Sprintf("HTTP - %s", target),
		"Target":      target,
		"Path":        path,
		"Port":        fmt.Sprintf("%d", port),
	}

	req = atlas.NewMeasurement()
	if mycnf.WantAF == WantBoth {

		opts["AF"] = "4"
		req.AddDefinition(opts)

		opts["AF"] = "6"
		req.AddDefinition(opts)
	} else {
		opts["AF"] = mycnf.WantAF
		req.AddDefinition(opts)
	}

	return
}

func cmdHTTP(c *cli.Context) error {
	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a hostname/site!")
	}

	// We expect target to be using [http|https]://<site>[:port]/path
	target := args[0]

	proto, site, path, port := analyzeTarget(target)
	if proto == "" || site == "" {
		return fmt.Errorf("Invalid URL: %s", target)
	}

	log.Printf("Target: %s://%s:%d%s", proto, site, port, path)

	req := prepareHTTP(site, path, port)

	log.Printf("req=%#v", req)
{'fraction_non_alphanumeric': 0.1558891454965358, 'fraction_numerical': 0.003695150115473441, 'mean_word_length': 2.863514719000892, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1836994', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1572, 'n_words': 391}