<commit_before>package main

import (
	"bufio"
	"flag"
	"log"
	"net"
	"os"
	"runtime"
	"strings"

	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
)

type host struct {
	name     string
	protocol string
	endpoint string
	addrs    []string
}

var hosts []host

// readHosts reads in hosts from a file and populates hosts []host.
func readHosts() {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		hosts = append(hosts, host{name: scanner.Text()})
		// TODO: parse for urls, set host.protocol and host.endpoint. net/url.Parse seems like a good fit.
	}
	if err := scanner.Err(); err != nil {
		log.Printf("error reading hosts from %s:%s\n", filename, err)
	}
}

var filename string

func init() {
	// Check for '-f' flag for host file
	flag.StringVar(&filename, "f", "hosts", "File with hosts and urls to check.")
	flag.Parse()
}

func (h *host) htoi() error {
	if len(h.addrs) == 0 {
		addrs, err := net.LookupHost(h.name)
		for _, i := range addrs {
			// Skip if address is ipv6.
			if strings.Contains(i, ":") {
				continue
			}
			h.addrs = append(h.addrs, i)
		}
		return err
	}
	return nil
}

func (h host) ping() {
	switch runtime.GOOS {
	case "darwin":
	case "linux":
		log.Println("you may need to adjust the net.ipv4.ping_group_range kernel state")
	default:
		log.Println("not supported on", runtime.GOOS)
		return
	}

	c, err := icmp.ListenPacket("udp4", "0.0.0.0")
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	wm := icmp.Message{
		Type: ipv4.ICMPTypeEcho, Code: 0,
		Body: &icmp.Echo{
			ID: os.Getpid() & 0xffff, Seq: 1,
			Data: []byte("Ranger-Chooch"),
		},
	}
	wb, err := wm.Marshal(nil)
	if err != nil {
		log.Fatal(err)
	}

	// Cowardly taking the first address.
	if _, err := c.WriteTo(wb, &net.UDPAddr{IP: net.ParseIP(h.addrs[0])}); err != nil {
		log.Fatal(err)
	}

	rb := make([]byte, 1500)
	n, peer, err := c.ReadFrom(rb)
	if err != nil {
		log.Fatal(err)
	}
	rm, err := icmp.ParseMessage(1, rb[:n])
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("got reflection from %v", peer)
	log.Printf("got %+v", rm)
	log.Printf("got %s", rm.Body)
	log.Printf("rm.Body type: %T", rm.Body)
	log.Printf("os.Getpid(): %d", os.Getpid()&0xffff)

}

func main() {
	// if an entry is a url, send a GET request
	// if an entry is a hostname, send an ICMP ping
	// TODO: host method for GET
	// TODO: host method for ICMP
	// TODO: figure out how to represent responses.
	// TODO: store responses in google sheets.
	// TODO: cache writes to google sheets if network is unavailable.
	// TODO: rewrite host request methods as goroutines.
}<commit_msg>Improve `ping()`, create `addResp()`<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"strings"
	"time"

	"golang.org/x/net/icmp"
	"golang.org/x/net/ipv4"
)

type host struct {
	name     string
	protocol string
	endpoint string
	addrs    []string
	resps    []resp
}

type resp struct {
	id   int
	seq  int
	code int
	sent time.Time // TODO: Store as time.
	recv time.Time // TODO: Store as time.
	dur  time.Duration
	to   string
	from string
	body string // TODO: Find a more optimal storage. Works well for ICMP, but not for GET.
}

var hosts []host

// readHosts reads in hosts from a file and populates hosts []host.
func readHosts() {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		hosts = append(hosts, host{name: scanner.Text()})
		// TODO: parse for urls, set host.protocol and host.endpoint. net/url.Parse seems like a good fit.
	}
	if err := scanner.Err(); err != nil {
		log.Printf("error reading hosts from %s:%s\n", filename, err)
	}
}

var filename string

func init() {
	// Check for '-f' flag for host file
	flag.StringVar(&filename, "f", "hosts", "File with hosts and urls to check.")
	flag.Parse()
}

func (h *host) htoi() error {
	if len(h.addrs) == 0 {
		addrs, err := net.LookupHost(h.name)
		for _, i := range addrs {
			// Skip if address is ipv6.
			if strings.Contains(i, ":") {
				continue
			}
			h.addrs = append(h.addrs, i)
		}
		return err
	}
	return nil
}

func (h *host) ping() {
	switch runtime.GOOS {
	case "darwin":
	case "linux":
		log.Println("you may need to adjust the net.ipv4.ping_group_range kernel state")
	default:
		log.Println("not supported on", runtime.GOOS)
		return
	}

	c, err := icmp.ListenPacket("udp4", "0.0.0.0")
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	s := fmt.Sprintf("Ranger-Chooch-%s", h.name)
	var to net.IP
	var from string
	var sent, recv time.Time

	// TODO: icmp echo forever
	for i := 0; i < 1; i++ {
		wm := icmp.Message{
			Type: ipv4.ICMPTypeEcho, Code: 0,
			/* TODO: Use ID and Data as a sanity check for Echo Replies.
			 * Using ID was from the example from golang.org, and /sbin/ping uses its PID as ICMP's ID.
			 * Maybe there's a reason for this, and we should keep ID as our PID.
			 */

			Body: &icmp.Echo{
				ID: os.Getpid() & 0xffff, Seq: i & 0xffff,
				Data: []byte(s),
			},
		}
		wb, err := wm.Marshal(nil)
		if err != nil {
			log.Fatal(err)
		}

		// Cowardly taking the first address.
		to = net.ParseIP(h.addrs[0])

		if _, err := c.WriteTo(wb, &net.UDPAddr{IP: to}); err != nil {
			log.Fatal(err)
		}
		sent = time.Now()
	}

	rb := make([]byte, 1500)
	// TODO: icmp echoreply forever
	for i := 0; i < 1; i++ {
		n, peer, err := c.ReadFrom(rb)
		if err != nil {
			log.Fatal(err)
		}
		recv = time.Now()
		rm, err := icmp.ParseMessage(1, rb[:n])
		if err != nil {
			log.Fatal(err)
		}

		// TODO: check that ip and h.addrs[0] are the same.
		from, _, _ = net.SplitHostPort(peer.String())
		if h.addrs[0] != from {
			log.Printf("got echo reply from %s; want %s", from, to)
		}

		if rm.Type != ipv4.ICMPTypeEchoReply {
			log.Printf("received something other than ping: %d", rm.Type)
			continue
		}
		/* Helpful for inspecting rm.
		log.Printf("rm type: %T", rm)
		log.Printf("rm.Body type: %T", rm.Body)
		log.Printf("rm.Type type: %T", rm.Type)
		log.Println("")
		log.Printf("rm value: %+v", rm)
		log.Printf("rm.Body value: %+v", rm.Body)
		log.Printf("rm.Type value: %v", rm.Type)
		log.Println("")
		*/
		/* BUG: can't access rm.Body.Data
		 * log.Printf("rm.Body value: %+v", icmp.Echo(rm.Body).Data)
		 * Ideally, we'd unpack rm.Body.Data into host.resps
		 */
		body := string(rm.Body.(*icmp.Echo).Data)
		seq := rm.Body.(*icmp.Echo).Seq
		id := rm.Body.(*icmp.Echo).ID
		dur := sent.Sub(recv)
		h.addResp(id, seq, rm.Code, sent, recv, dur, to.String(), from, body)
		time.Sleep(time.Second)

	}
}

func (h *host) addResp(id, seq, code int, sent, recv time.Time, dur time.Duration, to, from, body string) {
	r := resp{
		id:   id,
		seq:  seq,
		code: code,
		sent: sent,
		recv: recv,
		dur:  dur,
		to:   to,
		from: from,
		body: body,
	}
	h.resps = append(h.resps, r)
}

func main() {
	// if an entry is a url, send a GET request
	// if an entry is a hostname, send an ICMP ping
	// TODO: host method for GET
	// TODO: host method for ICMP
	// TODO: figure out how to represent responses.
	// TODO: store responses in google sheets.
	// TODO: cache writes to google sheets if network is unavailable.
	// TODO: rewrite host request methods as goroutines.
	// TODO: intercept control-c, stop pings, drain responses, exit.
}
{'fraction_non_alphanumeric': 0.15905683521875427, 'fraction_numerical': 0.006269592476489028, 'mean_word_length': 2.996732026143791, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1710633', 'n_tokens_mistral': 3087, 'n_tokens_neox': 2859, 'n_words': 865}