<commit_before>package module

import (
	"fmt"
	"github.com/davidscholberg/irkbot/lib/configure"
	"github.com/davidscholberg/irkbot/lib/message"
	"golang.org/x/net/html"
	"io"
	"mvdan.cc/xurls"
	"net"
	"net/http"
	urllib "net/url"
	"os"
	"strings"
	"time"
)

// parseUrls attempts to fetch the title of the HTML document returned by a URL
func parseUrls(cfg *configure.Config, in *message.InboundMsg, actions *actions) bool {
	urls := xurls.Strict().FindAllString(in.Msg, -1)

	for _, urlStr := range urls {
		url, err := urllib.Parse(urlStr)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		if v, err := validateUrl(url); !v {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		host := url.Host
		title, err := getHtmlTitle(urlStr, cfg.Http.ResponseSizeLimit, cfg.Http.Timeout)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		// split string by newline characters
		titleFields := strings.FieldsFunc(title, func(c rune) bool {
			return c == '\n' || c == '\r'
		})
		// join by newline separator
		title = strings.Join(titleFields, " / ")
		title = strings.TrimSpace(title)
		actions.say(fmt.Sprintf("^ %s - [%s]", title, host))
	}

	// don't consume the message, in case there are commands in it
	return false
}

// validateUrl ensures that the given URL is safe to GET.
func validateUrl(url *urllib.URL) (bool, error) {
	privateSubnets := []string{
		"127.0.0.0/8",
		"10.0.0.0/8",
		"172.16.0.0/12",
		"192.168.0.0/16",
		"::1/128",
		"fc00::/7"}

	ips, err := net.LookupIP(url.Host)
	if err != nil {
		return false, err
	}
	if len(ips) == 0 {
		return false, fmt.Errorf("no IPs found for %s", url.Host)
	}

	hostNotAllowed, err := isCidrMatch(&ips[0], privateSubnets)
	if err != nil {
		return false, err
	}
	if hostNotAllowed {
		return false, fmt.Errorf("host not allowed")
	}

	return true, nil
}<commit_msg>url: fixed whitespace handling in displaying title<commit_after>package module

import (
	"fmt"
	"github.com/davidscholberg/irkbot/lib/configure"
	"github.com/davidscholberg/irkbot/lib/message"
	"golang.org/x/net/html"
	"io"
	"mvdan.cc/xurls"
	"net"
	"net/http"
	urllib "net/url"
	"os"
	"strings"
	"time"
)

// parseUrls attempts to fetch the title of the HTML document returned by a URL
func parseUrls(cfg *configure.Config, in *message.InboundMsg, actions *actions) bool {
	urls := xurls.Strict().FindAllString(in.Msg, -1)

	for _, urlStr := range urls {
		url, err := urllib.Parse(urlStr)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		if v, err := validateUrl(url); !v {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		host := url.Host
		title, err := getHtmlTitle(urlStr, cfg.Http.ResponseSizeLimit, cfg.Http.Timeout)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			continue
		}
		cleanedTitle := cleanTitleWhiteSpace(title)
		actions.say(fmt.Sprintf("^ %s - [%s]", cleanedTitle, host))
	}

	// don't consume the message, in case there are commands in it
	return false
}

// cleanTitleWhiteSpace takes the contents of an html title and makes it fit
// to be printed on a single line.
func cleanTitleWhiteSpace(title string) string {
	// split string by newline characters
	titleFields := strings.FieldsFunc(title, func(c rune) bool {
		return c == '\n' || c == '\r'
	})
	// trim each field
	trimmedFields := []string{}
	for _, field := range titleFields {
		field = strings.TrimSpace(field)
		if field != "" {
			trimmedFields = append(trimmedFields, field)
		}
	}
	// join by newline separator
	return strings.Join(trimmedFields, " / ")
}

// validateUrl ensures that the given URL is safe to GET.
func validateUrl(url *urllib.URL) (bool, error) {
	privateSubnets := []string{
		"127.0.0.0/8",
		"10.0.0.0/8",
		"172.16.0.0/12",
		"192.168.0.0/16",
		"::1/128",
		"fc00::/7"}

	ips, err := net.LookupIP(url.Host)
	if err != nil {
		return false, err
	}
	if len(ips) == 0 {
		return false, fmt.Errorf("no IPs found for %s", url.Host)
	}

	hostNotAllowed, err := isCidrMatch(&ips[0], privateSubnets)
	if err != nil {
		return false, err
	}
	if hostNotAllowed {
		return false, fmt.Errorf("host not allowed")
	}

	return true, nil
}
{'fraction_non_alphanumeric': 0.1510808841389361, 'fraction_numerical': 0.02040320621811999, 'mean_word_length': 3.476086956521739, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '609339', 'n_tokens_mistral': 1687, 'n_tokens_neox': 1551, 'n_words': 403}