<commit_before>// Package urltitle is a bort IRC bot plugin that extracts titles for URLs
// posted in a channel
package urltitle

import (
	"log"
	"net/http"
	"regexp"

	"github.com/ianremmler/bort"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html"
)

var (
	urlRE *regexp.Regexp
)

func findNode(path []string, node *html.Node) *html.Node {
	for i := range path {
		if node == nil {
			return nil
		}
		for node = node.FirstChild; node != nil; node = node.NextSibling {
			if node.Type == html.ElementNode && node.Data == path[i] {
				break
			}
		}
	}
	return node
}

func extractTitle(in, out *bort.Message) error {
	url := urlRE.FindString(in.Text)
	if url == "" {
		return nil
	}
	resp, err := http.Get(url)
	if err != nil {
		return nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil
	}
	page, err := html.Parse(resp.Body)
	if err != nil {
		return nil
	}
	title := findNode([]string{"html", "head", "title"}, page)
	if title != nil && title.FirstChild != nil {
		out.Type = bort.PrivMsg
		out.Text = title.FirstChild.Data
	}
	return nil
}

func init() {
	var err error
	if urlRE, err = xurls.StrictMatchingScheme("http"); err != nil {
		log.Println("urltitle: error setting regexp")
		return
	}
	if _, err = bort.RegisterMatcher(bort.PrivMsg, urlRE.String(), extractTitle); err != nil {
		log.Println("urltitle: error registering plugin")
	}
}<commit_msg>urltitle: we get the match from bort; don't do it again.<commit_after>// Package urltitle is a bort IRC bot plugin that extracts titles for URLs
// posted in a channel
package urltitle

import (
	"log"
	"net/http"

	"github.com/ianremmler/bort"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html"
)

func findNode(path []string, node *html.Node) *html.Node {
	for i := range path {
		if node == nil {
			return nil
		}
		for node = node.FirstChild; node != nil; node = node.NextSibling {
			if node.Type == html.ElementNode && node.Data == path[i] {
				break
			}
		}
	}
	return node
}

func extractTitle(in, out *bort.Message) error {
	resp, err := http.Get(in.Match)
	if err != nil {
		return nil
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return nil
	}
	page, err := html.Parse(resp.Body)
	if err != nil {
		return nil
	}
	title := findNode([]string{"html", "head", "title"}, page)
	if title != nil && title.FirstChild != nil {
		out.Type = bort.PrivMsg
		out.Text = title.FirstChild.Data
	}
	return nil
}

func init() {
	urlRE, err := xurls.StrictMatchingScheme("http")
	if err != nil {
		log.Println("urltitle: error setting regexp")
		return
	}
	pat := "(" + urlRE.String() + ")"
	if _, err = bort.RegisterMatcher(bort.PrivMsg, pat, extractTitle); err != nil {
		log.Println("urltitle: error registering plugin")
	}
}
{'fraction_non_alphanumeric': 0.1485148514851485, 'fraction_numerical': 0.0022002200220022, 'mean_word_length': 3.3858520900321545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2945401', 'n_tokens_mistral': 1067, 'n_tokens_neox': 1009, 'n_words': 273}