<commit_before>// Package urltitle implements a plugin to watch web URLs, fetch and display title.
package urltitle

import (
	"errors"
	"fmt"
	bot "github.com/StalkR/goircbot"
	irc "github.com/fluffle/goirc/client"
	"html"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"
)

func timeoutDialer(d time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		return net.DialTimeout(netw, addr, d)
	}
}

// Title gets an URL and returns its title.
func Title(url string) (string, error) {
	client := &http.Client{
		Transport: &http.Transport{
			Dial: timeoutDialer(3 * time.Second),
		},
	}
	resp, err := client.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	r, err := regexp.Compile("<title[^>]*>([^<]+)<")
	if err != nil {
		return "", err
	}
	matches := r.FindSubmatch([]byte(contents))
	if len(matches) < 2 {
		return "", errors.New("no title found in page")
	}
	return html.UnescapeString(strings.TrimSpace(string(matches[1]))), nil
}

func watchLine(b *bot.Bot, line *irc.Line, ignoremap map[string]bool) {
	target := line.Args[0]
	if !strings.HasPrefix(target, "#") {
		return
	}
	if _, ignore := ignoremap[line.Nick]; ignore {
		return
	}
	text := line.Args[1]
	if !strings.Contains(text, "http") {
		return
	}
	r, err := regexp.Compile("(?:^|\\s)(https?://[^\\s]+)")
	if err != nil {
		return
	}
	matches := r.FindSubmatch([]byte(text))
	if len(matches) < 2 {
		return
	}
	url := string(matches[1])
	title, err := Title(url)
	if err != nil {
		log.Println("urltitle:", err)
		return
	}
	if len(title) > 200 {
		title = title[:200]
	}
	b.Conn.Privmsg(target, fmt.Sprintf("%s :: %s", url, title))
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot, ignore []string) {
	ignoremap := make(map[string]bool)
	for _, nick := range ignore {<commit_msg>urltitle: magic word for silence<commit_after>// Package urltitle implements a plugin to watch web URLs, fetch and display title.
package urltitle

import (
	"errors"
	"fmt"
	bot "github.com/StalkR/goircbot"
	irc "github.com/fluffle/goirc/client"
	"html"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strings"
	"time"
)

// When matched, urltitle do not read line.
var silenceRegexp = "(^|\\s)tg(\\s|$)"

func timeoutDialer(d time.Duration) func(net, addr string) (c net.Conn, err error) {
	return func(netw, addr string) (net.Conn, error) {
		return net.DialTimeout(netw, addr, d)
	}
}

// Title gets an URL and returns its title.
func Title(url string) (string, error) {
	client := &http.Client{
		Transport: &http.Transport{
			Dial: timeoutDialer(3 * time.Second),
		},
	}
	resp, err := client.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	r, err := regexp.Compile("<title[^>]*>([^<]+)<")
	if err != nil {
		return "", err
	}
	matches := r.FindSubmatch([]byte(contents))
	if len(matches) < 2 {
		return "", errors.New("no title found in page")
	}
	return html.UnescapeString(strings.TrimSpace(string(matches[1]))), nil
}

func watchLine(b *bot.Bot, line *irc.Line, ignoremap map[string]bool) {
	target := line.Args[0]
	if !strings.HasPrefix(target, "#") {
		return
	}
	if _, ignore := ignoremap[line.Nick]; ignore {
		return
	}
	text := line.Args[1]
	if m, err := regexp.Match(silenceRegexp, []byte(text)); err != nil || m {
		return
	}
	r, err := regexp.Compile("(?:^|\\s)(https?://[^\\s]+)")
	if err != nil {
		return
	}
	matches := r.FindSubmatch([]byte(text))
	if len(matches) < 2 {
		return
	}
	url := string(matches[1])
	title, err := Title(url)
	if err != nil {
		log.Println("urltitle:", err)
		return
	}
	if len(title) > 200 {
		title = title[:200]
	}
	b.Conn.Privmsg(target, fmt.Sprintf("%s :: %s", url, title))
}

// Register registers the plugin with a bot.
func Register(b *bot.Bot, ignore []string) {
	ignoremap := make(map[string]bool)
	for _, nick := range ignore {
{'fraction_non_alphanumeric': 0.1817957572767637, 'fraction_numerical': 0.006413418845584608, 'mean_word_length': 3.856287425149701, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4864998', 'n_tokens_mistral': 1617, 'n_tokens_neox': 1545, 'n_words': 391}