<commit_before>package url

import (
	"github.com/fabioxgn/go-bot"
	"github.com/fabioxgn/go-bot/web"
	"html"
	"net/url"
	"regexp"
	"strings"
)

const (
	minDomainLength = 3
)

func canBeURLWithouProtocol(text string) bool {
	return len(text) > minDomainLength &&
		!strings.HasPrefix(text, "http") &&
		strings.Contains(text, ".")
}

func extractURL(text string) string {
	extractedURL := ""
	for _, value := range strings.Split(text, " ") {
		if canBeURLWithouProtocol(value) {
			value = "http://" + value
		}

		parsedURL, err := url.Parse(value)
		if err != nil {
			continue
		}
		if strings.HasPrefix(parsedURL.Scheme, "http") {
			extractedURL = parsedURL.String()
			break
		}
	}
	return extractedURL
}

func getTitle(text string, get web.GetBodyFunc) (string, error) {
	URL := extractURL(text)

	if URL == "" {
		return "", nil
	}

	er := regexp.MustCompile("<title>(.*?)<\\/title>")
	body, err := get(URL)
	if err != nil {
		return "", err
	}

	title := er.FindString(string(body))
	if title == "" {
		return "", nil
	}

	title = title[strings.Index(title, ">")+1 : strings.LastIndex(title, "<")]
	return html.UnescapeString(title), nil
}

func urlTitle(command *bot.PassiveCmd) (string, error) {
	return getTitle(command.Raw, web.GetBody)
}

func init() {
	bot.RegisterPassiveCommand(
		"url",
		urlTitle)
}
<commit_msg>Fixed typo<commit_after>package url

import (
	"github.com/fabioxgn/go-bot"
	"github.com/fabioxgn/go-bot/web"
	"html"
	"net/url"
	"regexp"
	"strings"
)

const (
	minDomainLength = 3
)

func canBeURLWithoutProtocol(text string) bool {
	return len(text) > minDomainLength &&
		!strings.HasPrefix(text, "http") &&
		strings.Contains(text, ".")
}

func extractURL(text string) string {
	extractedURL := ""
	for _, value := range strings.Split(text, " ") {
		if canBeURLWithoutProtocol(value) {
			value = "http://" + value
		}

		parsedURL, err := url.Parse(value)
		if err != nil {
			continue
		}
		if strings.HasPrefix(parsedURL.Scheme, "http") {
			extractedURL = parsedURL.String()
			break
		}
	}
	return extractedURL
}

func getTitle(text string, get web.GetBodyFunc) (string, error) {
	URL := extractURL(text)

	if URL == "" {
		return "", nil
	}

	er := regexp.MustCompile("<title>(.*?)<\\/title>")
	body, err := get(URL)
	if err != nil {
		return "", err
	}

	title := er.FindString(string(body))
	if title == "" {
		return "", nil
	}

	title = title[strings.Index(title, ">")+1 : strings.LastIndex(title, "<")]
	return html.UnescapeString(title), nil
}

func urlTitle(command *bot.PassiveCmd) (string, error) {
	return getTitle(command.Raw, web.GetBody)
}

func init() {
	bot.RegisterPassiveCommand(
		"url",
		urlTitle)
}

{'fraction_non_alphanumeric': 0.16835776024051108, 'fraction_numerical': 0.001503194287861706, 'mean_word_length': 3.8933823529411766, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6675435', 'n_tokens_mistral': 1043, 'n_tokens_neox': 986, 'n_words': 201}