<commit_before>package clio

import (
	"net/http"
	"regexp"
	"strings"

	"github.com/davidmz/mustbe"
	"github.com/juju/errors"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var urlLikeRe = regexp.MustCompile(`^(https?|ftp)://`)

func deHTML(text string) (result string, links []string) {
	var (
		inAnchor bool
		aText    string
		aTitle   string
		aHref    string
	)
	z := html.NewTokenizer(strings.NewReader(text))
	for {
		tt := z.Next()
		switch tt {

		case html.ErrorToken:
			return

		case html.StartTagToken:
			t := z.Token()
			if t.DataAtom != atom.A {
				mustbe.OK(errors.Errorf("unexpected HTML tag %q", t.DataAtom))
			}
			inAnchor = true
			aText = ""
			aTitle = ""
			aHref = ""
			for _, a := range t.Attr {
				switch a.Key {
				case "title":
					aTitle = a.Val
				case "href":
					aHref = a.Val
				}
			}

		case html.TextToken:
			t := z.Token()
			if inAnchor {
				aText += t.Data
			} else {
				result += t.Data
			}

		case html.EndTagToken:
			inAnchor = false
			if !urlLikeRe.MatchString(aText) {
				result += aText
			} else if aTitle != "" {
				links = append(links, aTitle)
				result += aTitle
			} else {
				url := unshorten(aHref)
				links = append(links, url)
				result += url
			}
		}
	}
}

func unshorten(u string) string {
	isShort := strings.HasPrefix(u, "http://t.co/")
	if !isShort {
		return u
	}
	resp, err := http.Get(u) // not http.Head because of https://github.com/golang/go/issues/19895
	if err != nil {
		return u
	}
	resp.Body.Close()
	return resp.Request.URL.String()
}<commit_msg>Add more shortener services to unshortener<commit_after>package clio

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/davidmz/mustbe"
	"github.com/juju/errors"
	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

var urlLikeRe = regexp.MustCompile(`^(https?|ftp)://`)

func deHTML(text string) (result string, links []string) {
	var (
		inAnchor bool
		aText    string
		aTitle   string
		aHref    string
	)
	z := html.NewTokenizer(strings.NewReader(text))
	for {
		tt := z.Next()
		switch tt {

		case html.ErrorToken:
			return

		case html.StartTagToken:
			t := z.Token()
			if t.DataAtom != atom.A {
				mustbe.OK(errors.Errorf("unexpected HTML tag %q", t.DataAtom))
			}
			inAnchor = true
			aText = ""
			aTitle = ""
			aHref = ""
			for _, a := range t.Attr {
				switch a.Key {
				case "title":
					aTitle = a.Val
				case "href":
					aHref = a.Val
				}
			}

		case html.TextToken:
			t := z.Token()
			if inAnchor {
				aText += t.Data
			} else {
				result += t.Data
			}

		case html.EndTagToken:
			inAnchor = false
			if !urlLikeRe.MatchString(aText) {
				result += aText
			} else if aTitle != "" {
				links = append(links, aTitle)
				result += aTitle
			} else {
				url := unshorten(aHref)
				links = append(links, url)
				result += url
			}
		}
	}
}

var shortDomains = []string{
	"t.co",
	"bit.ly",
	"bitly.com",
	"bitly.is",
	"j.mp",
	"goo.gl",
	"tinyurl.com",
	"ow.ly",
}

func unshorten(u string) string {
	pURL, err := url.Parse(u)
	if err != nil {
		return u
	}
	isShort := false
	for _, domain := range shortDomains {
		if pURL.Hostname() == domain {
			isShort = true
			break
		}
	}
	if !isShort {
		return u
	}

	// turn off log because of https://github.com/golang/go/issues/19895
	log.SetOutput(ioutil.Discard)
	resp, err := http.Head(u)
	log.SetOutput(os.Stderr)
	if err != nil {
		return u
	}
	resp.Body.Close()
	return resp.Request.URL.String()
}
{'fraction_non_alphanumeric': 0.1526150328665333, 'fraction_numerical': 0.002857959416976279, 'mean_word_length': 2.58974358974359, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3358961', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1393, 'n_words': 304}