<commit_before>/*
 Handles all of the rewriting tasks
*/
package rewrite

import (
	"log"
	"net/http"
	"strings"
)

var (
	linkPrefixes = []string{"t.co", "cl.ly", "www", "bit.ly", "j.mp", "tcrn.ch", "http"}
	httpClient   = &http.Client{}
)

// GetRewrittenLinks takes an input line and rewrite any links that are shortened links into their full representation
// the return value is a slice of those rewritten links
func GetRewrittenLinks(input string) (result []string) {
	for _, link := range strings.Split(input, " ") {
		rewritten, err := expandUrl(link)
		if err == nil && rewritten != "" {
			result = append(result, rewritten)
		}
	}
	return
}

// Rewrite takes an input string, tokenizes it on whitespace, and then attempte to rewrite
// each token. The final result is joined back together at the end
func Rewrite(input string) string {
	tokens := strings.Split(input, " ")
	for idx, token := range tokens {
		rewritten, err := expandUrl(token)
		if err == nil && rewritten != "" {
			tokens[idx] = rewritten
		}
	}
	return strings.Join(tokens, " ")
}

// expandUrl is a rewriter that expands shortened links
func expandUrl(url string) (result string, err error) {
	for _, prefix := range linkPrefixes {
		if strings.HasPrefix(url, prefix) {
			break
		}
		return "", nil
	}

	if !strings.HasPrefix(url, "http") {
		log.Printf("Adding HTTP to url %s\n", url)
		url = "http://" + url
	}
	log.Printf("Expanding link %s\n", url)
	resp, err := httpClient.Head(url) // will follow redirects
	if err != nil {
		return "", err
	}
	defer resp.Body.Close() // not sure if i have to do this with a head response
	expanded := resp.Request.URL.String()
	if expanded != url {
		return expanded, nil
	}
	return "", nil
}<commit_msg>Create expander type to support other expansions<commit_after>/*
 Handles all of the rewriting tasks
*/
package rewrite

import (
	"log"
	"net/http"
	"strings"
)

var (
	linkPrefixes = []string{"t.co", "cl.ly", "www", "bit.ly", "j.mp", "tcrn.ch", "http"}
	httpClient   = &http.Client{}
	expanders    = []expander{expandUrl}
)

// GetRewrittenLinks takes an input line and rewrite any links that are shortened links into their full representation
// the return value is a slice of those rewritten links
func GetRewrittenLinks(input string) (result []string) {
	for _, link := range strings.Split(input, " ") {
		rewritten, err := expandAll(link)
		if err == nil && rewritten != "" {
			result = append(result, rewritten)
		}
	}
	return
}

// Rewrite takes an input string, tokenizes it on whitespace, and then attempte to rewrite
// each token. The final result is joined back together at the end
func Rewrite(input string) string {
	tokens := strings.Split(input, " ")
	for idx, token := range tokens {
		rewritten, err := expandAll(token)
		if err == nil && rewritten != "" {
			tokens[idx] = rewritten
		}
	}
	return strings.Join(tokens, " ")
}

// expandUrl is a rewriter that expands shortened links
func expandUrl(url string) (result string, err error) {
	for _, prefix := range linkPrefixes {
		if strings.HasPrefix(url, prefix) {
			break
		}
		return "", nil
	}

	if !strings.HasPrefix(url, "http") {
		log.Printf("Adding HTTP to url %s\n", url)
		url = "http://" + url
	}
	log.Printf("Expanding link %s\n", url)
	resp, err := httpClient.Head(url) // will follow redirects
	if err != nil {
		return "", err
	}
	defer resp.Body.Close() // not sure if i have to do this with a head response
	expanded := resp.Request.URL.String()
	if expanded != url {
		return expanded, nil
	}
	return "", nil
}

// an expander is something that takes in a string and possibly expands it
type expander func(string) (string, error)

// thoroughly expand the input string by running it through the expander functions
func expandAll(input string) (string, error) {
	known := make(map[string]bool) // to track what we've seen already
	current := input
	known[current] = true
	for {
		rewritten := false
		for _, fn := range expanders {
			if result, err := fn(input); result != "" && err == nil {
				if known[result] {
					break
				}
				current = result
				known[current] = true
				rewritten = true
				break
			}
		}
		if !rewritten {
			break
		}
	}
	if current == input {
		return "", nil
	}
	return current, nil
}
{'fraction_non_alphanumeric': 0.12933679490205333, 'fraction_numerical': 0.0, 'mean_word_length': 3.4846560846560846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612506', 'n_tokens_mistral': 1466, 'n_tokens_neox': 1344, 'n_words': 480}