<commit_before>/* 
 * Package normalize normalizes URLs
 */
package normalize

import (
	"os"
	"regexp"
	"sort"
	"strings"
	"url"
)

var ipv6Regexp, _ = regexp.Compile("([0-9A-F]+:)+[0-9A-F]+")
var ipv4Regexp, _ = regexp.Compile("([0-9]+[.])+[0-9]+")

func Normalize(url *url.URL) (err os.Error) {
	addSlash(url)
	removeDefaultPort(url)
	lowerCaseScheme(url)
	lowerCaseDomain(url)
	removeDoubleSlashes(url)
	removeDirectoryDots(url)
	return nil
}

func removeDirectoryDots(url *url.URL) {
	url.Path = strings.Replace(url.Path, "/./", "/", -1)
	url.Path = strings.Replace(url.Path, "/../", "/", -1)
}

func removeDoubleSlashes(url *url.URL) {
	url.Path = strings.Replace(url.Path, "//", "/", -1)
}

func lowerCaseDomain(url *url.URL) {
	url.Host = strings.ToLower(url.Host)
}

func lowerCaseScheme(url *url.URL) {
	url.Scheme = strings.ToLower(url.Scheme)
}

func removeDefaultPort(url *url.URL) {
	/*<commit_msg>No point returning an error then.<commit_after>/* 
 * Package normalize normalizes URLs
 */
package normalize

import (
	"regexp"
	"sort"
	"strings"
	"url"
)

var ipv6Regexp, _ = regexp.Compile("([0-9A-F]+:)+[0-9A-F]+")
var ipv4Regexp, _ = regexp.Compile("([0-9]+[.])+[0-9]+")

func Normalize(url *url.URL) {
	addSlash(url)
	removeDefaultPort(url)
	lowerCaseScheme(url)
	lowerCaseDomain(url)
	removeDoubleSlashes(url)
	removeDirectoryDots(url)
}

func removeDirectoryDots(url *url.URL) {
	url.Path = strings.Replace(url.Path, "/./", "/", -1)
	url.Path = strings.Replace(url.Path, "/../", "/", -1)
}

func removeDoubleSlashes(url *url.URL) {
	url.Path = strings.Replace(url.Path, "//", "/", -1)
}

func lowerCaseDomain(url *url.URL) {
	url.Host = strings.ToLower(url.Host)
}

func lowerCaseScheme(url *url.URL) {
	url.Scheme = strings.ToLower(url.Scheme)
}

func removeDefaultPort(url *url.URL) {
	/*
{'fraction_non_alphanumeric': 0.1965764770844837, 'fraction_numerical': 0.01435670900055218, 'mean_word_length': 4.408955223880597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7266406', 'n_tokens_mistral': 747, 'n_tokens_neox': 738, 'n_words': 120}