<commit_before>package slug

import (
	"testing"
)

var (
	tests = []struct {
		iS   string
		iL   int
		iSep string
		oS   string
	}{
		{"", 50, "-", ""},
		{" ", 50, "-", ""},
		{"a lovely day", 50, "-", "a-lovely-day"},
		{"a lovely day", 50, "123", "a123lovely123day"},
		{"A Lovely Day", 50, "-", "a-lovely-day"},
		{"a lovely day", 10, "-", "a-lovely-d"},
		{"a lovely day", 9, "-", "a-lovely"},
		{"a lovely day", 8, "-", "a-lovely"},
		{"wo ai 北京", 50, "-", "wo-ai-bei-jing"},
		{"wo ai 北京", 12, "-", "wo-ai-bei-ji"},
		{"wo ai 北京", 10, "-", "wo-ai-bei"},
		{"wo ai 北京", 10, "-", "wo-ai-bei"},
		{"wo ai 北京", 7, "-", "wo-ai-b"},
		{"wo ai 北京", 1, "-", "w"},
		{"-a -- b__c", 50, "-", "a-b-c"},
	}
)

func TestSlug(t *testing.T) {
	for _, test := range tests {
		if out := Slug(test.iS, test.iL, test.iSep); out != test.oS {
			t.Errorf("For input string '%s' and maximum length %d and separator '%s', "+
				"expected output '%s' (len %d), was '%s' (len %d)",
				test.iS, test.iL, test.iSep,
				test.oS, len(test.oS), out, len(out))
		}
	}
}<commit_msg>Add tests for skipped characters<commit_after>package slug

import (
	"testing"
)

var (
	tests = []struct {
		iS   string
		iL   int
		iSep string
		oS   string
	}{
		{"", 50, "-", ""},
		{" ", 50, "-", ""},
		{"a lovely day", 50, "-", "a-lovely-day"},
		{"a lovely day", 50, "123", "a123lovely123day"},
		{"A Lovely Day", 50, "-", "a-lovely-day"},
		{"a lovely day", 10, "-", "a-lovely-d"},
		{"a lovely day", 9, "-", "a-lovely"},
		{"a lovely day", 8, "-", "a-lovely"},
		{"wo ai 北京", 50, "-", "wo-ai-bei-jing"},
		{"wo ai 北京", 12, "-", "wo-ai-bei-ji"},
		{"wo ai 北京", 10, "-", "wo-ai-bei"},
		{"wo ai 北京", 10, "-", "wo-ai-bei"},
		{"wo ai 北京", 7, "-", "wo-ai-b"},
		{"wo ai 北京", 1, "-", "w"},
		{"<NAME>", 50, "-", "antonin-dvorak"},
		{"-a -- b__c", 50, "-", "a-b-c"},
		{"Schindler's List", 50, "-", "schindlers-list"},
		{"Schindler's List", 10, "-", "schindlers"},
	}
)

func TestSlug(t *testing.T) {
	for _, test := range tests {
		if out := Slug(test.iS, test.iL, test.iSep); out != test.oS {
			t.Errorf("For input string '%s' and maximum length %d and separator '%s', "+
				"expected output '%s' (len %d), was '%s' (len %d)",
				test.iS, test.iL, test.iSep,
				test.oS, len(test.oS), out, len(out))
		}
	}
}
{'fraction_non_alphanumeric': 0.287527352297593, 'fraction_numerical': 0.03326039387308534, 'mean_word_length': 2.835570469798658, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5765213', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1043, 'n_words': 261}