<commit_before>package baseconv

import (
	"strings"
	"testing"
)

func TestErrors(t *testing.T) {
	tests := []struct {
		val, from, to string
	}{
		{"", DigitsHex, DigitsDec},
		{"0", "", DigitsDec},
		{"0", DigitsDec, ""},
		{"bad", DigitsBin, DigitsDec},
		{"BAD", DigitsHex, DigitsDec},
	}

	for i, test := range tests {
		_, err := Convert(test.val, test.from, test.to)
		if err == nil {
			t.Errorf("test %d Convert(%s, %s, %s) should produce error", i, test.val, test.from, test.to)
		}
	}
}

const (
	DigitsJapanese = `〇一二三四五六七八九`
	DigitsThai     = `๐๑๒๓๔๕๖๗๘๙`
	Digits96a      = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz~`!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?¿¡"
	Digits96b      = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ~`!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?¿¡"
)

func TestConvert(t *testing.T) {
	tests := []struct {
		from, to, val, exp string
	}{
		{DigitsDec, DigitsBin, "0", "0"},
		{DigitsDec, DigitsBin, "8", "1000"},
		{DigitsDec, DigitsBin, "15", "1111"},
		{DigitsDec, DigitsBin, "16", "10000"},
		{DigitsDec, DigitsBin, "88", "1011000"},
		{DigitsDec, DigitsBin, "10000", "10011100010000"},
<commit_msg>Adding sanity check for 0 and 1s<commit_after>package baseconv

import (
	"strings"
	"testing"
)

func TestErrors(t *testing.T) {
	tests := []struct {
		val, from, to string
	}{
		{"", DigitsHex, DigitsDec},
		{"0", "", DigitsDec},
		{"0", DigitsDec, ""},
		{"bad", DigitsBin, DigitsDec},
		{"BAD", DigitsHex, DigitsDec},
	}

	for i, test := range tests {
		_, err := Convert(test.val, test.from, test.to)
		if err == nil {
			t.Errorf("test %d Convert(%s, %s, %s) should produce error", i, test.val, test.from, test.to)
		}
	}
}

const (
	DigitsJapanese = `〇一二三四五六七八九`
	DigitsThai     = `๐๑๒๓๔๕๖๗๘๙`
	Digits96a      = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz~`!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?¿¡"
	Digits96b      = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ~`!@#$%^&*()-_=+[{]}\\|;:'\",<.>/?¿¡"
)

func TestZeroes(t *testing.T) {
	z := strings.Repeat("0", 1000000)
	for i, base := range []string{DigitsBin, DigitsOct, DigitsDec, DigitsHex, Digits36, Digits62, Digits64, Digits96a, Digits96b} {
		v, err := Convert(z, base, DigitsDec)
		if err != nil {
			t.Fatalf("test %d base expected no error, got: %v", i, err)
		}
		if v != "0" {
			t.Errorf("test %d expected 0, got: %s", i, v)
		}
		v, err = Convert(z, DigitsDec, base)
		if err != nil {
			t.Fatalf("test %d base expected no error, got: %v", i, err)
		}
		if v != "0" {
			t.Errorf("test %d expected 0, got: %s", i, v)
		}
	}

	z += "1"
	for i, base := range []string{DigitsBin, DigitsOct, DigitsDec, DigitsHex, Digits36, Digits62, Digits64, Digits96a, Digits96b} {
		v, err := Convert(z, base, DigitsDec)
		if err != nil {
			t.Fatalf("test %d base expected no error, got: %v", i, err)
		}
		if v != "1" {
			t.Errorf("test %d expected 1, got: %s", i, v)
		}
		v, err = Convert(z, DigitsDec, base)
		if err != nil {
			t.Fatalf("test %d base expected no error, got: %v", i, err)
		}
		if v != "1" {
			t.Errorf("test %d expected 1, got: %s", i, v)
		}
	}
}

func TestConvert(t *testing.T) {
	tests := []struct {
		from, to, val, exp string
	}{
		{DigitsDec, DigitsBin, "0", "0"},
		{DigitsDec, DigitsBin, "8", "1000"},
		{DigitsDec, DigitsBin, "15", "1111"},
		{DigitsDec, DigitsBin, "16", "10000"},
		{DigitsDec, DigitsBin, "88", "1011000"},
		{DigitsDec, DigitsBin, "10000", "10011100010000"},

{'fraction_non_alphanumeric': 0.21098265895953758, 'fraction_numerical': 0.05982658959537572, 'mean_word_length': 3.965566714490674, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5238755', 'n_tokens_mistral': 1776, 'n_tokens_neox': 1555, 'n_words': 342}