<commit_before>use v6;
unit class Hashids;

# An alphabet is a str at least 16 chars long, has no spaces and only has
# unique characters
subset Alphabet of Str where !.comb.repeated and .chars >= 16;
subset Salt of Str where .chars > 0;
subset PositiveInt of Int where * >= 0;

constant $RATIO-SEPARATORS = 3.5;
constant $RATIO-GUARDS = 12;

constant $DEFAULT_ALPHABET = ('a'…'z', 'A'…'Z', '1'…'9', '0').join;
constant $DEFAULT_SEPARATORS = <cfhistuCFHISTU>;

has Salt $.salt;
has Alphabet $.alphabet;
has Int $.min-hash-length where * >= 0;
has Str $.separators where !.comb.repeated;
has Str $.guards;

method new(Salt $salt, Alphabet :$alphabet? = $DEFAULT_ALPHABET,
           Int :$min-hash-length? = 0, Str :$separators? = $DEFAULT_SEPARATORS) {
    my $s = remove-str($separators, ($alphabet.comb.Bag ∖ $separators.comb.Bag).keys.join);
    my $a = remove-str($alphabet, $s);
    my $min-separators = round($a.chars / $RATIO-GUARDS);
    if !$s || $s.elems < $min-separators {
        $min-separators = 2 if $min-separators = 1;
    }

    $s = consistent-shuffle($s, $salt);
    $a = consistent-shuffle($a, $salt);

    my $num-guards = round($a.chars / $RATIO-GUARDS);
    my Str $guards;
    if $num-guards < 3 {
        $guards = $s.comb[0..^$num-guards].join;
        $s = $s.comb[$num-guards..*].join;
    } else {
        $guards = $a.comb[0..^$num-guards].join;
        $a = $a.comb[$num-guards..*].join;
    }
    return self.bless(:$salt, :alphabet($a), :$min-hash-length, :separators($s), :$guards);
}

our sub remove-str(Str $alphabet, Str $separators) {
    my $a = "";
    for $alphabet.comb -> $c {
        $a ~= $c unless $separators.index($c).defined;
    }
    return $a;
}

method encode(*@numbers where .all() >= 0) returns Str {
        my $alphabet = $.alphabet;
        my $len-alphabet = $alphabet.chars;
        my $len-separators = self.separators.chars;
        my $values-hash =  [+]  @numbers.pairs.map: { .value % (.key + 100) };
        my $lottery = self.alphabet.comb[$values-hash % self.alphabet.chars];
        my $encoded = $lottery;
        for @numbers.kv -> $index, $number {
            my $alphabet-salt = play-lottery($lottery, $.salt, $alphabet);
            $alphabet = consistent-shuffle($alphabet, $alphabet-salt);
            my $last = hash($number, $alphabet);
            $encoded ~= $last;
            my $new-index = (($number % $last[0].ord) + $index) % $len-separators;
            $encoded ~= $.separators.comb[$new-index];
        }
        $encoded = $encoded.comb[0..*-2].join;
        return $encoded if $encoded.chars >= $!min-hash-length;
        return self!ensure-length($encoded, $alphabet, $values-hash);
        die <This shouldn't happen>;
}

our sub play-lottery($lottery, $salt, $alphabet) {
    return ($lottery ~ $salt ~ $alphabet).comb[0..^$alphabet.chars].join
}

method decode(Str $id) {
}

method !ensure-length(Str $string, Str $alphabet, Int $values-hash) returns Str {
    my $len-separators = $!separators.chars;
    my $index = ($values-hash + $string.comb[0].ord) % $len-separators;
    my $encoded = $!separators.comb[$index] ~ $string;
<commit_msg>Implement multiple values encoding<commit_after>use v6;
unit class Hashids;

# An alphabet is a str at least 16 chars long, has no spaces and only has
# unique characters
subset Alphabet of Str where !.comb.repeated and .chars >= 16;
subset Salt of Str where .chars > 0;
subset PositiveInt of Int where * >= 0;

constant $RATIO-SEPARATORS = 3.5;
constant $RATIO-GUARDS = 12;

constant $DEFAULT_ALPHABET = ('a'…'z', 'A'…'Z', '1'…'9', '0').join;
constant $DEFAULT_SEPARATORS = <cfhistuCFHISTU>;

has Salt $.salt;
has Alphabet $.alphabet;
has Int $.min-hash-length where * >= 0;
has Str $.separators where !.comb.repeated;
has Str $.guards;

method new(Salt $salt, Alphabet :$alphabet? = $DEFAULT_ALPHABET,
           Int :$min-hash-length? = 0, Str :$separators? = $DEFAULT_SEPARATORS) {
    my $s = remove-str($separators, ($alphabet.comb.Bag ∖ $separators.comb.Bag).keys.join);
    my $a = remove-str($alphabet, $s);
    my $min-separators = round($a.chars / $RATIO-GUARDS);
    if !$s || $s.elems < $min-separators {
        $min-separators = 2 if $min-separators = 1;
    }

    $s = consistent-shuffle($s, $salt);
    $a = consistent-shuffle($a, $salt);

    my $num-guards = round($a.chars / $RATIO-GUARDS);
    my Str $guards;
    if $num-guards < 3 {
        $guards = $s.comb[0..^$num-guards].join;
        $s = $s.comb[$num-guards..*].join;
    } else {
        $guards = $a.comb[0..^$num-guards].join;
        $a = $a.comb[$num-guards..*].join;
    }
    return self.bless(:$salt, :alphabet($a), :$min-hash-length, :separators($s), :$guards);
}

our sub remove-str(Str $alphabet, Str $separators) {
    my $a = "";
    for $alphabet.comb -> $c {
        $a ~= $c unless $separators.index($c).defined;
    }
    return $a;
}

method encode(*@numbers where .all() >= 0) returns Str {
        my $alphabet = $.alphabet;
        my $len-alphabet = $alphabet.chars;
        my $len-separators = self.separators.chars;
        my $values-hash =  [+]  @numbers.pairs.map: { .value % (.key + 100) };
        my $lottery = self.alphabet.comb[$values-hash % self.alphabet.chars];
        my $encoded = $lottery;
        for @numbers.kv -> $index, $number {
            my $alphabet-salt = play-lottery($lottery, $.salt, $alphabet);
            $alphabet = consistent-shuffle($alphabet, $alphabet-salt);
            my $last = hash($number, $alphabet);
            $encoded ~= $last;
            my $new-index = ($number % ($last[0].ord + $index)) % $len-separators;
            $encoded ~= $.separators.comb[$new-index];
        }
        $encoded = $encoded.comb[0..*-2].join;

        return $encoded if $encoded.chars >= $.min-hash-length;
        return self!ensure-length($encoded, $alphabet, $values-hash);
        die <This shouldn't happen>;
}

our sub play-lottery($lottery, $salt, $alphabet) {
    return ($lottery ~ $salt ~ $alphabet).comb[0..^$alphabet.chars].join
}

method decode(Str $id) {
}

method !ensure-length(Str $string, Str $alphabet, Int $values-hash) returns Str {
    my $len-separators = $!separators.chars;
    my $index = ($values-hash + $string.comb[0].ord) % $len-separators;
    my $encoded = $!separators.comb[$index] ~ $string;

{'fraction_non_alphanumeric': 0.17904761904761904, 'fraction_numerical': 0.009523809523809525, 'mean_word_length': 2.8991336633663365, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2172748', 'n_tokens_mistral': 2334, 'n_tokens_neox': 2235, 'n_words': 533}