<commit_before>#!/usr/bin/perl
#
# mySociety/BaseN.pm:
# Arbitrary base-N encodings (like base64 etc.); and a pseudo-base-N encoding
# (a generalisation of Adobe's ASCII85) which is much faster.
#
# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved.
# Email: <EMAIL>; WWW: http://www.mysociety.org/
#
# $Id: BaseN.pm,v 1.1 2006-11-19 23:59:21 chris Exp $
#

package mySociety::BaseN;

use strict;

#
# Theory: obviously we can take an arbitrary binary string, treat it as an
# integer of arbitrary size, and turn it into a corresponding integer in
# another base. This is what the encode and decode functions below do. Note the
# subtlety that as well as the integer itself, we must communicate the total
# number of bytes encoded, so that we can reproduce the input string
# faithfully. This information is communicated implicitly in the length of the
# output string. For instance, in a naive encoding in the base 64 alphabet (in
# which "A" represents 0) you would expect "A", "AA", ... all to represent the
# number zero, and therefore a string of arbitrarily many zero bits. To resolve
# this ambiguity the length of the encoded string is used to communicate the
# length of the input; in MIME-standard base64, all output is padded to be a
# multiple of four output symbols, and "AA==" represents the byte 00, "AAA="
# the bytes 00 00, and "AAAA" the bytes 00 00 00; but we don't need the padding
# symbols, so we can just encode those cases as "AA", "AAA", and "AAAA".
#
# The pseudo-base-N encodings used by encodefast/decodefast divide the input
# message up into four-byte blocks (with potentially a short trailer) and
# base-N encode each block. Obviously this avoids the need for
# arbitrary-precision arithmetic, and is therefore much faster than the true
# base-N encodings. The implementation is made slightly hairy by perl foibles;
# if it gives trouble is should be replaced with one in C.
# 

use Carp;
use Math::BigInt lib => 'GMP';

my $std_alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';

my %std_alpha_key;
for (my $i = 0; $i < length($std_alpha); ++$i) {
    $std_alpha_key{substr($std_alpha, $i, 1)} = $i;
}

sub lg ($) {
    return log($_[0]) / log(2);
}

=item encode N MESSAGE [ALPHABET]

Encode the MESSAGE in base-N, using the given ALPHABET (or a standard one if it
is not specified).

=cut
sub encode ($$;$) {
    my $n = shift;
    croak "N must be a positive integer" unless ($n =~ /^[1-9]\d*$/);
    my $message = shift;
    croak "MESSAGE may not be undef" unless (defined($message));
    my $alphabet = $std_alpha;
    if (@_) {
        $alphabet = shift;
        croak "ALPHABET may not be undef" unless (defined($alphabet));
        croak "ALPHABET must contain $n characters"
            unless (length($alphabet) == $n);
    } elsif ($n > length($alphabet)) {
        croak "not enough characters in standard alphabet -- supply your own";
    } else {
        $alphabet = substr($alphabet, 0, $n);
    }

    # Number of output symbols to generate. Obviously each input symbol encodes
    # 8 bits, and each output symbol lg(N) bits; round up to get the number of
    # output symbols required.
    my $noutput = int((length($message) * 8) / lg($n) + 1);
    
    my $x = new Math::BigInt();
    for (my $i = 0; $i < length($message); ++$i) {
        $x->blsft(8);
        $x->bior(unpack('C', substr($message, $i, 1)));
    }
    my $res = '';
    for (my $i = 0; $i < $noutput; ++$i) {
        my ($quo, $rem) = $x->bdiv($n);
        $res .= substr($alphabet, $rem, 1);
    }

    return scalar(reverse($res));
}


=item decode N MESSAGE [ALPHABET]

Decode the base-N MESSAGE, using the given ALPHABET (or a standard one if it is
not specified). Returns undef if MESSAGE is invalid (contains characters not in
ALPHABET).

=cut
sub decode ($$;$) {
    my $n = shift;<commit_msg>Use base62 tokens; noddy PHP implementation, not as complete as the Perl module.<commit_after>#!/usr/bin/perl
#
# mySociety/BaseN.pm:
# Arbitrary base-N encodings (like base64 etc.); and a pseudo-base-N encoding
# (a generalisation of Adobe's ASCII85) which is much faster.
#
# Copyright (c) 2006 UK Citizens Online Democracy. All rights reserved.
# Email: <EMAIL>; WWW: http://www.mysociety.org/
#
# $Id: BaseN.pm,v 1.2 2006-11-20 15:12:23 matthew Exp $
#

package mySociety::BaseN;

use strict;

#
# Theory: obviously we can take an arbitrary binary string, treat it as an
# integer of arbitrary size, and turn it into a corresponding integer in
# another base. This is what the encode and decode functions below do. Note the
# subtlety that as well as the integer itself, we must communicate the total
# number of bytes encoded, so that we can reproduce the input string
# faithfully. This information is communicated implicitly in the length of the
# output string. For instance, in a naive encoding in the base 64 alphabet (in
# which "A" represents 0) you would expect "A", "AA", ... all to represent the
# number zero, and therefore a string of arbitrarily many zero bits. To resolve
# this ambiguity the length of the encoded string is used to communicate the
# length of the input; in MIME-standard base64, all output is padded to be a
# multiple of four output symbols, and "AA==" represents the byte 00, "AAA="
# the bytes 00 00, and "AAAA" the bytes 00 00 00; but we don't need the padding
# symbols, so we can just encode those cases as "AA", "AAA", and "AAAA".
#
# The pseudo-base-N encodings used by encodefast/decodefast divide the input
# message up into four-byte blocks (with potentially a short trailer) and
# base-N encode each block. Obviously this avoids the need for
# arbitrary-precision arithmetic, and is therefore much faster than the true
# base-N encodings. The implementation is made slightly hairy by perl foibles;
# if it gives trouble is should be replaced with one in C.
# 

use Carp;
use Math::BigInt lib => 'GMP';
use POSIX qw(ceil);

my $std_alpha = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';

my %std_alpha_key;
for (my $i = 0; $i < length($std_alpha); ++$i) {
    $std_alpha_key{substr($std_alpha, $i, 1)} = $i;
}

sub lg ($) {
    return log($_[0]) / log(2);
}

=item encode N MESSAGE [ALPHABET]

Encode the MESSAGE in base-N, using the given ALPHABET (or a standard one if it
is not specified).

=cut
sub encode ($$;$) {
    my $n = shift;
    croak "N must be a positive integer" unless ($n =~ /^[1-9]\d*$/);
    my $message = shift;
    croak "MESSAGE may not be undef" unless (defined($message));
    my $alphabet = $std_alpha;
    if (@_) {
        $alphabet = shift;
        croak "ALPHABET may not be undef" unless (defined($alphabet));
        croak "ALPHABET must contain $n characters"
            unless (length($alphabet) == $n);
    } elsif ($n > length($alphabet)) {
        croak "not enough characters in standard alphabet -- supply your own";
    } else {
        $alphabet = substr($alphabet, 0, $n);
    }

    # Number of output symbols to generate. Obviously each input symbol encodes
    # 8 bits, and each output symbol lg(N) bits; round up to get the number of
    # output symbols required.
    my $noutput = ceil((length($message) * 8) / lg($n));
    
    my $x = new Math::BigInt();
    for (my $i = 0; $i < length($message); ++$i) {
        $x->blsft(8);
        $x->bior(unpack('C', substr($message, $i, 1)));
    }
    my $res = '';
    for (my $i = 0; $i < $noutput; ++$i) {
        my ($quo, $rem) = $x->bdiv($n);
        $res .= substr($alphabet, $rem, 1);
    }

    return scalar(reverse($res));
}


=item decode N MESSAGE [ALPHABET]

Decode the base-N MESSAGE, using the given ALPHABET (or a standard one if it is
not specified). Returns undef if MESSAGE is invalid (contains characters not in
ALPHABET).

=cut
sub decode ($$;$) {
    my $n = shift;
{'fraction_non_alphanumeric': 0.11706755373592631, 'fraction_numerical': 0.01701637666325486, 'mean_word_length': 3.4745277618775043, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288982', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2460, 'n_words': 1053}