<commit_before>'use strict';
var _ = require('lodash');
var sjcl = require('./utils').sjcl;
var utils = require('./utils');
var extend = require('extend');
var convertBase = require('./baseconverter');

var Base = {};

var alphabets = Base.alphabets = {
  ripple: 'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz',
  tipple: 'RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz',
  bitcoin: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
};

extend(Base, {
  VER_NONE: 1,
  VER_NODE_PUBLIC: 28,
  VER_NODE_PRIVATE: 32,
  VER_ACCOUNT_ID: 0,
  VER_ACCOUNT_PUBLIC: 35,
  VER_ACCOUNT_PRIVATE: 34,
  VER_FAMILY_GENERATOR: 41,
  VER_FAMILY_SEED: 33
});

function sha256(bytes) {
  return sjcl.codec.bytes.fromBits(
    sjcl.hash.sha256.hash(sjcl.codec.bytes.toBits(bytes)));
}

function encodeString(alphabet, input) {
  if (input.length === 0) {
    return '';
  }

  var leadingZeros = _.takeWhile(input, function(d) {
    return d === 0;
  });
  var out = convertBase(input, 256, 58).map(function(digit) {
    if (digit < 0 || digit >= alphabet.length) {
      throw new Error('Value ' + digit + ' is out of bounds for encoding');
    }
    return alphabet[digit];
  });
  var prefix = leadingZeros.map(function() {
    return alphabet[0];
  });
  return prefix.concat(out).join('');
}

function decodeString(indexes, input) {
  if (input.length === 0) {
    return [];
  }

  var input58 = input.split('').map(function(c) {
    var charCode = c.charCodeAt(0);
    if (charCode >= indexes.length) {
      throw new Error('Character ' + c + ' is not valid for encoding');
    }
    return indexes[charCode];
  });<commit_msg>[FIX] fix Base:decode<commit_after>'use strict';
var _ = require('lodash');
var sjcl = require('./utils').sjcl;
var utils = require('./utils');
var extend = require('extend');
var convertBase = require('./baseconverter');

var Base = {};

var alphabets = Base.alphabets = {
  ripple: 'rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz',
  tipple: 'RPShNAF39wBUDnEGHJKLM4pQrsT7VWXYZ2bcdeCg65jkm8ofqi1tuvaxyz',
  bitcoin: '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
};

extend(Base, {
  VER_NONE: 1,
  VER_NODE_PUBLIC: 28,
  VER_NODE_PRIVATE: 32,
  VER_ACCOUNT_ID: 0,
  VER_ACCOUNT_PUBLIC: 35,
  VER_ACCOUNT_PRIVATE: 34,
  VER_FAMILY_GENERATOR: 41,
  VER_FAMILY_SEED: 33
});

function sha256(bytes) {
  return sjcl.codec.bytes.fromBits(
    sjcl.hash.sha256.hash(sjcl.codec.bytes.toBits(bytes)));
}

function encodeString(alphabet, input) {
  if (input.length === 0) {
    return '';
  }

  var leadingZeros = _.takeWhile(input, function(d) {
    return d === 0;
  });
  var out = convertBase(input, 256, 58).map(function(digit) {
    if (digit < 0 || digit >= alphabet.length) {
      throw new Error('Value ' + digit + ' is out of bounds for encoding');
    }
    return alphabet[digit];
  });
  var prefix = leadingZeros.map(function() {
    return alphabet[0];
  });
  return prefix.concat(out).join('');
}

function decodeString(indexes, input) {
  if (input.length === 0) {
    return [];
  }

  var input58 = input.split('').map(function(c) {
    var charCode = c.charCodeAt(0);
    if (charCode >= indexes.length || indexes[charCode] === -1) {
      throw new Error('Character ' + c + ' is not valid for encoding');
    }
    return indexes[charCode];
  });
{'fraction_non_alphanumeric': 0.14931178934769598, 'fraction_numerical': 0.03620586475164572, 'mean_word_length': 4.127300613496932, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6472999', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1266, 'n_words': 242}