<commit_before>var createHash = require('create-hash')
var pbkdf2 = require('pbkdf2').pbkdf2Sync
var randomBytes = require('randombytes')

// use unorm until String.prototype.normalize gets better browser support
var unorm = require('unorm')

function lpad (str, padString, length) {
  while (str.length < length) str = padString + str
  return str
}

var ENGLISH_WORDLIST = require('./wordlists/english.json')
var FRENCH_WORDLIST = require('./wordlists/french.json')
var ITALIAN_WORDLIST = require('./wordlists/italian.json')
var JAPANESE_WORDLIST = require('./wordlists/japanese.json')
var SPANISH_WORDLIST = require('./wordlists/spanish.json')
var DEFAULT_WORDLIST = ENGLISH_WORDLIST

var INVALID_MNEMONIC = 'Invalid mnemonic'
var INVALID_ENTROPY = 'Invalid entropy'

function salt (password) {
  return 'mnemonic' + (password || '')
}

function mnemonicToSeed (mnemonic, password) {
  var mnemonicBuffer = new Buffer(unorm.nfkd(mnemonic), 'utf8')
  var saltBuffer = new Buffer(salt(unorm.nfkd(password)), 'utf8')

  return pbkdf2(mnemonicBuffer, saltBuffer, 2048, 64, 'sha512')
}

function mnemonicToSeedHex (mnemonic, password) {
  return mnemonicToSeed(mnemonic, password).toString('hex')
}

function mnemonicToEntropy (mnemonic, wordlist) {
  wordlist = wordlist || DEFAULT_WORDLIST

  var words = unorm.nfkd(mnemonic).split(' ')
  if (words.length % 3 !== 0) throw new Error(INVALID_MNEMONIC)
  if (words.some(function (word) {
    return wordlist.indexOf(word) === -1
  })) throw new Error(INVALID_MNEMONIC)

  // convert word indices to 11 bit binary strings
  var bits = words.map(function (word) {
    var index = wordlist.indexOf(word)
    return lpad(index.toString(2), '0', 11)
  }).join('')

  if (bits.length < 128) throw new Error(INVALID_MNEMONIC)
  if (bits.length > 264) throw new Error(INVALID_MNEMONIC)

  // split the binary string into ENT/CS
  var dividerIndex = Math.floor(bits.length / 33) * 32
  var entropy = bits.slice(0, dividerIndex)
  var checksum = bits.slice(dividerIndex)
<commit_msg>remove double iteration of words<commit_after>var createHash = require('create-hash')
var pbkdf2 = require('pbkdf2').pbkdf2Sync
var randomBytes = require('randombytes')

// use unorm until String.prototype.normalize gets better browser support
var unorm = require('unorm')

function lpad (str, padString, length) {
  while (str.length < length) str = padString + str
  return str
}

var ENGLISH_WORDLIST = require('./wordlists/english.json')
var FRENCH_WORDLIST = require('./wordlists/french.json')
var ITALIAN_WORDLIST = require('./wordlists/italian.json')
var JAPANESE_WORDLIST = require('./wordlists/japanese.json')
var SPANISH_WORDLIST = require('./wordlists/spanish.json')
var DEFAULT_WORDLIST = ENGLISH_WORDLIST

var INVALID_MNEMONIC = 'Invalid mnemonic'
var INVALID_ENTROPY = 'Invalid entropy'

function salt (password) {
  return '<PASSWORD>' + (password || '')
}

function mnemonicToSeed (mnemonic, password) {
  var mnemonicBuffer = new Buffer(unorm.nfkd(mnemonic), 'utf8')
  var saltBuffer = new Buffer(salt(unorm.nfkd(password)), 'utf8')

  return pbkdf2(mnemonicBuffer, saltBuffer, 2048, 64, 'sha512')
}

function mnemonicToSeedHex (mnemonic, password) {
  return mnemonicToSeed(mnemonic, password).toString('hex')
}

function mnemonicToEntropy (mnemonic, wordlist) {
  wordlist = wordlist || DEFAULT_WORDLIST

  var words = unorm.nfkd(mnemonic).split(' ')
  if (words.length % 3 !== 0) throw new Error(INVALID_MNEMONIC)

  // convert word indices to 11 bit binary strings
  var bits = words.map(function (word) {
    var index = wordlist.indexOf(word)
    if (index === -1) throw new Error(INVALID_MNEMONIC)

    return lpad(index.toString(2), '0', 11)
  }).join('')

  if (bits.length < 128) throw new Error(INVALID_MNEMONIC)
  if (bits.length > 264) throw new Error(INVALID_MNEMONIC)

  // split the binary string into ENT/CS
  var dividerIndex = Math.floor(bits.length / 33) * 32
  var entropy = bits.slice(0, dividerIndex)
  var checksum = bits.slice(dividerIndex)

{'fraction_non_alphanumeric': 0.12487512487512488, 'fraction_numerical': 0.017482517482517484, 'mean_word_length': 4.968703427719821, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6751489', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1367, 'n_words': 337}