<commit_before>import _ from 'lodash'

const separatorChar = '▁'

export function generateNoneUtterances(utterances: string[][], count: number): string[][] {
  const plainTokens = _.flatten(utterances)
    .map(x => x.replace(separatorChar, '')) // We want to discover real language-specific chars
    .filter(x => x.length > 1) // We want to exclude tokens that represent ponctuation etc (tokenizers will often split them alone)

  // We build a gramset, which is essentially a list of all the unique bigrams and trigrams
  // We'll create entirely new words from those grams
  const gramset = _.chain(plainTokens)
    .map(x => [ngram(x, 2), ngram(x, 3)])
    .flattenDeep()
    .uniq()
    .value()

  const realWords = _.uniq(plainTokens)
  const junkWords = _.range(0, 100).map(() => _.sampleSize(gramset, _.random(1, 4, false)).join('')) // 100 randomly generated words

  return _.range(0, count).map(() => {
    // create new utterances of varying length, made of real AND junk words (more junk)
    const fromVocab = _.random(1, 2, false)
    const fromJunk = _.random(2, 8, false)

    return _.shuffle([..._.sampleSize(realWords, fromVocab), ..._.sampleSize(junkWords, fromJunk)]).map(
      x => separatorChar + x // Add token separator at the begining of each word
    )<commit_msg>added examples<commit_after>import _ from 'lodash'

const separatorChar = '▁'

// Example input: [ [ '▁he', 'llo', '▁what', '▁is', '▁your', '▁name', '▁?' ] ]
// Example output: [ [ '▁haatlo', '▁llmeour', '▁your', '▁name', '▁ame', '▁atyona', '▁he' ], [ '▁amheur', '▁am', '▁llo', '▁is' ] ]
export function generateNoneUtterances(utterances: string[][], count: number): string[][] {
  const plainTokens = _.flatten(utterances)
    .map(x => x.replace(separatorChar, '')) // We want to discover real language-specific chars
    .filter(x => x.length > 1) // We want to exclude tokens that represent ponctuation etc (tokenizers will often split them alone)

  // We build a gramset, which is essentially a list of all the unique bigrams and trigrams
  // We'll create entirely new words from those grams
  const gramset = _.chain(plainTokens)
    .map(x => [ngram(x, 2), ngram(x, 3)])
    .flattenDeep()
    .uniq()
    .value()

  const realWords = _.uniq(plainTokens)
  const junkWords = _.range(0, 100).map(() => _.sampleSize(gramset, _.random(1, 4, false)).join('')) // 100 randomly generated words

  return _.range(0, count).map(() => {
    // create new utterances of varying length, made of real AND junk words (more junk)
    const fromVocab = _.random(1, 2, false)
    const fromJunk = _.random(2, 8, false)

    return _.shuffle([..._.sampleSize(realWords, fromVocab), ..._.sampleSize(junkWords, fromJunk)]).map(
      x => separatorChar + x // Add token separator at the begining of each word
    )
{'fraction_non_alphanumeric': 0.155, 'fraction_numerical': 0.012142857142857143, 'mean_word_length': 3.6996644295302015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4042257', 'n_tokens_mistral': 923, 'n_tokens_neox': 900, 'n_words': 297}