<commit_before>require 'rubygems'
require 'bundler'
begin
  Bundler.setup(:default, :development)
rescue Bundler::BundlerError => e
  $stderr.puts e.message
  $stderr.puts "Run `bundle install` to install missing gems"
  exit e.status_code
end
require 'riot'
require 'udon'<commit_msg>Some utilities to help generate random strings etc.<commit_after>require 'rubygems'
require 'bundler'
begin
  Bundler.setup(:default, :development)
rescue Bundler::BundlerError => e
  $stderr.puts e.message
  $stderr.puts "Run `bundle install` to install missing gems"
  exit e.status_code
end
require 'riot'
require 'udon'
require 'poisson'

class String
  def udon
    Udon.parse(self)
  end
end

class Array
  def gcd; inject(){|n1,n2| n1.gcd(n2)} end
end

def poisson(mean)
  el = Math.exp(-mean); k = 0; p = 1
  begin
    k += 1
    p = p * rand
  end while p > el
  return k - 1
end

def randstr(avg_length, char_dists = [[0.1,' '],[0.1,('A'..'Z')],[0.8,('a'..'z')]])
  ret = ''
  char_dists = char_dists.scan /./umn if char_dists.is_a?(String)
  chrs = char_dists.sort_by{|n|n[0]}
  chrs.map! do |k,ch|
    if ch.nil?
      ch = k
      k = 1.0 / chrs.size.to_f
    end
    if ch.is_a?(Range)
      if ch.max.is_a?(String)
        [k, ((ch.min.unpack('U')[0])..(ch.max.unpack('U')[0]))]
      else [k, ch] end
    elsif ch.is_a?(Integer)
      [k, (ch..ch)]
    elsif ch.is_a?(String)
      [k, ((ch.unpack('U')[0])..(ch.unpack('U')[0]))]
    else
      raise ArgumentError, 'Ranges and strings only'
    end
  end
  (0..poisson(avg_length)).each do
    range_sel = rand
    prob_sum = 0.0
    chrs.each do |prob,cr|
      prob_sum += prob
      if prob_sum > range_sel
        ret << [(cr.min + rand(cr.max - cr.min + 1))].pack('U')
        break
      end
    end
  end
  return ret
end
{'fraction_non_alphanumeric': 0.1601123595505618, 'fraction_numerical': 0.014044943820224719, 'mean_word_length': 2.9229074889867843, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1022735', 'n_tokens_mistral': 728, 'n_tokens_neox': 715, 'n_words': 176}