<commit_before>module Expgen
  module Randomizer
    extend self

    def repeat(number)
      if number == "*"
        ""
      elsif number == "+"
        yield
      elsif number
        number[:int].to_i.times.map { yield }.join
      else
        yield
      end
    end

    def randomize(tree)
      case tree
        when Array              then tree.map { |el| randomize(el) }.join
        when Nodes::Alternation then randomize(tree.options.sample)
        when Nodes::Group       then repeat(tree.repeat) { randomize(tree.elements) }
        when Nodes::Character   then repeat(tree.repeat) { tree.chars.sample }
      end
    end
  end
end
<commit_msg>Make it a bit more random<commit_after>module Expgen
  module Randomizer
    extend self

    def range(number)
      if number == "*"
        [0,5]
      elsif number == "+"
        [1,5]
      elsif number
        [number[:int].to_i, number[:int].to_i]
      else
        [1,1]
      end
    end

    def repeat(number)
      first, last = range(number)
      number = rand(last - first + 1) + first
      number.times.map { yield }.join
    end

    def randomize(tree)
      case tree
        when Array              then tree.map { |el| randomize(el) }.join
        when Nodes::Alternation then randomize(tree.options.sample)
        when Nodes::Group       then repeat(tree.repeat) { randomize(tree.elements) }
        when Nodes::Character   then repeat(tree.repeat) { tree.chars.sample }
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.09946236559139784, 'fraction_numerical': 0.004704301075268817, 'mean_word_length': 1.925343811394892, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7286142', 'n_tokens_mistral': 457, 'n_tokens_neox': 448, 'n_words': 121}