<commit_before># proposed pattern options
# %c - lowercase character
# %C - uppercase character
# %d - digit
# %D - non-zero digit / no-leading zero digit if longer than 1
# %s - alphanumeric character
# %w - upper and lower alpha character
# %p - URL-safe punctuation
#
# Any pattern can be followed by a number, representing how many of that type to generate

module Mongoid
  module Token
    module Generator
      REPLACE_PATTERN = /%((?<character>[cCdDpsw]{1})(?<length>\d+(,\d+)?)?)/

      def self.generate(pattern)
        pattern.gsub REPLACE_PATTERN do |match|
          match_data = $~
          type = match_data[:character]
          length = [match_data[:length].to_i, 1].max

          case type
          when 'c'
            down_character(length)
          when 'C'
            up_character(length)
          when 'd'
            digits(length)
          when 'D'
            integer(length)
          when 's'
            alphanumeric(length)
          when 'w'
            alpha(length)
          when 'p'
            "-"
          end
        end
      end

      private
      def self.down_character(length = 1)
        Array.new(length).map{['a'..'z'].map{|r|r.to_a}.flatten[rand(26)]}.join
      end

      def self.up_character(length = 1)
        Array.new(length).map{['A'..'Z'].map{|r|r.to_a}.flatten[rand(26)]}.join
      end

      def self.integer(length = 1)
        (rand(10**length - 10**(length-1)) + 10**(length-1)).to_s
      end

      def self.digits(length = 1)
        rand(10**length).to_s.rjust(length, "0")
      end

      def self.alpha(length = 1)
        Array.new(length).map{['A'..'Z','a'..'z'].map{|r|r.to_a}.flatten[rand(52)]}.join
      end

      def self.alphanumeric(length = 1)
        (1..length).collect { (i = Kernel.rand(62); i += ((i < 10) ? 48 : ((i < 36) ? 55 : 61 ))).chr }.join
      end

      def self.punctuation(length = 1)
        Array.new(length).map{['.','-','_','=','+','$'].map{|r|r.to_a}.flatten[rand(52)]}.join<commit_msg>Small duplication refactor on token generator methods<commit_after># proposed pattern options
# %c - lowercase character
# %C - uppercase character
# %d - digit
# %D - non-zero digit / no-leading zero digit if longer than 1
# %s - alphanumeric character
# %w - upper and lower alpha character
# %p - URL-safe punctuation
#
# Any pattern can be followed by a number, representing how many of that type to generate

module Mongoid
  module Token
    module Generator
      REPLACE_PATTERN = /%((?<character>[cCdDpsw]{1})(?<length>\d+(,\d+)?)?)/

      def self.generate(pattern)
        pattern.gsub REPLACE_PATTERN do |match|
          match_data = $~
          type = match_data[:character]
          length = [match_data[:length].to_i, 1].max

          case type
          when 'c'
            down_character(length)
          when 'C'
            up_character(length)
          when 'd'
            digits(length)
          when 'D'
            integer(length)
          when 's'
            alphanumeric(length)
          when 'w'
            alpha(length)
          when 'p'
            "-"
          end
        end
      end

      private
      def self.rand_string_from_chars(chars, length = 1)
        Array.new(length).map{ chars.sample }.join
      end

      def self.down_character(length = 1)
        self.rand_string_from_chars ('a'..'z').to_a, length
      end

      def self.up_character(length = 1)
        self.rand_string_from_chars ('A'..'Z').to_a, length
      end

      def self.integer(length = 1)
        (rand(10**length - 10**(length-1)) + 10**(length-1)).to_s
      end

      def self.digits(length = 1)
        rand(10**length).to_s.rjust(length, "0")
      end

      def self.alpha(length = 1)
        self.rand_string_from_chars (('A'..'Z').to_a + ('a'..'z').to_a), length
      end

      def self.alphanumeric(length = 1)
        (1..length).collect { (i = Kernel.rand(62); i += ((i < 10) ? 48 : ((i < 36) ? 55 : 61 ))).chr }.join
      end

      def self.punctuation(length = 1)
        self.rand_string_from_chars ['.','-','_','=','+','$'], length
{'fraction_non_alphanumeric': 0.15032999266682961, 'fraction_numerical': 0.018821803959912002, 'mean_word_length': 2.053731343283582, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7564243', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1337, 'n_words': 311}