<commit_before>require 'mongoid/token/exceptions'
require 'mongoid/token/options'
require 'mongoid/token/generator'
require 'mongoid/token/finders'
require 'mongoid/token/collision_resolver'

module Mongoid
  module Token
    extend ActiveSupport::Concern

    module ClassMethods
      def initialize_copy(source)
        super(source)
        self.token = nil
      end

      def token(*args)
        options = Mongoid::Token::Options.new(args.extract_options!)

        self.field options.field_name, :type => String, :default => nil
        self.index({ options.field_name => 1 }, { :unique => true, :sparse => true })

        resolver = Mongoid::Token::CollisionResolver.new(self, options.field_name, options.retry_count)
        resolver.create_new_token = Proc.new do |document|
          document.send(:create_token, options.field_name, options.pattern)
        end

        if options.skip_finders? == false
          Finders.define_custom_token_finder_for(self, options.field_name)
        end

        set_callback(:create, :before) do |document|
          document.create_token_if_nil options.field_name, options.pattern
        end

        set_callback(:save, :before) do |document|
          document.create_token_if_nil options.field_name, options.pattern
        end

        if options.override_to_param?
          self.send(:define_method, :to_param) do
            self.send(options.field_name) || super(*args)
          end
        end
      end
    end

    protected
    def create_token(field_name, pattern)<commit_msg>Complexity refactor on #token method<commit_after>require 'mongoid/token/exceptions'
require 'mongoid/token/options'
require 'mongoid/token/generator'
require 'mongoid/token/finders'
require 'mongoid/token/collision_resolver'

module Mongoid
  module Token
    extend ActiveSupport::Concern

    module ClassMethods
      def initialize_copy(source)
        super(source)
        self.token = nil
      end

      def token(*args)
        options = Mongoid::Token::Options.new(args.extract_options!)

        add_token_field_and_index(options)
        add_token_collision_resolver(options)
        set_token_callbacks(options)
        
        define_custom_finders(options) if options.skip_finders? == false
        override_to_param(options) if options.override_to_param?
      end

      private
      def add_token_field_and_index(options)
        self.field options.field_name, :type => String, :default => nil
        self.index({ options.field_name => 1 }, { :unique => true, :sparse => true })
      end

      def add_token_collision_resolver(options)
        resolver = Mongoid::Token::CollisionResolver.new(self, options.field_name, options.retry_count)
        resolver.create_new_token = Proc.new do |document|
          document.send(:create_token, options.field_name, options.pattern)
        end
      end

      def define_custom_finders(options)
        Finders.define_custom_token_finder_for(self, options.field_name)
      end

      def set_token_callbacks(options)
        set_callback(:create, :before) do |document|
          document.create_token_if_nil options.field_name, options.pattern
        end

        set_callback(:save, :before) do |document|
          document.create_token_if_nil options.field_name, options.pattern
        end
      end

      def override_to_param(options)
        self.send(:define_method, :to_param) do
          self.send(options.field_name) || super(*args)
        end
      end
    end

    protected
    def create_token(field_name, pattern)
{'fraction_non_alphanumeric': 0.08932093547478163, 'fraction_numerical': 0.0005635390250774866, 'mean_word_length': 2.970917225950783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512716', 'n_tokens_mistral': 1046, 'n_tokens_neox': 1021, 'n_words': 202}