<commit_before>$: << File.expand_path("../../lib", __FILE__)

require 'database_cleaner'
require 'mongoid'
require 'mongoid-rspec'
require 'mongoid_token'
require 'benchmark'

Mongoid.configure do |config|
  config.master = Mongo::Connection.new.db("mongoid_token_benchmark")
end

DatabaseCleaner.strategy = :truncation

# start benchmarks

@token_length = 3

class Link
  include Mongoid::Document
  include Mongoid::Token
  field :url
  token :length => 3, :contains => :alphanumeric
  index :token, :unique => true
end

class NoTokenLink
  include Mongoid::Document
  field :url
end

def create_link(token = true)
  if token
    Link.create(:url => "http://involved.com.au")
  else
    NoTokenLink.create(:url => "http://involved.com.au")
  end
end

Link.destroy_all
num_records = [1, 50, 100, 1000, 2000, 3000, 4000]
puts "-- Alphanumeric token of length #{@token_length} (#{62**@token_length} possible tokens)"
Benchmark.bm do |b|
  num_records.each do |qty|<commit_msg>Fix in benchmark (indexes should be created exactly)<commit_after>$: << File.expand_path("../../lib", __FILE__)

require 'database_cleaner'
require 'mongoid'
require 'mongoid-rspec'
require 'mongoid_token'
require 'benchmark'

Mongoid.configure do |config|
  config.master = Mongo::Connection.new.db("mongoid_token_benchmark")
end

DatabaseCleaner.strategy = :truncation

# start benchmarks

@token_length = 7

class Link
  include Mongoid::Document
  include Mongoid::Token
  field :url
  token :length => @token_length, :contains => :alphanumeric
  index :token, :unique => true
end

class NoTokenLink
  include Mongoid::Document
  field :url
end

def create_link(token = true)
  if token
    Link.create(:url => "http://involved.com.au")
  else
    NoTokenLink.create(:url => "http://involved.com.au")
  end
end

Link.destroy_all
Link.create_indexes
num_records = [1, 50, 100, 1000, 2000, 3000, 4000]
puts "-- Alphanumeric token of length #{@token_length} (#{62**@token_length} possible tokens)"
Benchmark.bm do |b|
  num_records.each do |qty|
{'fraction_non_alphanumeric': 0.13161801088570016, 'fraction_numerical': 0.025235032162295893, 'mean_word_length': 4.877906976744186, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '761119', 'n_tokens_mistral': 735, 'n_tokens_neox': 678, 'n_words': 179}