<commit_before>require "spec_helper"
require "./lib/human_history/history_tokenizer"
require "./lib/human_history/history_token_registry"
require "./lib/human_history/registered_token"

describe HumanHistory::HistoryTokenRegistry do
  before { described_class.clear }

  describe ".register" do
    it "adds the token type to the registry" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, _, _) { key == "actor_id" }

      expect(described_class.registered_tokens.length).to eq 1
      expect(described_class.registered_tokens.first.type).to eq HumanHistory::ActorHistoryToken
    end

    it "does not register the same token type twice" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, _, _) { key == "actor_id" }
      described_class.register HumanHistory::ActorHistoryToken, ->(key, _, _) { key == "actor_id" }

      expect(described_class.registered_tokens.length).to eq 1
    end

    it "adds the selector from the type if it responds to `tokenizable`" do
      class TestToken
        def self.tokenizable?(key, value, changeset); end
      end

      described_class.register TestToken
      expect(described_class.registered_tokens.length).to eq 1
      expect(described_class.registered_tokens.first.selector).to be_kind_of Proc
    end
  end

  describe ".unregister" do
    it "removes the token type from the registry" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, _, _) { key == "actor_id" }
      described_class.unregister HumanHistory::ActorHistoryToken

      expect(described_class.registered_tokens).to be_empty
    end
  end

  describe ".for" do
    it "returns the tokens for the provided selector" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.for("actor_id", nil, nil).map(&:type)).to \
        eq [HumanHistory::ActorHistoryToken]
    end

    it "returns an empty array if the selector is not found" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.for("blah", nil, nil)).to be_empty
    end
  end
end
<commit_msg>fix bugs in exports controller spec<commit_after>require "spec_helper"
require "./lib/human_history/history_tokenizer"
require "./lib/human_history/history_token_registry"
require "./lib/human_history/registered_token"

describe HumanHistory::HistoryTokenRegistry do
  before { described_class.clear }

  describe ".register" do
    it "adds the token type to the registry" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.registered_tokens.length).to eq 1
      expect(described_class.registered_tokens.first.type).to eq HumanHistory::ActorHistoryToken
    end

    it "does not register the same token type twice" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.registered_tokens.length).to eq 1
    end

    it "adds the selector from the type if it responds to `tokenizable`" do
      class TestToken
        def self.tokenizable?(key, value, changeset); end
      end

      described_class.register TestToken
      expect(described_class.registered_tokens.length).to eq 1
      expect(described_class.registered_tokens.first.selector).to be_kind_of Proc
    end
  end

  describe ".unregister" do
    it "removes the token type from the registry" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }
      described_class.unregister HumanHistory::ActorHistoryToken

      expect(described_class.registered_tokens).to be_empty
    end
  end

  describe ".for" do
    it "returns the tokens for the provided selector" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.for("actor_id", nil, nil).map(&:type)).to \
        eq [HumanHistory::ActorHistoryToken]
    end

    it "returns an empty array if the selector is not found" do
      described_class.register HumanHistory::ActorHistoryToken, ->(key, value, set) { key == "actor_id" }

      expect(described_class.for("blah", nil, nil)).to be_empty
    end
  end
end

{'fraction_non_alphanumeric': 0.09901433691756273, 'fraction_numerical': 0.0013440860215053765, 'mean_word_length': 3.9282560706401766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6713524', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1251, 'n_words': 345}