<commit_before>module Chronic
  class Token #:nodoc:
    attr_accessor :word, :tags

    def initialize(word)
      @word = word
      @tags = []
    end

    # Tag this token with the specified tag
    def tag(new_tag)
      @tags << new_tag
    end

    # Remove all tags of the given class
    def untag(tag_class)
      @tags = @tags.select { |m| !m.kind_of? tag_class }
    end

    # Return true if this token has any tags
    def tagged?
      @tags.size > 0
    end

    # Return the Tag that matches the given class
    def get_tag(tag_class)
      @tags.find { |m| m.kind_of? tag_class }
    end

    # Print this Token in a pretty way
    def to_s<commit_msg>use delete_if over select ! for tokens<commit_after>module Chronic
  class Token #:nodoc:
    attr_accessor :word, :tags

    def initialize(word)
      @word = word
      @tags = []
    end

    # Tag this token with the specified tag
    def tag(new_tag)
      @tags << new_tag
    end

    # Remove all tags of the given class
    def untag(tag_class)
      @tags.delete_if { |m| m.kind_of? tag_class }
    end

    # Return true if this token has any tags
    def tagged?
      @tags.size > 0
    end

    # Return the Tag that matches the given class
    def get_tag(tag_class)
      @tags.find { |m| m.kind_of? tag_class }
    end

    # Print this Token in a pretty way
    def to_s
{'fraction_non_alphanumeric': 0.07799852832965416, 'fraction_numerical': 0.0014716703458425313, 'mean_word_length': 2.215130023640662, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4970286', 'n_tokens_mistral': 459, 'n_tokens_neox': 440, 'n_words': 159}