<commit_before>require 'raingrams/extensions'

module Raingrams
  class Ngram < Array

    #
    # Creates a new Ngram object with the specified _objects_.
    #
    def initialize(objects)
      super(objects.map { |obj| obj.to_gram })
    end

    #
    # Creates a new Ngram object from the specified _objects_.
    #
    def self.[](*objects)
      self.new(objects)
    end

    #
    # Creates a new Ngram object by appending the specified _grams_ to the
    # ngram.
    #
    def +(grams)
      if grams.kind_of?(Array)
        return self.class.new(super(grams.map { |gram|
          gram.to_gram
        }))
      else
        return self.class.new(super([grams.to_gram]))
      end
    end

    def <<(gram)
      super(gram.to_gram)
    end

    #
    # Returns the prefix of the ngram.
    #
    def prefix
      self[0...length-1]
    end

    #
    # Returns +true+ if the ngram is prefixed by the specified
    # _smaller_ngram_.
    #
    def prefixed_by?(smaller_ngram)
      prefix == smaller_ngram
    end

    def postfix
      self[1..-1]
    end

    def postfixed_by?(ngram)
      postfix == ngram
    end

    def starts_with?(obj)
      self.first == obj.to_gram
    end

    def ends_with?(obj)
      self.last == obj.to_gram
    end

    def include?(obj)
      super(obj.to_gram)
    end

    def flatten
      self.dup
    end

    def flatten!
      self
    end

    def to_s
      join(', ')
    end

    def inspect
      'Ngram[' + self.map { |gram| gram.inspect }.join(', ') + ']'
    end

  end
end<commit_msg>* Added Ngram#includes.<commit_after>require 'raingrams/extensions'

module Raingrams
  class Ngram < Array

    #
    # Creates a new Ngram object with the specified _objects_.
    #
    def initialize(objects)
      super(objects.map { |obj| obj.to_gram })
    end

    #
    # Creates a new Ngram object from the specified _objects_.
    #
    def self.[](*objects)
      self.new(objects)
    end

    #
    # Creates a new Ngram object by appending the specified _grams_ to the
    # ngram.
    #
    def +(grams)
      if grams.kind_of?(Array)
        return self.class.new(super(grams.map { |gram|
          gram.to_gram
        }))
      else
        return self.class.new(super([grams.to_gram]))
      end
    end

    def <<(gram)
      super(gram.to_gram)
    end

    #
    # Returns the prefix of the ngram.
    #
    def prefix
      self[0...length-1]
    end

    #
    # Returns +true+ if the ngram is prefixed by the specified
    # _smaller_ngram_.
    #
    def prefixed_by?(smaller_ngram)
      prefix == smaller_ngram
    end

    def postfix
      self[1..-1]
    end

    def postfixed_by?(ngram)
      postfix == ngram
    end

    def starts_with?(obj)
      self.first == obj.to_gram
    end

    def ends_with?(obj)
      self.last == obj.to_gram
    end

    def include?(obj)
      super(obj.to_gram)
    end

    def includes?(*grams)
      (self & grams) == grams
    end

    def flatten
      self.dup
    end

    def flatten!
      self
    end

    def to_s
      join(', ')
    end

    def inspect
      'Ngram[' + self.map { |gram| gram.inspect }.join(', ') + ']'
    end

  end
end
{'fraction_non_alphanumeric': 0.10034711265383402, 'fraction_numerical': 0.0025244556642473968, 'mean_word_length': 1.8278322925958965, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3062665', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1069, 'n_words': 279}