<commit_before>module Wool
  # This is a set of methods that get provided to Warnings so they can perform
  # parse-tree analysis of their bodies.
  module SexpAnalysis
    # Parses the given text.
    #
    # @param [String] body (self.body) The text to parse
    # @return [Sexp, NilClass] the sexp representing the input text.
    def parse(body = self.body)
      Ripper.sexp(body)
    end
    
    # Finds all sexps of the given type in the given Sexp tree.
    #
    # @param [Symbol] type the type of sexp to search for
    # @param [Sexp] tree (self.parse(self.body)) The tree to search in. Leave
    #   blank to search the entire body.
    # @return [Array<Sexp>] all sexps in the input tree (or whole body) that
    #   are of the given type.
    def find_sexps(type, tree = self.parse(self.body))
      result = tree[0] == type ? [tree] : []
      tree.each do |node|
        result.concat find_sexps(type, node) if node.is_a?(Array)
      end
      result
    end
  end
end<commit_msg>Replaced the experiment with RubyParser sexps with my own sexp. Adding custom annotations for the tree next.<commit_after>module Wool
  # This is a set of methods that get provided to Warnings so they can perform
  # parse-tree analysis of their bodies.
  module SexpAnalysis
    # Replaces the ParseTree Sexps by adding a few handy-dandy methods.
    class Sexp < Array
      # Returns a mutable reference to the list of annotations that will run
      # upon initializing a new Sexp.
      #
      # @return [Array[Class < Annotation]] the activated annotations, in the
      #    order they will run in.
      def self.annotations
        @annotations ||= []
      end
      
      # Initializes the Sexp with the contents of the array returned by Ripper.
      #
      # @param [Array<Object>] other the other 
      def initialize(other)
        replace other
        replace_children!
        self.class.annotations.each do |klass|
          klass.new.annotate!(self)
        end
      end
      
      # @return [Array<Object>] the children of the node.
      def children
        self[1..-1]
      end
      
      # @return [Symbol] the type of the node.
      def type
        self[0]
      end
      
      # Replaces the children with Sexp versions of them
      def replace_children!
        replace([self[0]] + self[1..-1].map do |x|
          case x
          when Array
            self.class.new(x)
          else x
          end
        end)
      end
      private :replace_children!
    end
    
    # Parses the given text.
    #
    # @param [String] body (self.body) The text to parse
    # @return [Sexp, NilClass] the sexp representing the input text.
    def parse(body = self.body)
      Sexp.new Ripper.sexp(body)
    end
    
    # Finds all sexps of the given type in the given Sexp tree.
    #
    # @param [Symbol] type the type of sexp to search for
    # @param [Sexp] tree (self.parse(self.body)) The tree to search in. Leave
    #   blank to search the entire body.
    # @return [Array<Sexp>] all sexps in the input tree (or whole body) that
    #   are of the given type.
    def find_sexps(type, tree = self.parse(self.body))
      result = tree[0] == type ? [tree] : []
      tree.each do |node|
        result.concat find_sexps(type, node) if node.is_a?(Array)
      end
      result
    end
  end
end
{'fraction_non_alphanumeric': 0.081437125748503, 'fraction_numerical': 0.0023952095808383233, 'mean_word_length': 2.166824644549763, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2536307', 'n_tokens_mistral': 1009, 'n_tokens_neox': 994, 'n_words': 392}