<commit_before>require_relative 'subset'

module DeepCover
  class Analyser::Statement < Analyser
    include Analyser::Subset
    # Returns a map of Range => runs
    def results
      each_node.flat_map do |node, sub_statements|
        shatter(node, sub_statements).map{|r| [r, node_runs(node)]}
      end.to_h
    end

    private

    IGNORE_KEYWORDS = %w[do begin end].to_set
    # returns a list of [proper_range, node], where
    # the nodes may be repeating, the proper_ranges are non-intersecting but non ordered.
    def shatter(node, sub_statements)
      subs = sub_statements.map{|n| n.expression}.compact
      range = node.expression
      subs.reject!{|r| r.disjoint?(range) } # This is the case iff using heredocs
      proper = range.split(*subs)
      proper.map!{|r| r.lstrip(/(\s*#.*\n)+/) }  # Strip comment blocks
      proper.map!{|r| r.strip(/[^a-zA-Z0-9'"\[\]{}_:$]*/) } # Ignore whitespace & punctuation
      proper.reject!(&:empty?)
      proper.reject!{|r| IGNORE_KEYWORDS.include?(r.source) }
      proper
    end<commit_msg>There's no real point in trying to get distinct ranges<commit_after>require_relative 'subset'

module DeepCover
  class Analyser::Statement < Analyser
    include Analyser::Subset
    # Returns a map of Range => runs
    def results
      each_node.map do |node, _sub_statements|
        [node.expression, node_runs(node)]
      end.to_h
    end

    private
{'fraction_non_alphanumeric': 0.11872791519434629, 'fraction_numerical': 0.0014134275618374558, 'mean_word_length': 3.1769911504424777, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5546860', 'n_tokens_mistral': 488, 'n_tokens_neox': 462, 'n_words': 119}