<commit_before>module Parse

  class Results
    def initialize
      @result = nil
    end

    def pull_from_hash(hash, key)
      @hash = hash
      @key = key

      if @hash.include?(@key)
        @result = @hash[@key]
      else
        @hash.each_pair do |k, v|
          if v.class == Hash
            pull_from_hash(v, @key)
          end
        end
      end
      @result
    end
  end

  # Extract portions of text from Wiki article
  class Text < Results
    def initialize(request, prop='extract')
      @request = self.pull_from_hash(request, prop)
    end

    # Returns first paragraph of the Wiki article
    #
    # TODO: refactor to take number of paragraphs as argument
    def paragraph
      @start = @request.split("</p>")[0]
      @start = @start.split("<p>")[1]
      return @start
    end

    # Removes HTML tags from a String
    #
    # string - a String that contains HTML tags.
    #
    # Returns the string without HTML tags.
    def only_text(string)
      no_html_tags = string.gsub(/<\/?.*?>/,'')
    end

    def wikitext_sections
      @sidebar = @request.sub(/^(.*?)'''/)
    end

    # Return the text from the sidebar, if one exists
    def sidebar

    end

    # Return all refrences
    def refs
    end

    # Return all paragraphs under a given heading
    def header(name)
    end

    # Returns user-defined number of words before and/or a user-defined search term.
    def search(term, words, options={})
    end
  end

  class Media < Results
    def images
    end
  end
end<commit_msg>add helper method for prop=revision queries. plus other things<commit_after>module Parse

  class Results
    def initialize
      @result = nil
    end

    def pull_from_hash(hash, key)
      @hash = hash
      @key = key

      if @hash.include?(@key)
        @result = @hash[@key]
      else
        @hash.each_pair do |k, v|
          if v.class == Hash
            pull_from_hash(v, @key)
          end
        end
      end
      @result
    end
  end

  # Extract portions of text from Wiki article
  class Text < Results
    attr_reader :request
    def initialize(request, prop='extract')
      @request = self.pull_from_hash(request, prop)
      if @request.class == Array
        @request = self.pull_from_hash(@request[0], "*")
      end
    end

    # Returns first paragraph of the Wiki article
    #
    # TODO: refactor to take number of paragraphs as argument
    def paragraph
      @start = @request.split("</p>")[0]
      @start = @start.split("<p>")[1]
      return @start
    end

    # Removes HTML tags from a String
    #
    # string - a String that contains HTML tags.
    #
    # Returns the string without HTML tags.
    def only_text(string)
      no_html_tags = string.gsub(/<\/?.*?>/,'')
    end

    def wikitext_sections
      
    end

    # Return the text from the sidebar, if one exists
    def sidebar
      @sidebar = content_split(0)

    end

    # Return all refrences
    def refs
      @content = content_split(1, 2)
    
      #add all references to an array. this does not work
      @refs = @content.match(/<ref>(.*?)<\/ref>/)

    end

    # Return all paragraphs under a given heading
    def header(name)
    end


    def content_split(start, finish=nil)
      @content = @request.split("'''")
      if finish == nil
        return @content[start]
      else
        return @content[start..finish].join
    end 


  end

  class Media < Results
    def images
    end
  end
end
{'fraction_non_alphanumeric': 0.09628134909195733, 'fraction_numerical': 0.0023061400980109543, 'mean_word_length': 2.0358705161854767, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266902', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1034, 'n_words': 348}