<commit_before>require 'kramdown'

module MarkdownLint
  # Representation of the markdown document passed to rule checks
  class Doc
    attr_reader :lines, :parsed, :elements

    def initialize(text)
      @lines = text.split("\n")
      @parsed = Kramdown::Document.new(text)
      @elements = @parsed.root.children
      add_levels(@elements)
    end

    def self.new_from_file(filename)
      # Alternate 'constructor' passing in a filename
      self.new(File.read(filename))
    end

    def add_levels(elements, level=1)
      # Adds a 'level' option to all elements to show how nested they are
      elements.each do |e|
        e.options[:element_level] = level
        add_levels(e.children, level+1)
      end
    end

    def find_type(type)
      find_type_elements(type).map { |e| e.options }
    end

    def find_type_elements(type, elements=@elements)
      results = []
      elements.each do |e|
        results.push(e) if e.type == type
        if not e.children.empty?
          results.concat(find_type_elements(type, e.children))
        end
      end
      results
    end

    def element_linenumber(element)
      element = element.options if element.is_a?(Kramdown::Element)
      element[:location]
    end

    def element_line(element)
      @lines[element_linenumber(element) - 1]
    end

    def element_linenumbers(elements)
      elements.map { |e| element_linenumber(e) }
    end

    def element_lines(elements)
      # Gets the raw text lines where elements are located
      elements.map { |e| element_line(e) }
    end

    def header_style(header)
      if header.type != :header
        raise "header_style called with non-header element"
      end
      line = element_line(header)
      if line.start_with?("#")
        if line.strip.end_with?("#")
          :atx_closed
        else
          :atx
        end
      else
        :setext
      end
    end

    def bullet_style(bullet)
      if bullet.type != :li
        raise "bullet_style called with non-bullet element"
      end
      line = element_line(bullet).strip
      if line.start_with?("*")
        :asterisk
      elsif line.start_with?("+")
        :plus
      elsif line.start_with?("-")
        :dash
      else
        :unknown
      end
    end

    def indent_for(line)
      # Returns how much a given line is indented (hard tabs are 8 spaces)
      return line.match(/^\s*/)[0].gsub("\t", " " * 8).length
    end
  end
end<commit_msg>Document the Markdownlint::Doc class<commit_after>require 'kramdown'

module MarkdownLint
  ##
  # Representation of the markdown document passed to rule checks

  class Doc
    ##
    # A list of raw markdown source lines. Note that the list is 0-indexed,
    # while line numbers in the parsed source are 1-indexed, so you need to
    # subtract 1 from a line number to get the correct line. The element_line*
    # methods take care of this for you.

    attr_reader :lines

    ##
    # A Kramdown::Document object containing the parsed markdown document.

    attr_reader :parsed

    ##
    # A list of top level Kramdown::Element objects from the parsed document.

    attr_reader :elements

    ##
    # Create a new document given a string containing the markdown source

    def initialize(text)
      @lines = text.split("\n")
      @parsed = Kramdown::Document.new(text)
      @elements = @parsed.root.children
      add_levels(@elements)
    end

    ##
    # Alternate 'constructor' passing in a filename

    def self.new_from_file(filename)
      self.new(File.read(filename))
    end

    ##
    # Find all elements of a given type, returning their options hash. The
    # options hash has most of the useful data about an element and often you
    # can just use this in your rules.
    #
    #   # Returns [ { :location => 1, :element_level => 2 }, ... ]
    #   elements = find_type(:li)

    def find_type(type)
      find_type_elements(type).map { |e| e.options }
    end

    ##
    # Find all elements of a given type, returning a list of the element
    # objects themselves.

    def find_type_elements(type, elements=@elements)
      results = []
      elements.each do |e|
        results.push(e) if e.type == type
        if not e.children.empty?
          results.concat(find_type_elements(type, e.children))
        end
      end
      results
    end

    ##
    # Returns the line number a given element is located on in the source
    # file. You can pass in either an element object or an options hash here.

    def element_linenumber(element)
      element = element.options if element.is_a?(Kramdown::Element)
      element[:location]
    end

    ##
    # Returns the actual source line for a given element. You can pass in an
    # element object or an options hash here. This is useful if you need to
    # examine the source line directly for your rule to make use of
    # information that isn't present in the parsed document.

    def element_line(element)
      @lines[element_linenumber(element) - 1]
    end

    ##
    # Returns a list of line numbers for all elements passed in. You can pass
    # in a list of element objects or a list of options hashes here.

    def element_linenumbers(elements)
      elements.map { |e| element_linenumber(e) }
    end

    ##
    # Returns the actual source lines for a list of elements. You can pass in
    # a list of elements objects or a list of options hashes here.

    def element_lines(elements)
      elements.map { |e| element_line(e) }
    end

    ##
    # Returns the header 'style' - :atx (hashes at the beginning), :atx_closed
    # (atx header style, but with hashes at the end of the line also), :setext
    # (underlined). You can pass in the element object or an options hash
    # here.

    def header_style(header)
      if header.type != :header
        raise "header_style called with non-header element"
      end
      line = element_line(header)
      if line.start_with?("#")
        if line.strip.end_with?("#")
          :atx_closed
        else
          :atx
        end
      else
        :setext
      end
    end

    ##
    # Returns the bullet style for an unordered list - :asterisk, :plus,
    # :dash, depending on which symbol is used to denote the list item. You
    # can pass in either the element itself or an options hash here.

    def bullet_style(bullet)
      if bullet.type != :li
        raise "bullet_style called with non-bullet element"
      end
      line = element_line(bullet).strip
      if line.start_with?("*")
        :asterisk
      elsif line.start_with?("+")
        :plus
      elsif line.start_with?("-")
        :dash
      else
        :unknown
      end
    end

    ##
    # Returns how much a given line is indented. Hard tabs are treated as an
    # indent of 8 spaces. You need to pass in the raw string here.

    def indent_for(line)
      return line.match(/^\s*/)[0].gsub("\t", " " * 8).length
    end

    private

    ##
    # Adds a 'level' option to all elements to show how nested they are

    def add_levels(elements, level=1)
      elements.each do |e|
        e.options[:element_level] = level
        add_levels(e.children, level+1)
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.08592880978865405, 'fraction_numerical': 0.002363737486095662, 'mean_word_length': 2.334724153917478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985058', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2098, 'n_words': 737}