<commit_before>module Hologram
  class DocParser
    DEFAULT_SUPPORTED_EXTENSIONS = ['.css', '.scss', '.less', '.sass', '.styl', '.js', '.md', '.markdown', '.erb' ]
    attr_accessor :source_path, :pages, :doc_blocks, :nav_level

    def initialize(source_path, index_name = nil, plugins=[], opts={})
      @plugins = plugins
      @source_paths = Array(source_path)
      @index_name = index_name
      @nav_level = opts[:nav_level] || 'page'
      @pages = {}
      @output_files_by_category = {}
      @supported_extensions = DEFAULT_SUPPORTED_EXTENSIONS
      @supported_extensions += opts[:custom_extensions] if opts[:custom_extensions]
      @ignore_paths = opts[:ignore_paths] || []
    end

    def parse
      # recursively traverse our directory structure looking for files that
      # match our "parseable" file types. Open those files pulling out any
      # comments matching the hologram doc style /*doc */ and create DocBlock
      # objects from those comments, then add those to a collection object which
      # is then returned.

      doc_block_collection = DocBlockCollection.new

      @source_paths.each do |source_path|
        process_dir(source_path, doc_block_collection)
      end

      # doc blocks can define parent/child relationships that will nest their
      # documentation appropriately. we can't put everything into that structure
      # on our first pass through because there is no guarantee we'll parse files
      # in the correct order. This step takes the full collection and creates the
      # proper structure.
      doc_block_collection.create_nested_structure


      # hand off our properly nested collection to the output generator
      build_output(doc_block_collection.doc_blocks)

      @plugins.finalize(@pages)

      # if we have an index category defined in our config copy that
      # page to index.html
      if @index_name
        name = @index_name + '.html'
        if @pages.has_key?(name)
          @pages['index.html'] = @pages[name]
          title, _ = @output_files_by_category.rassoc(name)<commit_msg>Deep copy index page hash on parse<commit_after>module Hologram
  class DocParser
    DEFAULT_SUPPORTED_EXTENSIONS = ['.css', '.scss', '.less', '.sass', '.styl', '.js', '.md', '.markdown', '.erb' ]
    attr_accessor :source_path, :pages, :doc_blocks, :nav_level

    def initialize(source_path, index_name = nil, plugins=[], opts={})
      @plugins = plugins
      @source_paths = Array(source_path)
      @index_name = index_name
      @nav_level = opts[:nav_level] || 'page'
      @pages = {}
      @output_files_by_category = {}
      @supported_extensions = DEFAULT_SUPPORTED_EXTENSIONS
      @supported_extensions += opts[:custom_extensions] if opts[:custom_extensions]
      @ignore_paths = opts[:ignore_paths] || []
    end

    def parse
      # recursively traverse our directory structure looking for files that
      # match our "parseable" file types. Open those files pulling out any
      # comments matching the hologram doc style /*doc */ and create DocBlock
      # objects from those comments, then add those to a collection object which
      # is then returned.

      doc_block_collection = DocBlockCollection.new

      @source_paths.each do |source_path|
        process_dir(source_path, doc_block_collection)
      end

      # doc blocks can define parent/child relationships that will nest their
      # documentation appropriately. we can't put everything into that structure
      # on our first pass through because there is no guarantee we'll parse files
      # in the correct order. This step takes the full collection and creates the
      # proper structure.
      doc_block_collection.create_nested_structure


      # hand off our properly nested collection to the output generator
      build_output(doc_block_collection.doc_blocks)

      @plugins.finalize(@pages)

      # if we have an index category defined in our config copy that
      # page to index.html
      if @index_name
        name = @index_name + '.html'
        if @pages.has_key?(name)
          @pages['index.html'] = Marshal.load(Marshal.dump(@pages[name]))
          title, _ = @output_files_by_category.rassoc(name)
{'fraction_non_alphanumeric': 0.08716361038342463, 'fraction_numerical': 0.0, 'mean_word_length': 3.1791044776119404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '633450', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1120, 'n_words': 377}