<commit_before>module Awestruct
  module Extensions
    module Guide
      Change = Struct.new(:sha, :author, :date, :message)

      class Index
        include Guide
        @@transformers_registered = false
        
        def initialize(path_prefix, num_changes = nil)
          @path_prefix = path_prefix
          @num_changes = num_changes
        end

        # transform gets called twice in the process of loading the pipeline, so
        # we use a class variable to detect this scenario and shortcircuit
        def transform(transformers)
          if not @@transformers_registered
              transformers << WrapHeaderAndAssignHeadingIds.new
              @@transformers_registered = true
          end
        end

        def execute(site)
          guides = []
          
          site.pages.each do |page|
            if ( page.relative_source_path =~ /^#{@path_prefix}\/(?!index)/ )
              
              guide = OpenStruct.new
              page.guide = guide
              site.engine.set_urls([page])
              guide.url = page.url
              guide.title = page.title
              if page.description.nil?
                page.description = page.guide_summary
              end
              guide.summary = page.description
              
              # FIXME contributors should be listed somewhere on the page, but not automatically authors
              # perhaps as little pictures like on github

              # Add the Authors to Page and Guide based on Git Commit history
              #git_page_contributors = page_contributors(page, @num_changes)
              #if not page.authors
              #  page.authors = git_page_contributors
              #end
              #guide.authors = page.authors

              guide.changes = page_changes(page, @num_changes)
              
              # NOTE page.content forces the source path to be rendered
              page_content = Hpricot(page.content)
              chapters = []

              page_content.search('h3').each do |header_html|
                chapter = OpenStruct.new
                chapter.text = header_html.inner_html
                # FIXME we need a better way to generate link ids
                chapter.link_id = chapter.text.gsub(' ', '_').gsub('&#8217;', '_').gsub(/[\(\)\.!]/, '').downcase
                chapters << chapter
              end

              # make "extra chapters" a setting of the extension?
              chapter = OpenStruct.new
              chapter.text = 'Share the Knowledge'
              chapter.link_id = 'share'
              chapters << chapter

              guide.chapters = chapters

              page_languages = findLanguages(page)
              page.languages = page_languages if page_languages.size > 0

              guide.languages = page.languages

              # only add the main guide to the guide index (i.e., it doesn't have a locale suffix)
              if !(page.relative_source_path =~ /.*_[a-z]{2}(_[a-z]{2})?\..*/)
                guide.group = page.guide_group
                guide.order = if page.guide_order then page.guide_order else 100 end
                # default guide language is english
                guide.language = site.languages.en
                guides << guide
              end
            end
          end
          
          site.guides = guides
        end

        def findLanguages(page)<commit_msg>strip ? from heading ids<commit_after>module Awestruct
  module Extensions
    module Guide
      Change = Struct.new(:sha, :author, :date, :message)

      class Index
        include Guide
        @@transformers_registered = false
        
        def initialize(path_prefix, num_changes = nil)
          @path_prefix = path_prefix
          @num_changes = num_changes
        end

        # transform gets called twice in the process of loading the pipeline, so
        # we use a class variable to detect this scenario and shortcircuit
        def transform(transformers)
          if not @@transformers_registered
              transformers << WrapHeaderAndAssignHeadingIds.new
              @@transformers_registered = true
          end
        end

        def execute(site)
          guides = []
          
          site.pages.each do |page|
            if ( page.relative_source_path =~ /^#{@path_prefix}\/(?!index)/ )
              
              guide = OpenStruct.new
              page.guide = guide
              site.engine.set_urls([page])
              guide.url = page.url
              guide.title = page.title
              if page.description.nil?
                page.description = page.guide_summary
              end
              guide.summary = page.description
              
              # FIXME contributors should be listed somewhere on the page, but not automatically authors
              # perhaps as little pictures like on github

              # Add the Authors to Page and Guide based on Git Commit history
              #git_page_contributors = page_contributors(page, @num_changes)
              #if not page.authors
              #  page.authors = git_page_contributors
              #end
              #guide.authors = page.authors

              guide.changes = page_changes(page, @num_changes)
              
              # NOTE page.content forces the source path to be rendered
              page_content = Hpricot(page.content)
              chapters = []

              page_content.search('h3').each do |header_html|
                chapter = OpenStruct.new
                chapter.text = header_html.inner_html
                # FIXME we need a better way to generate link ids
                chapter.link_id = chapter.text.gsub(' ', '_').gsub('&#8217;', '_').gsub(/[\(\)\.!\?]/, '').downcase
                chapters << chapter
              end

              # make "extra chapters" a setting of the extension?
              chapter = OpenStruct.new
              chapter.text = 'Share the Knowledge'
              chapter.link_id = 'share'
              chapters << chapter

              guide.chapters = chapters

              page_languages = findLanguages(page)
              page.languages = page_languages if page_languages.size > 0

              guide.languages = page.languages

              # only add the main guide to the guide index (i.e., it doesn't have a locale suffix)
              if !(page.relative_source_path =~ /.*_[a-z]{2}(_[a-z]{2})?\..*/)
                guide.group = page.guide_group
                guide.order = if page.guide_order then page.guide_order else 100 end
                # default guide language is english
                guide.language = site.languages.en
                guides << guide
              end
            end
          end
          
          site.guides = guides
        end

        def findLanguages(page)
{'fraction_non_alphanumeric': 0.0767880649407635, 'fraction_numerical': 0.003217785578470089, 'mean_word_length': 1.5232472324723247, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6221024', 'n_tokens_mistral': 1732, 'n_tokens_neox': 1642, 'n_words': 488}