<commit_before>require 'forwardable'
require 'fileutils'

module Amber
  class Site
    extend Forwardable

    attr_accessor :page_list
    attr_accessor :root
    attr_accessor :continue_on_error

    # @config is the primary SiteConfiguration
    def_delegators :@config, :pages_dir, :dest_dir, :locales, :default_locale, :path_prefix

    def initialize(root_dir)
      @continue_on_error = true
      @config = SiteConfiguration.load(self, root_dir)
    end

    def add_config(config)
      @config.children << config
    end

    def load_pages
      @root          = nil
      @pages_by_path = {}  # hash of pages keyed by page path
      @pages_by_name = {}  # hash of pages keyed by page name
      @page_list     = PageArray.new
      @dir_list      = []  # an array of non-page directories
      @page_paths    = []  # an array of page paths, used for greping through paths.

      # some paths are specific to only one locale (e.g aliases)
      @pages_by_locale_path = POSSIBLE_LANGUAGES.keys.inject(Hash.new({})) {|h,locale| h[locale] = {}; h}

      add_configuration(@config)
    end

    #def reload_pages_if_needed
    #  if @pages_by_path.nil? || @config.pages_changed?
    #    puts "Reloading pages ................."
    #    load_pages
    #  end
    #end

    def render
      @page_list.each do |page|
        page.render_to_file(@config.dest_dir)
        putc '.'; $stdout.flush
      end
      @dir_list.each do |directory|
        src = File.join(@config.pages_dir, directory)
        dst = File.join(@config.dest_dir, directory)
        Render::Asset.render_dir(src, dst)
        putc '.'; $stdout.flush
      end
      if @config.short_paths
        render_short_path_symlinks
      end
      Render::Apache.write_htaccess(@config, @config.pages_dir, @config.dest_dir)
      puts
    end

    def clear
      Dir.glob("#{@config.dest_dir}/*").each do |file|
        FileUtils.rm_r(file)
      end
    end

    def with_destination(new_dest)
      dest_dir = @config.dest_dir
      @config.dest_dir = new_dest
      yield
      @config.dest_dir = dest_dir
    end

    #
    # find pages by a filter.
    # filter is a string composing a path segment.
    # For example:
    #   "chat/security"
    # Which would match "/services/chat/security" but not "/services/security"
    #
    def find_pages(filter)
      filter = filter.downcase
      if filter =~ /\//
        path = filter.split('/').map{|segment| segment.gsub(/[^0-9a-z_-]/, '')}
        path_str = path.join('/')
        if (page = @pages_by_path[path_str])
          page
        elsif matched_path = @page_paths.grep(/#{Regexp.escape(path_str)}/).first
          @pages_by_path[matched_path]
        elsif page = @pages_by_name[path.last]
          page
        else
          nil
        end
      elsif @pages_by_path[filter]
        @pages_by_path[filter]
      else
        @pages_by_name[filter]
      end
    end

    def find_page(filter)
      find_pages(filter)
    end

    def all_pages
      @page_list
    end

    def find_page_by_path(path, locale=I18n.default_locale)
      if locale.is_a? String
        if I18n.locale_available?(locale)
          locale = locale.to_sym
        end
      end
      if path.is_a? Array
        path = path.join('/')
      end
      @pages_by_locale_path[locale][path] || @pages_by_path[path]
    end

    def find_page_by_name(name)
      @pages_by_name[name]
    end

    #
    # returns the shortest possible path for a page, such that the path doesn't collide with
    # another page or another page's aliases.
    #
    def shortest_path(page)
      path_so_far = []
      page.path.reverse.each do |path_segment|
        path_so_far.push(path_segment)
        path_str = path_so_far.join('/')
        if @pages_by_path[path_str].nil? && short_paths[path_str] == page
          return path_so_far
        end
      end
      return []
    end

    private

    def add_configuration(config)
      config.reset_timestamp

      # create base_page
      base_page = begin
        if config.path.nil?
          @root = StaticPage.new(nil, 'root', config.pages_dir)
          add_page(@root)
          @root
        else
          name = File.basename(config.path)
          sub_root = StaticPage.new(find_parent(config.path), name, config.pages_dir, config.path_prefix)
          add_page(sub_root)
          sub_root
        end
      end
      base_page.config = config

      # load menu and locals
      I18n.load_path += Dir[File.join(config.locales_dir, '/*.{rb,yml,yaml}')] if config.locales_dir

      # add the full directory tree
      base_page.scan_directory_tree do |page, asset_dir|
        add_page(page) if page
        @dir_list << asset_dir if asset_dir
      end
      @page_paths += @pages_by_path.keys

      # recursively add sub-sites
      config.children.each do |sub_config|
        add_configuration(sub_config)
      end
    end

    #
    # registers a page with the site, indexing the page path in our various hashes
    #
    def add_page(page)
      @pages_by_name[page.name] ||= page
      @pages_by_path[page.path.join('/')] = page
      add_aliases(I18n.default_locale, page, @pages_by_path)
      page.locales.each do |locale|
        next if locale == I18n.default_locale
        add_aliases(locale, page, @pages_by_locale_path[locale])
      end<commit_msg>add support for nested locale dirs<commit_after>require 'forwardable'
require 'fileutils'

module Amber
  class Site
    extend Forwardable

    attr_accessor :page_list
    attr_accessor :root
    attr_accessor :continue_on_error

    # @config is the primary SiteConfiguration
    def_delegators :@config, :pages_dir, :dest_dir, :locales, :default_locale, :path_prefix

    def initialize(root_dir)
      @continue_on_error = true
      @config = SiteConfiguration.load(self, root_dir)
    end

    def add_config(config)
      @config.children << config
    end

    def load_pages
      @root          = nil
      @pages_by_path = {}  # hash of pages keyed by page path
      @pages_by_name = {}  # hash of pages keyed by page name
      @page_list     = PageArray.new
      @dir_list      = []  # an array of non-page directories
      @page_paths    = []  # an array of page paths, used for greping through paths.

      # some paths are specific to only one locale (e.g aliases)
      @pages_by_locale_path = POSSIBLE_LANGUAGES.keys.inject(Hash.new({})) {|h,locale| h[locale] = {}; h}

      add_configuration(@config)
    end

    #def reload_pages_if_needed
    #  if @pages_by_path.nil? || @config.pages_changed?
    #    puts "Reloading pages ................."
    #    load_pages
    #  end
    #end

    def render
      @page_list.each do |page|
        page.render_to_file(@config.dest_dir)
        putc '.'; $stdout.flush
      end
      @dir_list.each do |directory|
        src = File.join(@config.pages_dir, directory)
        dst = File.join(@config.dest_dir, directory)
        Render::Asset.render_dir(src, dst)
        putc '.'; $stdout.flush
      end
      if @config.short_paths
        render_short_path_symlinks
      end
      Render::Apache.write_htaccess(@config, @config.pages_dir, @config.dest_dir)
      puts
    end

    def clear
      Dir.glob("#{@config.dest_dir}/*").each do |file|
        FileUtils.rm_r(file)
      end
    end

    def with_destination(new_dest)
      dest_dir = @config.dest_dir
      @config.dest_dir = new_dest
      yield
      @config.dest_dir = dest_dir
    end

    #
    # find pages by a filter.
    # filter is a string composing a path segment.
    # For example:
    #   "chat/security"
    # Which would match "/services/chat/security" but not "/services/security"
    #
    def find_pages(filter)
      filter = filter.downcase
      if filter =~ /\//
        path = filter.split('/').map{|segment| segment.gsub(/[^0-9a-z_-]/, '')}
        path_str = path.join('/')
        if (page = @pages_by_path[path_str])
          page
        elsif matched_path = @page_paths.grep(/#{Regexp.escape(path_str)}/).first
          @pages_by_path[matched_path]
        elsif page = @pages_by_name[path.last]
          page
        else
          nil
        end
      elsif @pages_by_path[filter]
        @pages_by_path[filter]
      else
        @pages_by_name[filter]
      end
    end

    def find_page(filter)
      find_pages(filter)
    end

    def all_pages
      @page_list
    end

    def find_page_by_path(path, locale=I18n.default_locale)
      if locale.is_a? String
        if I18n.locale_available?(locale)
          locale = locale.to_sym
        end
      end
      if path.is_a? Array
        path = path.join('/')
      end
      @pages_by_locale_path[locale][path] || @pages_by_path[path]
    end

    def find_page_by_name(name)
      @pages_by_name[name]
    end

    #
    # returns the shortest possible path for a page, such that the path doesn't collide with
    # another page or another page's aliases.
    #
    def shortest_path(page)
      path_so_far = []
      page.path.reverse.each do |path_segment|
        path_so_far.push(path_segment)
        path_str = path_so_far.join('/')
        if @pages_by_path[path_str].nil? && short_paths[path_str] == page
          return path_so_far
        end
      end
      return []
    end

    private

    def add_configuration(config)
      config.reset_timestamp

      # create base_page
      base_page = begin
        if config.path.nil?
          @root = StaticPage.new(nil, 'root', config.pages_dir)
          add_page(@root)
          @root
        else
          name = File.basename(config.path)
          sub_root = StaticPage.new(find_parent(config.path), name, config.pages_dir, config.path_prefix)
          add_page(sub_root)
          sub_root
        end
      end
      base_page.config = config

      # load menu and locals
      I18n.load_path += Dir[File.join(config.locales_dir, '/**/*.{rb,yml,yaml}')] if config.locales_dir

      # add the full directory tree
      base_page.scan_directory_tree do |page, asset_dir|
        add_page(page) if page
        @dir_list << asset_dir if asset_dir
      end
      @page_paths += @pages_by_path.keys

      # recursively add sub-sites
      config.children.each do |sub_config|
        add_configuration(sub_config)
      end
    end

    #
    # registers a page with the site, indexing the page path in our various hashes
    #
    def add_page(page)
      @pages_by_name[page.name] ||= page
      @pages_by_path[page.path.join('/')] = page
      add_aliases(I18n.default_locale, page, @pages_by_path)
      page.locales.each do |locale|
        next if locale == I18n.default_locale
        add_aliases(locale, page, @pages_by_locale_path[locale])
      end
{'fraction_non_alphanumeric': 0.10003736222678872, 'fraction_numerical': 0.0022417336073229962, 'mean_word_length': 2.269312977099237, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5585447', 'n_tokens_mistral': 3518, 'n_tokens_neox': 3431, 'n_words': 780}