<commit_before>module Para
  module SeoTools
    module Skeleton
      extend ActiveSupport::Autoload

      autoload :Site
      autoload :Page
      autoload :Worker

      mattr_accessor :site, :config

      def self.load
        # Ensure routes are loaded to allow skeleton to call routes name helpers
        Rails.application.reload_routes!
        # Skeleton file path
        skeleton_path = Rails.root.join('config', 'skeleton.rb')
        # Load the skeleton file
        require skeleton_path if File.exists?(skeleton_path)
      end

      def self.draw(lazy: false, &block)
        self.config = block
        build unless lazy
      end

      def self.build
        return if migrating?
        return unless ActiveRecord::Base.connection.table_exists?(Para::SeoTools::Page.table_name)

        self.site = Skeleton::Site.new
        # Evaluate the configuration block
        site.instance_exec(&config)

        # Save all the pages to database
        site.pages.each do |page|
          page.model.save!
        end

        # Delete pages not in skeleton
        destroy_deleted_pages!
      end

      def self.migrating?
        ActiveRecord::Migrator.needs_migration?
      end

      def self.destroy_deleted_pages!
        identifiers = site.pages.map(&:identifier)
        pages = Para::SeoTools::Page.where.not(identifier: identifiers)

        if (count = pages.count) > 0
          Rails.logger.info("Destroying #{ count } removed pages from Skeleton")
          pages.destroy_all
        end
      end
    end
  end
end
<commit_msg>wrap skeleton building database queries in a single transaction and add build steps logging<commit_after>module Para
  module SeoTools
    module Skeleton
      extend ActiveSupport::Autoload

      autoload :Site
      autoload :Page
      autoload :Worker

      mattr_accessor :site, :config

      def self.load
        # Ensure routes are loaded to allow skeleton to call routes name helpers
        Rails.application.reload_routes!
        # Skeleton file path
        skeleton_path = Rails.root.join('config', 'skeleton.rb')
        # Load the skeleton file
        require skeleton_path if File.exists?(skeleton_path)
      end

      def self.draw(lazy: false, &block)
        self.config = block
        build unless lazy
      end

      def self.build
        return if migrating?
        return unless ActiveRecord::Base.connection.table_exists?(Para::SeoTools::Page.table_name)

        puts "   * Building app skeleton pages ..."

        self.site = Skeleton::Site.new
        # Evaluate the configuration block
        site.instance_exec(&config)

        # Save all the pages to database
        ActiveRecord::Base.transaction do
          puts "   * Saving generated pages ..."

          site.pages.each do |page|
            page.model.save!
          end

          puts "   * Destroying old pages ..."

          # Delete pages not in skeleton
          destroy_deleted_pages!
        end
      end

      def self.migrating?
        ActiveRecord::Migrator.needs_migration?
      end

      def self.destroy_deleted_pages!
        identifiers = site.pages.map(&:identifier)
        pages = Para::SeoTools::Page.where.not(identifier: identifiers)

        if (count = pages.count) > 0
          Rails.logger.info("Destroying #{ count } removed pages from Skeleton")
          pages.destroy_all
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.07341327873647266, 'fraction_numerical': 0.000584966364434045, 'mean_word_length': 2.131868131868132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4913993', 'n_tokens_mistral': 966, 'n_tokens_neox': 924, 'n_words': 263}