<commit_before>require 'github_api'
require 'fileutils'
require 'socket'

module Overlay
  class Github
    include Overlay::Engine.routes.url_helpers
    # Cycle through all configured repositories and overlay
    #
    def self.process_overlays
      # This can be called in an application initializer which will
      # load anytime the environment is loaded.  Make sure we are prepared to run
      # this.
      #
      return unless (config.host_port || ENV['SERVER_HOST_PORT'] || defined? Rails::Server)

      # Configure github api
      configure

      Overlay.configuration.repositories.each do |repo_config|
        next unless repo_config.class == GithubRepo

        # Validate repository config
        raise 'Respository config missing user' if (!repo_config[:user] || repo_config[:user].nil?)
        raise 'Respository config missing repo' if (!repo_config[:repo] || repo_config[:repo].nil?)

        repo_config[:branch] ||= 'master'

        register_web_hook(repo_config)

        overlay_repo repo_config
      end
    end

    # Register our listener on the repo
    #
    def self.register_web_hook repo_config
      # Make sure our routes are loaded
      Rails.application.reload_routes!

      # Build hook url
      host = config.host_name || ENV['SERVER_HOST_NAME'] || Socket.gethostname
      port = config.host_port || ENV['SERVER_HOST_PORT'] || Rails::Server.new.options[:Port]
      path = Overlay::Engine.routes.url_for({:controller=>"overlay/github", :action=>"update", :only_path => true})
      uri  = ActionDispatch::Http::URL::url_for({:host => host, :port => port, :path => "#{config.relative_root_url}#{path}"})

      # Retrieve current web hooks
      current_hooks = github_repo.hooks.list(repo_config[:user], repo_config[:repo]).response.body
      if current_hooks.find {|hook| hook.config.url == uri}.nil?
        # register hook
        github_repo.hooks.create(repo_config[:user], repo_config[:repo], name: 'web', active: true, config: {:url => uri, :content_type => 'json'})
      end
    end

    def self.overlay_repo repo_config
      Thread.new do
        # Get our root entries
        #
        root = repo_config[:root_source_path] || '/'

        # If we have a root defined, jump right into it
        #
        if (root != '/')
          overlay_directory(root, repo_config)
        else
          root_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], root, ref: repo_config[:branch]).response.body

          # We aren't pulling anything out of root.  Cycle through directories and overlay
          #
          root_entries.each do |entry|
            if entry.type == 'dir'
              overlay_directory(entry.path, repo_config)
            end
          end
        end
      end
    end

    def self.overlay_directory path, repo_config
      root_path = repo_config[:root_dest_path].empty? ? "#{Rails.application.root}" : "#{Rails.application.root}/#{repo_config[:root_dest_path]}"
      dynamic_path = path.partition(repo_config[:root_source_path]).last

      FileUtils.mkdir_p "#{root_path}/#{dynamic_path}"
      directory_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], path, ref: repo_config[:branch]).response.body<commit_msg>Add logging for thread processing<commit_after>require 'github_api'
require 'fileutils'
require 'socket'

module Overlay
  class Github
    include Overlay::Engine.routes.url_helpers
    # Cycle through all configured repositories and overlay
    #
    def self.process_overlays
      # This can be called in an application initializer which will
      # load anytime the environment is loaded.  Make sure we are prepared to run
      # this.
      #
      return unless (config.host_port || ENV['SERVER_HOST_PORT'] || defined? Rails::Server)

      # Configure github api
      configure

      Overlay.configuration.repositories.each do |repo_config|
        next unless repo_config.class == GithubRepo

        # Validate repository config
        raise 'Respository config missing user' if (!repo_config[:user] || repo_config[:user].nil?)
        raise 'Respository config missing repo' if (!repo_config[:repo] || repo_config[:repo].nil?)

        repo_config[:branch] ||= 'master'

        register_web_hook(repo_config)

        overlay_repo repo_config
      end
    end

    # Register our listener on the repo
    #
    def self.register_web_hook repo_config
      # Make sure our routes are loaded
      Rails.application.reload_routes!

      # Build hook url
      host = config.host_name || ENV['SERVER_HOST_NAME'] || Socket.gethostname
      port = config.host_port || ENV['SERVER_HOST_PORT'] || Rails::Server.new.options[:Port]
      path = Overlay::Engine.routes.url_for({:controller=>"overlay/github", :action=>"update", :only_path => true})
      uri  = ActionDispatch::Http::URL::url_for({:host => host, :port => port, :path => "#{config.relative_root_url}#{path}"})

      # Retrieve current web hooks
      current_hooks = github_repo.hooks.list(repo_config[:user], repo_config[:repo]).response.body
      if current_hooks.find {|hook| hook.config.url == uri}.nil?
        # register hook
        github_repo.hooks.create(repo_config[:user], repo_config[:repo], name: 'web', active: true, config: {:url => uri, :content_type => 'json'})
      end
    end

    def self.overlay_repo repo_config
      Thread.new do
        Rails.logger.info "Start processing repo with config #{repo_config.inspect}"
        # Get our root entries
        #
        root = repo_config[:root_source_path] || '/'

        # If we have a root defined, jump right into it
        #
        if (root != '/')
          overlay_directory(root, repo_config)
        else
          root_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], root, ref: repo_config[:branch]).response.body

          # We aren't pulling anything out of root.  Cycle through directories and overlay
          #
          root_entries.each do |entry|
            if entry.type == 'dir'
              overlay_directory(entry.path, repo_config)
            end
          end
        end
        Rails.logger.info "Finished processing repo with config #{repo_config.inspect}"
      end
    end

    def self.overlay_directory path, repo_config
      root_path = repo_config[:root_dest_path].empty? ? "#{Rails.application.root}" : "#{Rails.application.root}/#{repo_config[:root_dest_path]}"
      dynamic_path = path.partition(repo_config[:root_source_path]).last

      FileUtils.mkdir_p "#{root_path}/#{dynamic_path}"
      directory_entries = github_repo.contents.get(repo_config[:user], repo_config[:repo], path, ref: repo_config[:branch]).response.body
{'fraction_non_alphanumeric': 0.11159182455616888, 'fraction_numerical': 0.0, 'mean_word_length': 3.0803408399269627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7620510', 'n_tokens_mistral': 1950, 'n_tokens_neox': 1915, 'n_words': 475}