<commit_before>require File.join(File.dirname(__FILE__), 'init')

require 'yard'
require 'sinatra'
require 'json'
require 'fileutils'
require 'airbrake'

require 'extensions'
require 'scm_router'
require 'scm_checkout'
require 'gems_router'
require 'featured_router'
require 'stdlib_router'
require 'recent_store'

class Hash; alias blank? empty? end
class NilClass; def blank?; true end end

class NoCacheEmptyBody
  def initialize(app) @app = app end
  def call(env)
    status, headers, body = *@app.call(env)
    headers['Cache-Control'] = 'max-age=0' if headers['Content-Length'].length == 0
    [status, headers, body]
  end
end

class DocServer < Sinatra::Base
  include YARD::Server

  def self.adapter_options
    caching = %w(staging production).include?(ENV['RACK_ENV']) ? $CONFIG.caching : false
    {
      :libraries => {},
      :options => {caching: caching, single_library: false},
      :server_options => {DocumentRoot: STATIC_PATH}
    }
  end

  def self.load_configuration
    set :name, 'RubyDoc.info'
    set :url, 'http://rubydoc.info'

    set :disallowed_projects, []
    set :disallowed_gems, []
    set :whitelisted_projects, []
    set :whitelisted_gems, []
    set :caching, false

    if $CONFIG.varnish_host
      set :protection, :origin_whitelist => ["http://#{$CONFIG.varnish_host}"]
    end

    puts ">> Loading #{CONFIG_FILE}"
    $CONFIG.each do |key, value|
      set key, value
    end
  end

  def self.copy_static_files
    # Copy template files
    puts ">> Copying static system files..."
    YARD::Templates::Engine.template(:default, :fulldoc, :html).full_paths.each do |path|
      %w(css js images).each do |ext|
        srcdir, dstdir = File.join(path, ext), File.join('public', ext)
        next unless File.directory?(srcdir)
        system "mkdir -p #{dstdir} && cp #{srcdir}/* #{dstdir}/"
      end
    end
  end

  def self.load_gems_adapter
    remote_file = REMOTE_GEMS_FILE
    contents = File.readlines(remote_file)
    puts ">> Loading remote gems list..."
    opts = adapter_options
    contents.each do |line|
      name, *versions = *line.split(/\s+/)
      opts[:libraries][name] = versions.map do |v|
        ver, platform = *v.split(',')
        lib = LibraryVersion.new(name, ver, nil, :remote_gem)
        lib.platform = platform
        lib
      end
    end
    opts[:options][:router] = GemsRouter
    set :gems_adapter, $gems_adapter = RackAdapter.new(*opts.values)
  rescue Errno::ENOENT
    log.error "No remote_gems file to load remote gems from, not serving gems."
  end

  def self.load_scm_adapter
    opts = adapter_options
    opts[:options][:router] = ScmRouter
    opts[:libraries] = ScmLibraryStore.new
    set :scm_adapter, RackAdapter.new(*opts.values)
  end

  def self.find_featured_yardoc(name, libdir)
    [File.join(FEATURED_PATH, libdir, '.yardoc'), File.join(libdir, '.yardoc')].each do |path|
      return path if File.directory?(path)
    end
    log.error "Invalid featured repository #{libdir} for #{name}"
    exit
  end

  def self.load_featured_adapter
    if !$CONFIG.featured
      log.error "No featured section in config.yaml, not serving featured docs."
      set :featured_adapter, nil
      return
    end

    opts = adapter_options
    opts[:options][:router] = FeaturedRouter
    $CONFIG.featured.each do |key, value|
      opts[:libraries][key] = case value
      when String
        if value == "gem"
          $gems_adapter && $gems_adapter.libraries[key] ? $gems_adapter.libraries[key] : []
        else
          [LibraryVersion.new(key, nil, find_featured_yardoc(key, value))]
        end
      when Array, Hash
        value = value.first if Array === value
        value.map do |version, libdir|
          LibraryVersion.new(key, version, find_featured_yardoc(key, libdir))
        end
      end
    end
    set :featured_adapter, RackAdapter.new(*opts.values)
  end


  def self.load_stdlib_adapter
    unless File.directory?(STDLIB_PATH)
      log.error "No stdlib repository, not serving standard library"
      return
    end

    opts = adapter_options
    opts[:options][:router] = StdlibRouter
    versions = Dir.glob(File.join(STDLIB_PATH, '*'))
    versions.sort_by {|v| File.basename(v) }.each do |version|
      next unless File.directory?(version)
      version = File.basename(version)
      libs = Dir.glob(File.join(STDLIB_PATH, version, '*'))
      libs.each do |lib|
        next unless File.directory?(lib)
        libname = File.basename(lib)
        yardoc = File.join(lib, '.yardoc')
        opts[:libraries][libname] ||= []
        opts[:libraries][libname] << LibraryVersion.new(libname, version, nil, :disk_on_demand)
      end
    end
    set :stdlib_adapter, RackAdapter.new(*opts.values)
  end

  def self.post_all(*args, &block)
    args.each {|arg| post(arg, &block) }
  end

  use Rack::ConditionalGet
  use Rack::Head
  use NoCacheEmptyBody

  enable :static
  enable :dump_errors
  enable :lock
  disable :raise_errors

  set :views, TEMPLATES_PATH
  set :public_folder, STATIC_PATH
  set :repos, REPOS_PATH
  set :tmp, TMP_PATH

  configure(:production) do
    enable :logging
    # log to file
    file = File.open("log/sinatra.log", "a")
    STDOUT.reopen(file)
    STDERR.reopen(file)
  end

  configure do
    load_configuration
    load_gems_adapter
    load_scm_adapter
    load_featured_adapter
    load_stdlib_adapter
    copy_static_files
  end<commit_msg>Use ETag Rack middleware<commit_after>require File.join(File.dirname(__FILE__), 'init')

require 'yard'
require 'sinatra'
require 'json'
require 'fileutils'
require 'airbrake'

require 'extensions'
require 'scm_router'
require 'scm_checkout'
require 'gems_router'
require 'featured_router'
require 'stdlib_router'
require 'recent_store'

require 'rack/etag'

class Hash; alias blank? empty? end
class NilClass; def blank?; true end end

class NoCacheEmptyBody
  def initialize(app) @app = app end
  def call(env)
    status, headers, body = *@app.call(env)
    headers['Cache-Control'] = 'max-age=0' if headers['Content-Length'].length == 0
    [status, headers, body]
  end
end

class DocServer < Sinatra::Base
  include YARD::Server

  def self.adapter_options
    caching = %w(staging production).include?(ENV['RACK_ENV']) ? $CONFIG.caching : false
    {
      :libraries => {},
      :options => {caching: caching, single_library: false},
      :server_options => {DocumentRoot: STATIC_PATH}
    }
  end

  def self.load_configuration
    set :name, 'RubyDoc.info'
    set :url, 'http://rubydoc.info'

    set :disallowed_projects, []
    set :disallowed_gems, []
    set :whitelisted_projects, []
    set :whitelisted_gems, []
    set :caching, false

    if $CONFIG.varnish_host
      set :protection, :origin_whitelist => ["http://#{$CONFIG.varnish_host}"]
    end

    puts ">> Loading #{CONFIG_FILE}"
    $CONFIG.each do |key, value|
      set key, value
    end
  end

  def self.copy_static_files
    # Copy template files
    puts ">> Copying static system files..."
    YARD::Templates::Engine.template(:default, :fulldoc, :html).full_paths.each do |path|
      %w(css js images).each do |ext|
        srcdir, dstdir = File.join(path, ext), File.join('public', ext)
        next unless File.directory?(srcdir)
        system "mkdir -p #{dstdir} && cp #{srcdir}/* #{dstdir}/"
      end
    end
  end

  def self.load_gems_adapter
    remote_file = REMOTE_GEMS_FILE
    contents = File.readlines(remote_file)
    puts ">> Loading remote gems list..."
    opts = adapter_options
    contents.each do |line|
      name, *versions = *line.split(/\s+/)
      opts[:libraries][name] = versions.map do |v|
        ver, platform = *v.split(',')
        lib = LibraryVersion.new(name, ver, nil, :remote_gem)
        lib.platform = platform
        lib
      end
    end
    opts[:options][:router] = GemsRouter
    set :gems_adapter, $gems_adapter = RackAdapter.new(*opts.values)
  rescue Errno::ENOENT
    log.error "No remote_gems file to load remote gems from, not serving gems."
  end

  def self.load_scm_adapter
    opts = adapter_options
    opts[:options][:router] = ScmRouter
    opts[:libraries] = ScmLibraryStore.new
    set :scm_adapter, RackAdapter.new(*opts.values)
  end

  def self.find_featured_yardoc(name, libdir)
    [File.join(FEATURED_PATH, libdir, '.yardoc'), File.join(libdir, '.yardoc')].each do |path|
      return path if File.directory?(path)
    end
    log.error "Invalid featured repository #{libdir} for #{name}"
    exit
  end

  def self.load_featured_adapter
    if !$CONFIG.featured
      log.error "No featured section in config.yaml, not serving featured docs."
      set :featured_adapter, nil
      return
    end

    opts = adapter_options
    opts[:options][:router] = FeaturedRouter
    $CONFIG.featured.each do |key, value|
      opts[:libraries][key] = case value
      when String
        if value == "gem"
          $gems_adapter && $gems_adapter.libraries[key] ? $gems_adapter.libraries[key] : []
        else
          [LibraryVersion.new(key, nil, find_featured_yardoc(key, value))]
        end
      when Array, Hash
        value = value.first if Array === value
        value.map do |version, libdir|
          LibraryVersion.new(key, version, find_featured_yardoc(key, libdir))
        end
      end
    end
    set :featured_adapter, RackAdapter.new(*opts.values)
  end


  def self.load_stdlib_adapter
    unless File.directory?(STDLIB_PATH)
      log.error "No stdlib repository, not serving standard library"
      return
    end

    opts = adapter_options
    opts[:options][:router] = StdlibRouter
    versions = Dir.glob(File.join(STDLIB_PATH, '*'))
    versions.sort_by {|v| File.basename(v) }.each do |version|
      next unless File.directory?(version)
      version = File.basename(version)
      libs = Dir.glob(File.join(STDLIB_PATH, version, '*'))
      libs.each do |lib|
        next unless File.directory?(lib)
        libname = File.basename(lib)
        yardoc = File.join(lib, '.yardoc')
        opts[:libraries][libname] ||= []
        opts[:libraries][libname] << LibraryVersion.new(libname, version, nil, :disk_on_demand)
      end
    end
    set :stdlib_adapter, RackAdapter.new(*opts.values)
  end

  def self.post_all(*args, &block)
    args.each {|arg| post(arg, &block) }
  end

  use Rack::ConditionalGet
  use Rack::Head
  use Rack::ETag
  #use NoCacheEmptyBody

  enable :static
  enable :dump_errors
  enable :lock
  disable :raise_errors

  set :views, TEMPLATES_PATH
  set :public_folder, STATIC_PATH
  set :repos, REPOS_PATH
  set :tmp, TMP_PATH

  configure(:production) do
    enable :logging
    # log to file
    file = File.open("log/sinatra.log", "a")
    STDOUT.reopen(file)
    STDERR.reopen(file)
  end

  configure do
    load_configuration
    load_gems_adapter
    load_scm_adapter
    load_featured_adapter
    load_stdlib_adapter
    copy_static_files
  end
{'fraction_non_alphanumeric': 0.11780922624517552, 'fraction_numerical': 0.0003675794890645102, 'mean_word_length': 3.1922187981510017, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5018731', 'n_tokens_mistral': 3634, 'n_tokens_neox': 3536, 'n_words': 876}