<commit_before>#
# Copyright (C) 2012 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#

# A small wrapper around the CutyCapt binary.
# 
# Requires a config file in RAILS_ROOT/config/cutycapt.yml that looks like this:
# 
# production:
#   path: /usr/bin/cutycapt
#   delay: 3000
#   timeout: 30000
#   display: ':0'
#
# delay is how many ms to wait before taking the snapshot (to let the page finish rendering)
# display is whatever display cutycapt should use. (You should probably use Xvfb.)

require 'resolv'
require 'netaddr'

class CutyCapt
  CUTYCAPT_DEFAULTS = {
    :delay => 3000,
    :timeout => 60000,
    :ip_blacklist => [ '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '169.254.169.254' ],
    :domain_blacklist => [ ],
    :allowed_schemes => [ 'http', 'https' ]
  }

  cattr_writer :config

  def self.config
    return @@config if defined?(@@config) && @@config
    setting = (Setting.from_config('cutycapt') || {}).symbolize_keys
    @@config = CUTYCAPT_DEFAULTS.merge(setting).with_indifferent_access
    self.process_config
    @@config = nil unless @@config[:path]
    @@config
  end

  def self.process_config
    @@config[:ip_blacklist] = @@config[:ip_blacklist].map {|ip| NetAddr::CIDR.create(ip) } if @@config[:ip_blacklist]
    @@config[:domain_blacklist] = @@config[:domain_blacklist].map {|domain| Resolv::DNS::Name.create(domain) } if @@config[:domain_blacklist]
  end

  def self.logger
    Rails.logger
  end

  def self.enabled?
    return !self.config.nil?
  end

  def self.verify_url(url)
    config = self.config

    uri = URI.parse(url)
    unless config[:allowed_schemes] && config[:allowed_schemes].include?(uri.scheme)
      logger.warn("Skipping non-http[s] URL: #{url}")
      return false
    end

    dns_host = Resolv::DNS::Name.create(uri.host)
    if config[:domain_blacklist] && config[:domain_blacklist].any? {|bl_host| dns_host == bl_host || dns_host.subdomain_of?(bl_host) }
      logger.warn("Skipping url because of blacklisted domain: #{url}")
      return false
    end

    addresses = Resolv.getaddresses(uri.host)
    if config[:ip_blacklist] && addresses.any? {|address| config[:ip_blacklist].any? {|cidr| cidr.matches?(address) rescue false } }
      logger.warn("Skipping url because of blacklisted IP address: #{url}")
      return false
    end

    true
  end

  def self.cuty_arguments(path, url, img_file, format, delay, timeout)
    [ path, "--url=#{url}", "--out=#{img_file}", "--out-format=#{format}", "--delay=#{delay}", "--max-wait=#{timeout}" ]
  end

  def self.snapshot_url(url, format = "png", &block)
    return nil unless config = self.config
    return nil unless self.verify_url(url)

    tmp_file = Tempfile.new(['websnappr', ".#{format}"])
    img_file = tmp_file.path
    # We need to finalize the tmp_file now, because if we don't then it will get closed
    # in the child process below, deleting it. This does introduce a potential race condition
    # but in practice shouldn't be a problem since Tempfiles normally include the process pid.
    tmp_file.close!
    success = true

    start = Time.now
    logger.info("Starting web capture of #{url}")

    if (pid = fork).nil?
      ENV["DISPLAY"] = config[:display] if config[:display]
      Kernel.exec(*cuty_arguments(config[:path], url, img_file, format, config[:delay], config[:timeout]))
    else
      begin
        Timeout::timeout(config[:timeout].to_i / 1000) do
          Process.waitpid(pid)
          unless $?.success?
            logger.error("Capture failed with code: #{$?.exitstatus}")<commit_msg>force cuty_capt to send 'Accept-Language:en'<commit_after>#
# Copyright (C) 2012 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#

# A small wrapper around the CutyCapt binary.
# 
# Requires a config file in RAILS_ROOT/config/cutycapt.yml that looks like this:
# 
# production:
#   path: /usr/bin/cutycapt
#   delay: 3000
#   timeout: 30000
#   display: ':0'
#
# delay is how many ms to wait before taking the snapshot (to let the page finish rendering)
# display is whatever display cutycapt should use. (You should probably use Xvfb.)

require 'resolv'
require 'netaddr'

class CutyCapt
  CUTYCAPT_DEFAULTS = {
    :delay => 3000,
    :timeout => 60000,
    :ip_blacklist => [ '10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', '169.254.169.254' ],
    :domain_blacklist => [ ],
    :allowed_schemes => [ 'http', 'https' ],
    :lang => 'en,*;q=0.9'
  }

  cattr_writer :config

  def self.config
    return @@config if defined?(@@config) && @@config
    setting = (Setting.from_config('cutycapt') || {}).symbolize_keys
    @@config = CUTYCAPT_DEFAULTS.merge(setting).with_indifferent_access
    self.process_config
    @@config = nil unless @@config[:path]
    @@config
  end

  def self.process_config
    @@config[:ip_blacklist] = @@config[:ip_blacklist].map {|ip| NetAddr::CIDR.create(ip) } if @@config[:ip_blacklist]
    @@config[:domain_blacklist] = @@config[:domain_blacklist].map {|domain| Resolv::DNS::Name.create(domain) } if @@config[:domain_blacklist]
  end

  def self.logger
    Rails.logger
  end

  def self.enabled?
    return !self.config.nil?
  end

  def self.verify_url(url)
    config = self.config

    uri = URI.parse(url)
    unless config[:allowed_schemes] && config[:allowed_schemes].include?(uri.scheme)
      logger.warn("Skipping non-http[s] URL: #{url}")
      return false
    end

    dns_host = Resolv::DNS::Name.create(uri.host)
    if config[:domain_blacklist] && config[:domain_blacklist].any? {|bl_host| dns_host == bl_host || dns_host.subdomain_of?(bl_host) }
      logger.warn("Skipping url because of blacklisted domain: #{url}")
      return false
    end

    addresses = Resolv.getaddresses(uri.host)
    if config[:ip_blacklist] && addresses.any? {|address| config[:ip_blacklist].any? {|cidr| cidr.matches?(address) rescue false } }
      logger.warn("Skipping url because of blacklisted IP address: #{url}")
      return false
    end

    true
  end

  def self.cuty_arguments(path, url, img_file, format, delay, timeout, lang)
    [ path, "--url=#{url}", "--out=#{img_file}", "--out-format=#{format}", "--delay=#{delay}", "--max-wait=#{timeout}", "--header=Accept-Language:#{lang}" ]
  end

  def self.snapshot_url(url, format = "png", &block)
    return nil unless config = self.config
    return nil unless self.verify_url(url)

    tmp_file = Tempfile.new(['websnappr', ".#{format}"])
    img_file = tmp_file.path
    # We need to finalize the tmp_file now, because if we don't then it will get closed
    # in the child process below, deleting it. This does introduce a potential race condition
    # but in practice shouldn't be a problem since Tempfiles normally include the process pid.
    tmp_file.close!
    success = true

    start = Time.now
    logger.info("Starting web capture of #{url}")

    if (pid = fork).nil?
      ENV["DISPLAY"] = config[:display] if config[:display]
      Kernel.exec(*cuty_arguments(config[:path], url, img_file, format, config[:delay], config[:timeout], config[:lang]))
    else
      begin
        Timeout::timeout(config[:timeout].to_i / 1000) do
          Process.waitpid(pid)
          unless $?.success?
            logger.error("Capture failed with code: #{$?.exitstatus}")
{'fraction_non_alphanumeric': 0.13071352249792237, 'fraction_numerical': 0.015671375994301316, 'mean_word_length': 3.68, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7343644', 'n_tokens_mistral': 2784, 'n_tokens_neox': 2662, 'n_words': 800}