<commit_before>require 'digest/sha2'

module Heathen
  # checks if the uploaded file has already been processed,
  # and returns an identical dragonfly job, or a new job
  class Inquisitor

    attr_reader :redis, :converter, :params

    def initialize(converter, params)
      @redis     = Heathen::App.redis
      @converter = converter
      @params    = params
    end

    def can_convert?(job)
      job.mime_type == 'application/pdf' || job.image? || Encoders.can_encode?(job)
    end

    def job

      job = make_job
      key = content_hash(job)

      if serialized = redis[key]
        job = converter.job_class.deserialize(serialized, converter)

      else
        job.meta.merge!(meta_data.merge(:mime_type => job.mime_type))

        uid = job.store
        job = converter.fetch(uid)

        redis[key] = job.serialize
      end

      [job, can_convert?(job)]
    end

    private

      def content_hash(job)
        Digest::SHA2.file(job.path).hexdigest
      end

      def make_job
        if file = params[:file]
          converter.log.info "\n\n\nfile: #{file.inspect}"
          converter.new_job(file.fetch(:tempfile), name: file.fetch(:filename))

        elsif url = params[:url]
          converter.fetch_url(url)<commit_msg>Do not fetch URLs from Dragonfly, let wkhtmltopdf handle it instead<commit_after>require 'digest/sha2'

module Heathen
  # checks if the uploaded file has already been processed,
  # and returns an identical dragonfly job, or a new job
  class Inquisitor

    attr_reader :redis, :converter, :params

    def initialize(converter, params)
      @redis     = Heathen::App.redis
      @converter = converter
      @params    = params
    end

    def can_convert?(job)
      job.mime_type == 'application/pdf' || job.image? || Encoders.can_encode?(job)
    end

    def job

      job = make_job
      key = content_hash(job)

      if serialized = redis[key]
        job = converter.job_class.deserialize(serialized, converter)

      else
        job.meta.merge!(meta_data.merge(:mime_type => job.mime_type))

        uid = job.store
        job = converter.fetch(uid)

        redis[key] = job.serialize
      end

      [job, can_convert?(job)]
    end

    private

      def content_hash(job)
        Digest::SHA2.file(job.path).hexdigest
      end

      def make_job
        if file = params[:file]
          converter.log.info "\n\n\nfile: #{file.inspect}"
          converter.new_job(file.fetch(:tempfile), name: file.fetch(:filename))

        elsif url = params[:url]
          converter.new_job(params[:url], name: params[:url].gsub(/[^\w]+/, '-'))
{'fraction_non_alphanumeric': 0.10966755827283149, 'fraction_numerical': 0.0015284677111196026, 'mean_word_length': 2.467549668874172, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7005816', 'n_tokens_mistral': 852, 'n_tokens_neox': 808, 'n_words': 193}