<commit_before>class StorePaperPDFJob < PaperJob
  include ActiveJob::Retry

  queue_as :store

  variable_retry delays: [5, 15, 30, 90],
                 retryable_exceptions: [Patron::TimeoutError]

  def perform(paper, options = {})
    options.reverse_merge!(force: false)

    logger.info "Downloading PDF for Paper [#{paper.body.state} #{paper.full_reference}]"
    fail 'AppStorage: Bucket not available!' if AppStorage.bucket.nil?

    if !AppStorage.bucket.files.head(paper.path).nil? && !options[:force]
      logger.info "PDF for Paper [#{paper.body.state} #{paper.full_reference}] already exists in Storage"
      return
    end

    ret = download_paper(paper)
    fail "Downloading Paper [#{paper.body.state} #{paper.full_reference}] failed" unless ret

    file = AppStorage.bucket.files.new(key: paper.path, public: true, body: File.open(paper.local_path))
    file.save

    ThumbnailFirstPageJob.perform_later(paper, force: options[:force]) if paper.thumbnail_url.blank? || options[:force]
    CountPageNumbersJob.perform_later(paper) if paper.page_count.blank? || options[:force]
    ExtractTextFromPaperJob.perform_later(paper) if paper.contents.blank? || options[:force]
    ExtractLastModifiedFromPaperJob.perform_later(paper) if paper.pdf_last_modified.blank?
  end

  def patron_session
    sess = Patron::Session.new
    sess.connect_timeout = 5
    sess.timeout = 60
    sess.headers['User-Agent'] = Rails.configuration.x.user_agent
    sess
  end

  def download_paper(paper)
    session = patron_session
    filepath = paper.local_path
    folder = filepath.dirname
    FileUtils.mkdir_p folder

    resp = session.get(paper.url)
    if resp.status != 200
      logger.warn "Download failed with status #{resp.status} for Paper [#{paper.body.state} #{paper.full_reference}]"
      return false
    end

    content_type = content_type(resp)

    if paper.body.state == 'HH' && content_type != 'application/pdf'
      js = resp.body.match(/location\.replace\(['"](.+)['"]\)/)
      fail "Could not extract PDF path for Hamburg Paper [#{paper.body.state} #{paper.full_reference}]" if js.nil?
      url = Addressable::URI.parse(paper.url).join(js[1]).normalize.to_s
      resp = session.get(url)
      if resp.status != 200
        logger.warn "Download failed with status #{resp.status} for Paper [#{paper.body.state} #{paper.full_reference}]"
        return false
      end
      content_type = content_type(resp)
    end
<commit_msg>SN: call viewer URL to make pdf downloadable again<commit_after>class StorePaperPDFJob < PaperJob
  include ActiveJob::Retry

  queue_as :store

  variable_retry delays: [5, 15, 30, 90],
                 retryable_exceptions: [Patron::TimeoutError]

  def perform(paper, options = {})
    options.reverse_merge!(force: false)

    logger.info "Downloading PDF for Paper [#{paper.body.state} #{paper.full_reference}]"
    fail 'AppStorage: Bucket not available!' if AppStorage.bucket.nil?

    if !AppStorage.bucket.files.head(paper.path).nil? && !options[:force]
      logger.info "PDF for Paper [#{paper.body.state} #{paper.full_reference}] already exists in Storage"
      return
    end

    ret = download_paper(paper)
    fail "Downloading Paper [#{paper.body.state} #{paper.full_reference}] failed" unless ret

    file = AppStorage.bucket.files.new(key: paper.path, public: true, body: File.open(paper.local_path))
    file.save

    ThumbnailFirstPageJob.perform_later(paper, force: options[:force]) if paper.thumbnail_url.blank? || options[:force]
    CountPageNumbersJob.perform_later(paper) if paper.page_count.blank? || options[:force]
    ExtractTextFromPaperJob.perform_later(paper) if paper.contents.blank? || options[:force]
    ExtractLastModifiedFromPaperJob.perform_later(paper) if paper.pdf_last_modified.blank?
  end

  def patron_session
    sess = Patron::Session.new
    sess.connect_timeout = 5
    sess.timeout = 60
    sess.headers['User-Agent'] = Rails.configuration.x.user_agent
    sess
  end

  def download_paper(paper)
    session = patron_session
    filepath = paper.local_path
    folder = filepath.dirname
    FileUtils.mkdir_p folder

    resp = session.get(paper.url)

    if resp.status == 404 && paper.body.state == 'SN'
      # Sachsen: call viewer URL to make pdf downloadable again
      m = paper.url.match(/\/(\d+)_Drs_(\d+)_(\d+)_.+.pdf$/)
      unless m.nil?
        session.get "http://edas.landtag.sachsen.de/viewer/viewer_navigation.aspx?dok_nr=#{m[2]}&dok_art=Drs&leg_per=#{m[1]}&pos_dok=#{m[3]}"
        # and try again
        resp = session.get(paper.url)
      end
    end

    if resp.status != 200
      logger.warn "Download failed with status #{resp.status} for Paper [#{paper.body.state} #{paper.full_reference}]"
      return false
    end

    content_type = content_type(resp)

    if paper.body.state == 'HH' && content_type != 'application/pdf'
      js = resp.body.match(/location\.replace\(['"](.+)['"]\)/)
      fail "Could not extract PDF path for Hamburg Paper [#{paper.body.state} #{paper.full_reference}]" if js.nil?
      url = Addressable::URI.parse(paper.url).join(js[1]).normalize.to_s
      resp = session.get(url)
      if resp.status != 200
        logger.warn "Download failed with status #{resp.status} for Paper [#{paper.body.state} #{paper.full_reference}]"
        return false
      end
      content_type = content_type(resp)
    end

{'fraction_non_alphanumeric': 0.12931995540691194, 'fraction_numerical': 0.007432181345224823, 'mean_word_length': 3.596925704526046, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5085207', 'n_tokens_mistral': 1756, 'n_tokens_neox': 1671, 'n_words': 375}