<commit_before>class ImportNewPapersJob < ActiveJob::Base
  queue_as :import

  def perform(body, legislative_term)
    fail "No scraper found for body #{body.state}" if body.scraper.nil?
    @body = body
    @legislative_term = legislative_term
    @scraper = @body.scraper::Overview.new(legislative_term)
    @load_details = @body.scraper.const_defined?(:Detail)
    if @scraper.supports_pagination?
      scrape_paginated
    else
      scrape_single_page
    end
    logger.info "Importing #{@body.state} done."
  end

  def scrape_paginated
    page = 1
    found_new_paper = false
    loop do
      logger.info "Importing #{@body.state} - Page #{page}"
      found_new_paper = false
      @scraper.scrape(page).each do |item|
        next if Paper.where(body: @body, legislative_term: item[:legislative_term], reference: item[:reference]).exists?
        on_item(item)
        found_new_paper = true
      end
      page += 1
      break unless found_new_paper
    end
  end

  def scrape_single_page
    logger.info "Importing #{@body.state} - Single Page"
    block = lambda do |item|
      return if Paper.where(body: @body, legislative_term: item[:legislative_term], reference: item[:reference]).exists?
      on_item(item)
    end
    if @scraper.supports_streaming?
      @scraper.scrape(&block)
    else
      @scraper.scrape.each(&block)
    end
  end

  def on_item(item)
    logger.info "New Paper: [#{item[:reference]}] \"#{item[:title]}\""
    paper = Paper.create!(item.except(:full_reference).merge(body: @body))
    LoadPaperDetailsJob.perform_later(paper) if (item[:originators].blank? || item[:answerers].blank?) && @load_details
    StorePaperPDFJob.perform_later(paper)
  end
end<commit_msg>ImportNewPapersJob: log more stats<commit_after>class ImportNewPapersJob < ActiveJob::Base
  queue_as :import

  def perform(body, legislative_term)
    fail "No scraper found for body #{body.state}" if body.scraper.nil?
    @body = body
    @legislative_term = legislative_term
    @scraper = @body.scraper::Overview.new(legislative_term)
    @load_details = @body.scraper.const_defined?(:Detail)
    @new_papers = 0
    @old_papers = 0
    if @scraper.supports_pagination?
      scrape_paginated
    else
      scrape_single_page
    end
    logger.info "Importing #{@body.state} #{@legislative_term} done. #{@new_papers} new Papers, #{@old_papers} old Papers."
  end

  def scrape_paginated
    page = 1
    found_new_paper = false
    loop do
      logger.info "Importing #{@body.state} - Page #{page}"
      found_new_paper = false
      @scraper.scrape(page).each do |item|
        if Paper.where(body: @body, legislative_term: item[:legislative_term], reference: item[:reference]).exists?
          @old_papers += 1
          next
        end
        on_item(item)
        found_new_paper = true
      end
      page += 1
      break unless found_new_paper
    end
  end

  def scrape_single_page
    logger.info "Importing #{@body.state} - Single Page"
    block = lambda do |item|
      if Paper.where(body: @body, legislative_term: item[:legislative_term], reference: item[:reference]).exists?
        @old_papers += 1
        return
      end
      on_item(item)
    end
    if @scraper.supports_streaming?
      @scraper.scrape(&block)
    else
      @scraper.scrape.each(&block)
    end
  end

  def on_item(item)
    logger.info "New Paper: [#{item[:reference]}] \"#{item[:title]}\""
    paper = Paper.create!(item.except(:full_reference).merge(body: @body))
    LoadPaperDetailsJob.perform_later(paper) if (item[:originators].blank? || item[:answerers].blank?) && @load_details
    StorePaperPDFJob.perform_later(paper)
    @new_papers += 1
  end
end
{'fraction_non_alphanumeric': 0.11790274381961424, 'fraction_numerical': 0.0024449877750611247, 'mean_word_length': 3.316529894490035, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4532764', 'n_tokens_mistral': 1253, 'n_tokens_neox': 1202, 'n_words': 259}