<commit_before>class FetchPapersBayernJob < FetchPapersJob

  @state = 'BY'

  def self.perform(*params)
    super

    import_new_papers
    load_paper_details
    download_papers
  end

  def self.import_new_papers
    result = BayernLandtagScraper::Overview.new.scrape
    result.each do |item|
      item['reference'] = item['full_reference'].split("/").last
      item.delete 'full_reference'
      unless Paper.where(body: @body, legislative_term: item['legislative_term'], reference: item['reference']).exists?
        puts "Got new Paper: [#{item['reference']}] \"#{item['title']}\""
        paper = Paper.new(item)
        paper.body = @body
        paper.save
      end
    end
  end

  def self.load_paper_details
    @papers = Paper.find_by_sql(
      ["SELECT p.* FROM papers p LEFT OUTER JOIN paper_originators o ON (o.paper_id = p.id) WHERE p.body_id = ? AND o.id IS NULL LIMIT 25", @body.id])

    @papers.each do |paper|
      puts "Loading details for Paper [#{paper.reference}]"
      org = Organization.find_or_create_by(name: detail[:originator])
      detail = BayernLandtagScraper::Detail.new(paper.legislative_term, paper.reference).scrape
      puts "- Originator: #{org.name}"
      unless paper.originator_organizations.include? org
        paper.originator_organizations << org
        paper.save
      end
    end
  end

end<commit_msg>Use lowercased organization name in paper details crawler<commit_after>class FetchPapersBayernJob < FetchPapersJob

  @state = 'BY'

  def self.perform(*params)
    super

    import_new_papers
    load_paper_details
    download_papers
  end

  def self.import_new_papers
    result = BayernLandtagScraper::Overview.new.scrape
    result.each do |item|
      item['reference'] = item['full_reference'].split("/").last
      item.delete 'full_reference'
      unless Paper.where(body: @body, legislative_term: item['legislative_term'], reference: item['reference']).exists?
        puts "Got new Paper: [#{item['reference']}] \"#{item['title']}\""
        paper = Paper.new(item)
        paper.body = @body
        paper.save
      end
    end
  end

  def self.load_paper_details
    @papers = Paper.find_by_sql(
      ["SELECT p.* FROM papers p LEFT OUTER JOIN paper_originators o ON (o.paper_id = p.id) WHERE p.body_id = ? AND o.id IS NULL LIMIT 25", @body.id])

    @papers.each do |paper|
      puts "Loading details for Paper [#{paper.reference}]"
      detail = BayernLandtagScraper::Detail.new(paper.legislative_term, paper.reference).scrape
      org = Organization.where('lower(name) = ?', detail[:originator].mb_chars.downcase.to_s).first_or_create(name: detail[:originator])
      puts "- Originator: #{org.name}"
      unless paper.originator_organizations.include? org
        paper.originator_organizations << org
        paper.save
      end
    end
  end

end
{'fraction_non_alphanumeric': 0.11756423794438578, 'fraction_numerical': 0.0014079549454417458, 'mean_word_length': 3.2995461422087744, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7301598', 'n_tokens_mistral': 916, 'n_tokens_neox': 885, 'n_words': 203}