<commit_before>require 'date'

module SaarlandScraper
  BASE_URL = 'http://www.landtag-saar.de'

  class Detail < DetailScraper
    def scrape
      text = SaarlandScraper.build_text_parameter(@legislative_term, @reference)
      parameters = "#{text}&r=saarlandcontenttype%3D%22drucksache%22%20saarlandwahlperiode%3D%22#{@legislative_term}%22"
      mp = mechanize.get "#{BASE_URL}/Service/Seiten/Suche.aspx?#{parameters}"
      entry = SaarlandScraper.extract_search_entry(mp, @legislative_term, @reference)
      return nil if entry.nil?
      SaarlandScraper.extract_paper_from_search_entry(entry, @legislative_term, @reference)
    end
  end

  class Overview < Scraper
    SEARCH_URL = BASE_URL + '/dokumente/drucksachen?FilterName=LinkFilenameNoMenu&FilterMultiValue=Aw*&SortField=DokumentDatum&SortDir=Desc&FilterField1=Wahlperiode&FilterValue1='

    def supports_streaming?
      true
    end

    def scrape(&block)
      @m ||= mechanize
      page = 1
      papers = []
      block = -> (paper) { papers << paper } unless block_given?
      mp = @m.get SEARCH_URL + "#{@legislative_term}" if mp.nil?
      loop do
        logger.debug "[scrape] page: #{page}"
        scrape_page(mp, &block)
        next_page_img = mp.search('//td[@id="bottomPagingCellWPQ1"]//img[contains(@src, "next.gif")]')[0]
        break if next_page_img.nil?
        page += 1
        link_el = next_page_img.parent
        mp = @m.get SaarlandScraper.extract_next_link_url(link_el)
      end
      papers unless block_given?
    end

    def scrape_page(mp, &block)
      streaming = block_given?
      papers = []
      SaarlandScraper.extract_entries(mp).each do |entry|
        begin
          paper = SaarlandScraper.extract_paper(entry)<commit_msg>Fix Saarland again<commit_after>require 'date'

#
# Saarland is based on SharePoint. (No idea why anyone thinks thats a sane idea)
#
# Get Version: https://www.landtag-saar.de/_vti_pvt/service.cnf
# # vti_extenderversion:SR|15.0.0.4797
#
# An request to https://www.landtag-saar.de/_vti_bin/ brings us the following header:
# # MicrosoftSharePointTeamServices:15.0.0.4569
#
# So it is an SharePoint 2013.
# But all the interesting endpoints (/_api, /_vti_bin/listdata.svc/) are locked down.
# Sad :(
#
module SaarlandScraper
  BASE_URL = 'http://www.landtag-saar.de'

  class Detail < DetailScraper
    def scrape
      text = SaarlandScraper.build_text_parameter(@legislative_term, @reference)
      parameters = "#{text}&r=saarlandcontenttype%3D%22drucksache%22%20saarlandwahlperiode%3D%22#{@legislative_term}%22"
      mp = mechanize.get "#{BASE_URL}/Service/Seiten/Suche.aspx?#{parameters}"
      entry = SaarlandScraper.extract_search_entry(mp, @legislative_term, @reference)
      return nil if entry.nil?
      SaarlandScraper.extract_paper_from_search_entry(entry, @legislative_term, @reference)
    end
  end

  class Overview < Scraper
    SEARCH_URL = BASE_URL + '/dokumente/drucksachen?FilterName=LinkFilenameNoMenu&FilterMultiValue=Aw*&SortField=DokumentDatum&SortDir=Desc&Filter=1&FilterField1=Wahlperiode&FilterValue1='

    def supports_streaming?
      true
    end

    def scrape(&block)
      @m ||= mechanize
      page = 1
      papers = []
      block = -> (paper) { papers << paper } unless block_given?
      mp = @m.get SEARCH_URL + "#{@legislative_term}" if mp.nil?
      loop do
        logger.debug "[scrape] page: #{page}"
        scrape_page(mp, &block)
        next_page_img = mp.search('//td[@id="bottomPagingCellWPQ1"]//img[contains(@src, "next.gif")]')[0]
        break if next_page_img.nil?
        page += 1
        link_el = next_page_img.parent
        mp = @m.get SaarlandScraper.extract_next_link_url(link_el)
      end
      papers unless block_given?
    end

    def scrape_page(mp, &block)
      streaming = block_given?
      papers = []
      SaarlandScraper.extract_entries(mp).each do |entry|
        begin
          paper = SaarlandScraper.extract_paper(entry)
{'fraction_non_alphanumeric': 0.11589068825910931, 'fraction_numerical': 0.014423076923076924, 'mean_word_length': 3.5594002306805077, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7226371', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1381, 'n_words': 244}