<commit_before>module SachsenScraper
  BASE_URL = 'http://edas.landtag.sachsen.de'
  SEARCH_FIELD = 'ctl00$masterContentCallback$content$suchmaske$tblSearch$tabSuche$panelUmSuchmaskeEinfach$suchmaskeEinfachCallback'
  SEARCH_FIELD2 =
    'ctl00_masterContentCallback_content_suchmaske_tblSearch_tabSuche_panelUmSuchmaskeEinfach_suchmaskeEinfachCallback_sb_Einf'

  def self.extract_overview_items(page)
    page.search('//td[@class="dxdvItem_EDAS"]/table')
  end

  def self.extract_title(item)
    item.search('.//tr[3]/td/b').text.strip
  end

  def self.extract_meta_text(item)
    text = item.search('.//tr[3]/td').text.split("\n").find { |t| t.include?('KlAnfr') || t.include?('GrAnfr') }
    text.strip if !text.nil?
  end

  def self.extract_meta_data(text)
    type = extract_type(text)
    if type == Paper::DOCTYPE_MAJOR_INTERPELLATION
      # empty braces for same result count
      m = text.match(/^GrAnfr ()(.+?) ([\d\.]+) Drs ([\d\/]+)$/)
    else
      m = text.match(/^KlAnfr (.+?) ([A-Z][a-zA-Z]{2}|[A-Z]{2,}[[:alnum:]\s]+) ([\d\.]+) Drs ([\d\/]+)$/)
    end
    return nil if m.nil?
    {
      person: m[1],
      party: m[2],
      published_at: Date.parse(m[3]),
      full_reference: m[4]
    }
  end

  def self.extract_type(text)
    if text[0..5] == 'KlAnfr'
      Paper::DOCTYPE_MINOR_INTERPELLATION
    else
      Paper::DOCTYPE_MAJOR_INTERPELLATION
    end
  end

  def self.extract_date(meta_text)
    Date.parse(meta_text.match(/\d+\.\d+\.\d{4}/).to_s)
  end

  def self.extract_overview_paper(item, doctype)
    meta_text = extract_meta_text(item)
    meta = extract_meta_data(meta_text)
    full_reference = meta[:full_reference]
    legislative_term, reference = full_reference.split('/')
    originators = {
      people: [meta[:person]].reject(&:blank?),
      party: [meta[:party]].reject(&:blank?)
    }
    {
      legislative_term: legislative_term,
      full_reference: full_reference,
      reference: reference,
      doctype: doctype,
      title: extract_title(item),
      # url
      url: nil,
      published_at: meta[:published_at],
      originators: originators,
      is_answer: nil
    }
  end

  def self.extract_detail_paper(item)
    meta_text = extract_meta_text(item)
    type = extract_type(meta_text)
    extract_overview_paper(item, type)
  end

  def self.switch_to_term(legislative_term, m, top)
    top.frame_with(name: 'head').click
    nav = m.get('/redirect.aspx?wahlperiode=' + legislative_term.to_s)<commit_msg>Sachsen: fix extraction of party<commit_after>module SachsenScraper
  BASE_URL = 'http://edas.landtag.sachsen.de'
  SEARCH_FIELD = 'ctl00$masterContentCallback$content$suchmaske$tblSearch$tabSuche$panelUmSuchmaskeEinfach$suchmaskeEinfachCallback'
  SEARCH_FIELD2 =
    'ctl00_masterContentCallback_content_suchmaske_tblSearch_tabSuche_panelUmSuchmaskeEinfach_suchmaskeEinfachCallback_sb_Einf'

  def self.extract_overview_items(page)
    page.search('//td[@class="dxdvItem_EDAS"]/table')
  end

  def self.extract_title(item)
    item.search('.//tr[3]/td/b').text.strip
  end

  def self.extract_meta_text(item)
    text = item.search('.//tr[3]/td').text.split("\n").find { |t| t.include?('KlAnfr') || t.include?('GrAnfr') }
    text.strip if !text.nil?
  end

  def self.extract_meta_data(text)
    type = extract_type(text)
    if type == Paper::DOCTYPE_MAJOR_INTERPELLATION
      # empty braces for same result count
      m = text.match(/^GrAnfr ()(.+?) ([\d\.]+) Drs ([\d\/]+)$/)
    else
      m = text.match(/^KlAnfr (.+?) ([A-Z][a-zA-Z]{2}|[A-Z]{2,}[[:alnum:]\s]+) ([\d\.]+) Drs ([\d\/]+)$/)
    end
    return nil if m.nil?
    {
      person: m[1],
      party: m[2],
      published_at: Date.parse(m[3]),
      full_reference: m[4]
    }
  end

  def self.extract_type(text)
    if text[0..5] == 'KlAnfr'
      Paper::DOCTYPE_MINOR_INTERPELLATION
    else
      Paper::DOCTYPE_MAJOR_INTERPELLATION
    end
  end

  def self.extract_date(meta_text)
    Date.parse(meta_text.match(/\d+\.\d+\.\d{4}/).to_s)
  end

  def self.extract_overview_paper(item, doctype)
    meta_text = extract_meta_text(item)
    meta = extract_meta_data(meta_text)
    full_reference = meta[:full_reference]
    legislative_term, reference = full_reference.split('/')
    originators = {
      people: [meta[:person]].reject(&:blank?),
      parties: [meta[:party]].reject(&:blank?)
    }
    {
      legislative_term: legislative_term,
      full_reference: full_reference,
      reference: reference,
      doctype: doctype,
      title: extract_title(item),
      # url
      url: nil,
      published_at: meta[:published_at],
      originators: originators,
      is_answer: nil
    }
  end

  def self.extract_detail_paper(item)
    meta_text = extract_meta_text(item)
    type = extract_type(meta_text)
    extract_overview_paper(item, type)
  end

  def self.switch_to_term(legislative_term, m, top)
    top.frame_with(name: 'head').click
    nav = m.get('/redirect.aspx?wahlperiode=' + legislative_term.to_s)
{'fraction_non_alphanumeric': 0.14950832831627534, 'fraction_numerical': 0.006421834236403773, 'mean_word_length': 3.543299908842297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6936027', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1823, 'n_words': 273}