<commit_before># coding: UTF-8

require File.dirname(__FILE__) + "/common"
require "libxml"
require 'uri'
require 'logger'
require 'open-uri'

include LibXML

$parties = $cartodb.query("select cartodb_id, name, siglas from #{POLITICAL_PARTIES}")[:rows].inject({}){ |h, row| h[row[:cartodb_id]] = row[:siglas]; h}
$rparties = $parties.invert

class MunicipalityParticipation
  include XML::SaxParser::Callbacks
  
  def on_start_document
    @result = {}
    @votantes_totales = nil
    @votos_en_blanco = nil
    @votos_nulos = nil
    @abstencion = nil
  end
  
  def on_start_element(element, attributes)
    case element
      when 'NULOS_ABSOLUTO'
        @in_element = element
      when 'EN_BLANCO_ABSOLUTO'
        @in_element = element
      when 'PARTICIPACION_ABSOLUTO'
        @in_element = element
      when 'ABSTENCION_ABSOLUTO'
        @in_element = element
    end
  end
  
  def on_end_element(element)
    @in_element = nil
  end
  
  def on_characters(chars)
    chars = chars.force_encoding('UTF-8').encode
    case @in_element
      when 'NULOS_ABSOLUTO'
        @votos_nulos = chars.to_i
      when 'EN_BLANCO_ABSOLUTO'
        @votos_en_blanco = chars.to_i
      when 'PARTICIPACION_ABSOLUTO'
        @votantes_totales = chars.to_i
      when 'ABSTENCION_ABSOLUTO'
        @abstencion = chars.to_i
    end
  end
  
  def on_end_document
    @final_result = {
      :mesas_electorales => nil, :censo_total=>(@votos_nulos + @votos_en_blanco + @votantes_totales + @abstencion), :votantes_totales=>@votantes_totales, :votos_validos=>(@votantes_totales - @votos_en_blanco - @votos_nulos), :votos_en_blanco=>@votos_en_blanco, :votos_nulos=>@votos_nulos, 
    }
  end
  
  def result
    @final_result
  end
  
end

class MunicipalityVotes
  include XML::SaxParser::Callbacks
  
  def initialize(municipio_id, codinemuni, codineprov)
    @municipio_id = municipio_id
    @codinemuni = codinemuni
    @codineprov = codineprov
  end
  
  def on_start_document
    @in_party = false
    @party_name = nil
    @result = {}
    @name = false
  end
  
  def on_start_element(element, attributes)
    if element == 'ZONA'
      @municipality_name = nil
    elsif element == 'PARTIDO'
      @in_party = true
      @party_name = nil
    elsif element == 'ID'
      @almost_party_id = true
    elsif element == 'SIGLAS'
      @almost_party_name = true
    elsif element == 'PORCENTAJE'
      @almost_percentage = true
    elsif element == 'VOTOS'
      @almost_votes = true
    elsif element == 'NOMBRE'
      unless @in_party
        @name = true
      end
    end    
  end
  
  def on_end_element(element)
    case element
      when 'PARTIDO'
        @in_party = false
      when'PORCENTAJE'
        @almost_percentage = false
      when 'VOTOS'
        @almost_votes = false
      when 'SIGLAS'
        @almost_party_name = false
      when 'NOMBRE'
        @name = false
      when 'ID'
        @almost_party_id = false
    end
  end
  
  def on_cdata_block(cdata)
    cdata = cdata.force_encoding('UTF-8').encode
    if @name
      @municipality_name = cdata
      @result["Partidos"] ||= {}
    elsif @almost_party_name
      @party_name = cdata
      @result["Partidos"][@party_name] ||= {"Votos" => nil, "Porcentaje" => nil, "ID" => @party_id}
      @party_id = nil
    end
  end
  
  def on_characters(chars)
    chars = chars.force_encoding('UTF-8').encode
    if @almost_percentage
      @result["Partidos"][@party_name]["Porcentaje"] = chars.strip.to_f
    elsif @almost_votes
      @result["Partidos"][@party_name]["Votos"] = chars.strip.to_f
    elsif @almost_party_id
      @party_id = chars.strip
    end
  end
  
  def on_end_document
    array = @result['Partidos'].map do |party_name,values|
      [party_name, values['Votos'],values['Porcentaje'], values['ID']]
    end
    # Insert all non existing parties
    array.each do |p|
      if $rparties[p[3]].nil?
        # $cartodb.query("INSERT INTO partidos_politicos (name) VALUES ('#{p[0].gsub(/\'/,"\\\'")}')").rows
        puts "Insertando #{p[0]} en partidos_politicos" 
        $parties = $cartodb.query("select cartodb_id, name, siglas from #{POLITICAL_PARTIES}")[:rows].inject({}){ |h, row| h[row[:cartodb_id]] = row[:siglas]; h}
        $rparties = $parties.invert
      end
    end
    
    @final_result = {
      :municipio_id => @municipio_id, :proceso_electoral_id => 76, 
      :mesas_electorales => nil, :censo_total=>nil, :votantes_totales=>nil, :votos_validos=>nil, :votos_en_blanco=>nil, :votos_nulos=>nil, 
      :primer_partido_id=>$rparties[array[0][0]], :segundo_partido_id=>$rparties[array[1][0]], :tercer_partido_id=>$rparties[array[2][0]], 
      :resto_partido_votos=>array[3..-1].inject(0){|sum,e| sum+=e[1]; sum}, :resto_partido_percent=>(100 - array[0][2].to_i - array[1][2].to_i - array[2][2].to_i), 
      :primer_partido_votos=>array[0][1], :primer_partido_percent=>array[0][2], 
      :segundo_partido_votos=>array[1][1], :segundo_partido_percent=>array[1][2], :tercer_partido_votos=>array[2][1], :tercer_partido_percent=>array[0][2], 
      :codinemuni=>@codinemuni, :codineprov=>@codineprov 
    }
  end
  
  def result
    @final_result
  end  
end
<commit_msg>Bugfixing<commit_after># coding: UTF-8

require File.dirname(__FILE__) + "/common"
require "libxml"
require 'uri'
require 'logger'
require 'open-uri'

include LibXML

rows = $cartodb.query("select cartodb_id, name, siglas from #{POLITICAL_PARTIES}")[:rows]
$parties_ids = rows.inject({}){ |h, row| h[row[:siglas]] = row[:cartodb_id]; h}
$parties = rows.inject({}){ |h, row| h[row[:cartodb_id]] = row[:name]; h}
$rparties = $parties.invert

class MunicipalityParticipation
  include XML::SaxParser::Callbacks
  
  def on_start_document
    @result = {}
    @votantes_totales = nil
    @votos_en_blanco = nil
    @votos_nulos = nil
    @abstencion = nil
  end
  
  def on_start_element(element, attributes)
    case element
      when 'NULOS_ABSOLUTO'
        @in_element = element
      when 'EN_BLANCO_ABSOLUTO'
        @in_element = element
      when 'PARTICIPACION_ABSOLUTO'
        @in_element = element
      when 'ABSTENCION_ABSOLUTO'
        @in_element = element
    end
  end
  
  def on_end_element(element)
    @in_element = nil
  end
  
  def on_characters(chars)
    chars = chars.force_encoding('UTF-8').encode
    case @in_element
      when 'NULOS_ABSOLUTO'
        @votos_nulos = chars.to_i
      when 'EN_BLANCO_ABSOLUTO'
        @votos_en_blanco = chars.to_i
      when 'PARTICIPACION_ABSOLUTO'
        @votantes_totales = chars.to_i
      when 'ABSTENCION_ABSOLUTO'
        @abstencion = chars.to_i
    end
  end
  
  def on_end_document
    @final_result = {
      :mesas_electorales => nil, :censo_total=>(@votos_nulos + @votos_en_blanco + @votantes_totales + @abstencion), :votantes_totales=>@votantes_totales, :votos_validos=>(@votantes_totales - @votos_en_blanco - @votos_nulos), :votos_en_blanco=>@votos_en_blanco, :votos_nulos=>@votos_nulos, 
    }
  end
  
  def result
    @final_result
  end
  
end

class MunicipalityVotes
  include XML::SaxParser::Callbacks
  
  def initialize(municipio_id, codinemuni, codineprov)
    @municipio_id = municipio_id
    @codinemuni = codinemuni
    @codineprov = codineprov
  end
  
  def on_start_document
    @in_party = false
    @party_name = nil
    @result = {}
    @name = false
  end
  
  def on_start_element(element, attributes)
    if element == 'ZONA'
      @municipality_name = nil
    elsif element == 'PARTIDO'
      @in_party = true
      @party_name = nil
    elsif element == 'ID'
      @almost_party_id = true
    elsif element == 'SIGLAS'
      @almost_party_name = true
    elsif element == 'PORCENTAJE'
      @almost_percentage = true
    elsif element == 'VOTOS'
      @almost_votes = true
    elsif element == 'NOMBRE'
      unless @in_party
        @name = true
      end
    end    
  end
  
  def on_end_element(element)
    case element
      when 'PARTIDO'
        @in_party = false
      when'PORCENTAJE'
        @almost_percentage = false
      when 'VOTOS'
        @almost_votes = false
      when 'SIGLAS'
        @almost_party_name = false
      when 'NOMBRE'
        @name = false
      when 'ID'
        @almost_party_id = false
    end
  end
  
  def on_cdata_block(cdata)
    cdata = cdata.force_encoding('UTF-8').encode
    if @name
      @municipality_name = cdata
      @result["Partidos"] ||= {}
    elsif @almost_party_name
      @party_name = cdata
      @result["Partidos"][@party_name] ||= {"Votos" => nil, "Porcentaje" => nil, "ID" => @party_id}
      @party_id = nil
    end
  end
  
  def on_characters(chars)
    chars = chars.force_encoding('UTF-8').encode
    if @almost_percentage
      @result["Partidos"][@party_name]["Porcentaje"] = chars.strip.to_f
    elsif @almost_votes
      @result["Partidos"][@party_name]["Votos"] = chars.strip.to_f
    elsif @almost_party_id
      @party_id = chars.strip
    end
  end
  
  def on_end_document
    array = @result['Partidos'].map do |party_name,values|
      [party_name, values['Votos'],values['Porcentaje'], values['ID']]
    end
    # Insert all non existing parties
    array.each do |p|
      if $rparties[p[0]].nil?
        if $parties_ids[p[3]].nil?
          # $cartodb.query("INSERT INTO partidos_politicos (name) VALUES ('#{p[0].gsub(/\'/,"\\\'")}')").rows
          puts "Insertando #{p[0]} en partidos_politicos" 
          rows = $cartodb.query("select cartodb_id, name, siglas from #{POLITICAL_PARTIES}")[:rows]
          $parties_ids = rows.inject({}){ |h, row| h[row[:siglas]] = row[:cartodb_id]; h}
          $parties = rows.inject({}){ |h, row| h[row[:cartodb_id]] = row[:name]; h}
          $rparties = $parties.invert
        end
      end
    end
        
    @final_result = {
      :municipio_id => @municipio_id, :proceso_electoral_id => 76, 
      :mesas_electorales => nil, :censo_total=>nil, :votantes_totales=>nil, :votos_validos=>nil, :votos_en_blanco=>nil, :votos_nulos=>nil, 
      :primer_partido_id=>$rparties[array[0][0]], :segundo_partido_id=>$rparties[array[1][0]], :tercer_partido_id=>$rparties[array[2][0]], 
      :resto_partido_votos=>array[3..-1].inject(0){|sum,e| sum+=e[1]; sum}, :resto_partido_percent=>(100 - array[0][2].to_i - array[1][2].to_i - array[2][2].to_i), 
      :primer_partido_votos=>array[0][1], :primer_partido_percent=>array[0][2], 
      :segundo_partido_votos=>array[1][1], :segundo_partido_percent=>array[1][2], :tercer_partido_votos=>array[2][1], :tercer_partido_percent=>array[0][2], 
      :codinemuni=>@codinemuni, :codineprov=>@codineprov 
    }
  end
  
  def result
    @final_result
  end  
end

{'fraction_non_alphanumeric': 0.13173031588873174, 'fraction_numerical': 0.007637906647807638, 'mean_word_length': 3.2509018036072144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '809093', 'n_tokens_mistral': 4129, 'n_tokens_neox': 4049, 'n_words': 707}