<commit_before>#!/usr/bin/env ruby

require 'nokogiri'
require 'builder'
require 'tempfile'

IMPORT_ROOT = File.expand_path("../..", __FILE__)

class Converter
  def self.from_file(path)
    doc = Nokogiri.XML(File.read(path))
    doc.remove_namespaces!

    new doc.first_element_child
  end

  def self.builder
    b = Builder::XmlMarkup.new :indent => 2
    b.instruct!

    b
  end

  def initialize(node)
    @doc = node
  end

  def to_xml
    finished_builder.target!
  end

  def finished_builder
    case @doc.name
    when 'partier_oversikt'
      convert_parties
    when 'dagensrepresentanter_oversikt'
      convert_representatives
    when 'komiteer_oversikt'
      convert_committees
    when 'emne_oversikt'
      convert_topics
    when 'fylker_oversikt'
      convert_districts
    when 'saker_oversikt'
      convert_issues
    else
      raise NotImplementedError, @doc.name
    end
  end

  private

  def convert_issues
    xml = create_builder

    xml.issues do |issues|
      @doc.css("saker_liste sak").each do |xi|
        issues.issue do |issue|
          issue.externalId xi.xpath("./id").first.text
          issue.summary xi.css("korttittel").first.text
          issue.description xi.css("tittel").first.text
          issue.type xi.css("type").first.text
          issue.status xi.css("status").first.text
          issue.lastUpdate xi.css("sist_oppdatert_dato").first.text
          issue.reference xi.css("henvisning").first.text
          issue.documentGroup xi.css("dokumentgruppe").first.text

          committee = xi.css("komite").first
          if committee && committee['nil'] != "true"
            issue.committee committee.css("navn").first.text
          end

          xtopics = xi.css("emne")
          if xtopics.any?
            issue.topics do |topics|
              xtopics.each do |xt|
                topics.topic xt.css("navn").first.text
              end
            end
          end

        end
      end
    end

    xml
  end

  def convert_districts
    xml = create_builder
    xml.districts do |districts|
      @doc.css("fylker_liste fylke").each do |xd|
        districts.district do |district|
          district.externalId xd.css("id").first.text
          district.name xd.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_committees
    xml = create_builder
    xml.committees do |committees|
      @doc.css("komiteer_liste komite").each do |xc|
        committees.committee do |committee|
          committee.externalId xc.css("id").first.text
          committee.name xc.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_topics
    xml = create_builder
    xml.topics { |topics|
      @doc.css("emne_liste > emne").each do |xt|
        topics.topic do |topic|
          add_topic(topic, xt)

          topic.subtopics do |subtopics|
            xt.css("underemne_liste > emne").each do |xst|
              subtopics.topic do |subtopic|
                add_topic(subtopic, xst)
              end
            end
          end
        end
      end
    }

    xml
  end

  def add_topic(builder, node)
    builder.externalId node.css("id").first.text
    builder.name node.css("navn").first.text
  end

  def convert_parties
    xml = create_builder
    xml.parties do |parties|
      @doc.css("partier_liste parti").each do |xp|
        parties.party do |party|
          party.externalId xp.css("id").first.text
          party.name xp.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_representatives
    xml = create_builder
    xml.representatives do |reps|
      @doc.css("dagensrepresentant").each do |xrep|
        RepresentativeBuilder.new(reps, xrep).build
      end
    end

    xml
  end

  def create_builder
    self.class.builder
  end
end

class RepresentativeBuilder
  def initialize(builder, node)
    @builder = builder
    @node = node
  end

  def build
    @builder.representative do |rep|
      rep.externalId  @node.css("id").first.text
      rep.firstName   @node.css("fornavn").first.text
      rep.lastName    @node.css("etternavn").first.text
      rep.district    @node.css("fylke navn").first.text
      rep.party       @node.css("parti navn").first.text
      rep.born        @node.css("foedselsdato").first.text

      rep.committees do |coms|
        @node.css("komite").each do |xcom|
          coms.committee xcom.css("navn").text
        end
      end
      rep.period("2011-2012") # FIXME

      yield rep if block_given?
    end
  end
end

class Importer

  FILES = {
    :parties         => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/partier/index.html?sesjonid=2011-2012"),
    :committees      => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/komiteer/index.html?SesjonId=2011-2012"),
    :districts       => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/fylker/index.html"),
    :representatives => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/dagensrepresentanter/index.html"),
    :topics          => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/emner/index.html"),
    :issues          => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/saker/index.html?sesjonid=2011-2012")
  }

  def self.execute(args)
    importer = new<commit_msg>dateOfBirth, dateOfDeath<commit_after>#!/usr/bin/env ruby

require 'nokogiri'
require 'builder'
require 'tempfile'

IMPORT_ROOT = File.expand_path("../..", __FILE__)

class Converter
  def self.from_file(path)
    doc = Nokogiri.XML(File.read(path))
    doc.remove_namespaces!

    new doc.first_element_child
  end

  def self.builder
    b = Builder::XmlMarkup.new :indent => 2
    b.instruct!

    b
  end

  def initialize(node)
    @doc = node
  end

  def to_xml
    finished_builder.target!
  end

  def finished_builder
    case @doc.name
    when 'partier_oversikt'
      convert_parties
    when 'dagensrepresentanter_oversikt'
      convert_representatives
    when 'komiteer_oversikt'
      convert_committees
    when 'emne_oversikt'
      convert_topics
    when 'fylker_oversikt'
      convert_districts
    when 'saker_oversikt'
      convert_issues
    else
      raise NotImplementedError, @doc.name
    end
  end

  private

  def convert_issues
    xml = create_builder

    xml.issues do |issues|
      @doc.css("saker_liste sak").each do |xi|
        issues.issue do |issue|
          issue.externalId xi.xpath("./id").first.text
          issue.summary xi.css("korttittel").first.text
          issue.description xi.css("tittel").first.text
          issue.type xi.css("type").first.text
          issue.status xi.css("status").first.text
          issue.lastUpdate xi.css("sist_oppdatert_dato").first.text
          issue.reference xi.css("henvisning").first.text
          issue.documentGroup xi.css("dokumentgruppe").first.text

          committee = xi.css("komite").first
          if committee && committee['nil'] != "true"
            issue.committee committee.css("navn").first.text
          end

          xtopics = xi.css("emne")
          if xtopics.any?
            issue.topics do |topics|
              xtopics.each do |xt|
                topics.topic xt.css("navn").first.text
              end
            end
          end

        end
      end
    end

    xml
  end

  def convert_districts
    xml = create_builder
    xml.districts do |districts|
      @doc.css("fylker_liste fylke").each do |xd|
        districts.district do |district|
          district.externalId xd.css("id").first.text
          district.name xd.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_committees
    xml = create_builder
    xml.committees do |committees|
      @doc.css("komiteer_liste komite").each do |xc|
        committees.committee do |committee|
          committee.externalId xc.css("id").first.text
          committee.name xc.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_topics
    xml = create_builder
    xml.topics { |topics|
      @doc.css("emne_liste > emne").each do |xt|
        topics.topic do |topic|
          add_topic(topic, xt)

          topic.subtopics do |subtopics|
            xt.css("underemne_liste > emne").each do |xst|
              subtopics.topic do |subtopic|
                add_topic(subtopic, xst)
              end
            end
          end
        end
      end
    }

    xml
  end

  def add_topic(builder, node)
    builder.externalId node.css("id").first.text
    builder.name node.css("navn").first.text
  end

  def convert_parties
    xml = create_builder
    xml.parties do |parties|
      @doc.css("partier_liste parti").each do |xp|
        parties.party do |party|
          party.externalId xp.css("id").first.text
          party.name xp.css("navn").first.text
        end
      end
    end

    xml
  end

  def convert_representatives
    xml = create_builder
    xml.representatives do |reps|
      @doc.css("dagensrepresentant").each do |xrep|
        RepresentativeBuilder.new(reps, xrep).build
      end
    end

    xml
  end

  def create_builder
    self.class.builder
  end
end

class RepresentativeBuilder
  def initialize(builder, node)
    @builder = builder
    @node = node
  end

  def build
    @builder.representative do |rep|
      rep.externalId   @node.css("id").first.text
      rep.firstName    @node.css("fornavn").first.text
      rep.lastName     @node.css("etternavn").first.text
      rep.district     @node.css("fylke navn").first.text
      rep.party        @node.css("parti navn").first.text
      rep.dateOfBirth  @node.css("foedselsdato").first.text
      rep.dateOfDeath  @node.css("doedsdato").first.text

      rep.committees do |coms|
        @node.css("komite").each do |xcom|
          coms.committee xcom.css("navn").text
        end
      end
      rep.period("2011-2012") # FIXME

      yield rep if block_given?
    end
  end
end

class Importer

  FILES = {
    :parties         => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/partier/index.html?sesjonid=2011-2012"),
    :committees      => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/komiteer/index.html?SesjonId=2011-2012"),
    :districts       => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/fylker/index.html"),
    :representatives => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/dagensrepresentanter/index.html"),
    :topics          => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/emner/index.html"),
    :issues          => File.join(IMPORT_ROOT, "folketingparser/rawdata/data.stortinget.no/eksport/saker/index.html?sesjonid=2011-2012")
  }

  def self.execute(args)
    importer = new
{'fraction_non_alphanumeric': 0.10907759882869693, 'fraction_numerical': 0.006039531478770132, 'mean_word_length': 2.329981718464351, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 38}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1490521', 'n_tokens_mistral': 3762, 'n_tokens_neox': 3601, 'n_words': 666}