<commit_before>
task :import_constructions => :environment do
  urls = ['http://frankfurt-gestalten.s3.amazonaws.com/traffic/baustellen.xml']
  urls.each do |url|
    uri = URI(url)
    request = Net::HTTP::Get.new(uri)
    response = Net::HTTP.start(uri.hostname, uri.port) do |conn|
      conn.request(request)
    end

    doc = Nokogiri::XML(response.body)
    doc.search('Ereignis').each do |item|
      Construction.create_from_xml(item, "baustelle")
      puts item
    end
  end<commit_msg>utf-8 encoding<commit_after>#encoding: utf-8

task :import_constructions => :environment do
  urls = ['http://frankfurt-gestalten.s3.amazonaws.com/traffic/baustellen.xml']
  urls.each do |url|
    uri = URI(url)
    request = Net::HTTP::Get.new(uri)
    response = Net::HTTP.start(uri.hostname, uri.port) do |conn|
      conn.request(request)
    end

    doc = Nokogiri::XML(response.body)
    doc.search('Ereignis').each do |item|
      Construction.create_from_xml(item, "baustelle")
      puts item
    end
  end
{'fraction_non_alphanumeric': 0.14201183431952663, 'fraction_numerical': 0.0039447731755424065, 'mean_word_length': 3.53125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2997232', 'n_tokens_mistral': 361, 'n_tokens_neox': 362, 'n_words': 65}