<commit_before>#!/usr/bin/env ruby

require 'bundler/setup'
require 'nokogiri'
require 'pry'
require 'open-uri'
#require 'sinatra'
#require 'sinatra/activerecord'
#require './environments'
require 'active_record'
require 'rgeo'


ActiveRecord::Base.establish_connection(
    :adapter  => 'postgis',
    :host     => 'localhost',
    :database => 'pdxcrimes',
    :encoding => 'utf8'
  )

class Entry < ActiveRecord::Base
end

def main
  feed = Nokogiri::XML(open("http://www.portlandonline.com/scripts/911incidents.cfm"))
  feed.remove_namespaces!


  feed.xpath('/feed/entry').each do |e|
     id = e.xpath('id')
     lat,lng = e.xpath('point').inner_text.split
     updated=Time.parse(e.xpath('updated').inner_text)
     published=Time.parse(e.xpath('published').inner_text)
     h=Nokogiri::HTML(e.xpath('content').inner_text)
     attrs={}
     h.search('dt').each do |node|
       key=node.text.sub(/:$/,'').sub(/\s+/,'_').downcase.to_sym
       attrs[key]=node.next_element.text
     end
     ent = Entry.find_by(entry_id: attrs[:id])
     is_new = ent.nil?
     ent = Entry.find_or_create_by(entry_id: attrs[:id])
     ent.call_type ||= attrs[:call_type]
     ent.latitude ||= lat
     ent.longitude ||= lng
     ent.updated ||= updated
     ent.published ||= published
     ent.agency ||= attrs[:agency]
     ent.address ||= attrs[:address]
     ent.save!
#     if is_new
#       puts "added: #{ent.inspect}" <commit_msg>always store the latest data.<commit_after>#!/usr/bin/env ruby

require 'bundler/setup'
require 'nokogiri'
require 'pry'
require 'open-uri'
#require 'sinatra'
#require 'sinatra/activerecord'
#require './environments'
require 'active_record'
require 'rgeo'


ActiveRecord::Base.establish_connection(
    :adapter  => 'postgis',
    :host     => 'localhost',
    :database => 'pdxcrimes',
    :encoding => 'utf8'
  )

class Entry < ActiveRecord::Base
end

def main
  feed = Nokogiri::XML(open("http://www.portlandonline.com/scripts/911incidents.cfm"))
  feed.remove_namespaces!


  feed.xpath('/feed/entry').each do |e|
     id = e.xpath('id')
     lat,lng = e.xpath('point').inner_text.split
     updated=Time.parse(e.xpath('updated').inner_text)
     published=Time.parse(e.xpath('published').inner_text)
     h=Nokogiri::HTML(e.xpath('content').inner_text)
     attrs={}
     h.search('dt').each do |node|
       key=node.text.sub(/:$/,'').sub(/\s+/,'_').downcase.to_sym
       attrs[key]=node.next_element.text
     end
     ent = Entry.find_by(entry_id: attrs[:id])
     is_new = ent.nil?
     ent = Entry.find_or_create_by(entry_id: attrs[:id])
     ent.call_type = attrs[:call_type]
     ent.latitude = lat
     ent.longitude = lng
     ent.updated = updated
     ent.published = published
     ent.agency = attrs[:agency]
     ent.address = attrs[:address]
     ent.save!
#     if is_new
#       puts "added: #{ent.inspect}" 
{'fraction_non_alphanumeric': 0.16818181818181818, 'fraction_numerical': 0.002797202797202797, 'mean_word_length': 3.456386292834891, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702053', 'n_tokens_mistral': 1045, 'n_tokens_neox': 1028, 'n_words': 170}