<commit_before>require 'nokogiri'

module ApplicationHelper
  def self.generate_events(event_data_source)
    if event_data_source.nil?
      EventDataSource.find_each do |eds|
        download_page(eds)
      end
    end
    #check if each event is duplicate
    #return onlyuniqueevents
  end

  def self.download_page(event_data_source)
    puts "Checking Source " + event_data_source.name
    page_url = event_data_source.url
    page = Nokogiri::HTML(open(page_url))
    item_selector = event_data_source.item_selector
    name_selector = event_data_source.name_selector
    page.css(item_selector).each do |item|
      e = Event.new
      e.name = item.css(name_selector)[0].text.strip
      e.url = item.css(name_selector)[0]["href"].strip
      old = Event.find_by_url(e.url)
      if old == nil then
        puts "Saved event " + e.name
        e.save!
      end
    end
  end
end<commit_msg>add support for relative urls<commit_after>require 'nokogiri'

module ApplicationHelper
  def self.generate_events(event_data_source)
    if event_data_source.nil?
      EventDataSource.find_each do |eds|
        download_page(eds)
      end
    end
    #check if each event is duplicate
    #return onlyuniqueevents
  end

  def self.download_page(event_data_source)
    puts "Checking Source " + event_data_source.name
    page_url = event_data_source.url
    page = Nokogiri::HTML(open(page_url))
    item_selector = event_data_source.item_selector
    name_selector = event_data_source.name_selector
    page.css(item_selector).each do |item|
      e = Event.new
      e.name = item.css(name_selector)[0].text.strip
      e.url = item.css(name_selector)[0]["href"].strip
      if !e.url.starts_with?("http") && !e.url.starts_with?("https") then
        e.url = page_url + e.url
      end
      old = Event.find_by_url(e.url)
      if old == nil then
        puts "Saved event " + e.name
        e.save!
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.09302325581395349, 'fraction_numerical': 0.0020671834625322996, 'mean_word_length': 3.0, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5428751', 'n_tokens_mistral': 645, 'n_tokens_neox': 646, 'n_words': 133}