<commit_before># == Schema Information
# Schema version: 20080705164959
#
# Table name: sources
#
#  id          :integer         not null, primary key
#  title       :string(255)     
#  url         :string(255)     
#  imported_at :datetime        
#  created_at  :datetime        
#  updated_at  :datetime        
#  reimport    :boolean         
#

# == Source
#
# A model that represents a source of events data, such as feeds for hCal, iCal, etc.
class Source < ActiveRecord::Base
  Tag # this class uses tagging. referencing the Tag class ensures that has_many_polymorphs initializes correctly across reloads.
  
  unless RAILS_ENV == 'test'
      acts_as_solr
  end
  validate :assert_url

  has_many :events
  has_many :updates

  # Create sources and events for the Array of +urls+. Returns a Hash of
  # Sources and the Events created.
  def self.create_sources_and_events_for!(*urls)
    sources2events = {}
    transaction do
      urls.flatten.each do |url|
        source = Source.find_or_create_by_url(:url => url)
        events = source.create_events!
        sources2events[source] = events
        source.save! # Updates the imported_at and other fields
      end
    end
    return sources2events
  end

  # Create events for this source. Returns the events created. URL must be set
  # for this source for this to work.
  def create_events!(opts={})
    cutoff = Time.now.yesterday # All events before this date will be skipped
    events = []
    transaction do
      for event in self.to_events(opts)
        if opts[:skip_old]
          next if event.title.blank? && event.description.blank? && event.url.blank?
          next if event.old?
        end
        
        # Skip invalid events that start after they end
        next if event.end_time && event.end_time < event.start_time
        
        # convert to local time, because time zone is simply discarded when event is saved
        event.start_time.localtime
        event.end_time.localtime if event.end_time
    
        # clear duplicate_of_id field in case to_events picked up orphaned duplicate
        # TODO clear the duplicate_of_id at the point where the object is created, not down here
        event.duplicate_of_id = nil if event.duplicate_of_id
        event.save!
        if event.venue
          event.venue.duplicate_of_id = nil if event.venue.duplicate_of_id
          event.venue.save! if event.venue
        end
        events << event
      end
      self.save!
    end
    return events
  end

  # Normalize the URL.
  def url=(value)
    begin
      url = URI.parse(value.strip)
      url.scheme = 'http' unless ['http','ftp'].include?(url.scheme) || url.scheme.nil?
      write_attribute(:url, url.scheme.nil? ? 'http://'+value.strip : url.to_s)
    rescue URI::InvalidURIError => e
      false
    end
  end

  # Returns an Array of Event objects that were read from this source.
  #
  # Options:
  # * :url -- URL of data to import. Defaults to record's #url attribute.
  # * :skip_old -- Should old events be skipped? Default is true.
  def to_events(opts={})
    self.imported_at = Time.now
    if valid?
      opts[:url] ||= self.url
      returning([]) do |events|
        SourceParser.to_abstract_events(opts).each do |abstract_event|
          event = Event.from_abstract_event(abstract_event, self)

          events << event
        end
      end<commit_msg>Fixed Source#url=, added "https" as legitimate URI scheme.<commit_after># == Schema Information
# Schema version: 20080705164959
#
# Table name: sources
#
#  id          :integer         not null, primary key
#  title       :string(255)     
#  url         :string(255)     
#  imported_at :datetime        
#  created_at  :datetime        
#  updated_at  :datetime        
#  reimport    :boolean         
#

# == Source
#
# A model that represents a source of events data, such as feeds for hCal, iCal, etc.
class Source < ActiveRecord::Base
  Tag # this class uses tagging. referencing the Tag class ensures that has_many_polymorphs initializes correctly across reloads.
  
  unless RAILS_ENV == 'test'
      acts_as_solr
  end
  validate :assert_url

  has_many :events
  has_many :updates

  # Create sources and events for the Array of +urls+. Returns a Hash of
  # Sources and the Events created.
  def self.create_sources_and_events_for!(*urls)
    sources2events = {}
    transaction do
      urls.flatten.each do |url|
        source = Source.find_or_create_by_url(:url => url)
        events = source.create_events!
        sources2events[source] = events
        source.save! # Updates the imported_at and other fields
      end
    end
    return sources2events
  end

  # Create events for this source. Returns the events created. URL must be set
  # for this source for this to work.
  def create_events!(opts={})
    cutoff = Time.now.yesterday # All events before this date will be skipped
    events = []
    transaction do
      for event in self.to_events(opts)
        if opts[:skip_old]
          next if event.title.blank? && event.description.blank? && event.url.blank?
          next if event.old?
        end
        
        # Skip invalid events that start after they end
        next if event.end_time && event.end_time < event.start_time
        
        # convert to local time, because time zone is simply discarded when event is saved
        event.start_time.localtime
        event.end_time.localtime if event.end_time
    
        # clear duplicate_of_id field in case to_events picked up orphaned duplicate
        # TODO clear the duplicate_of_id at the point where the object is created, not down here
        event.duplicate_of_id = nil if event.duplicate_of_id
        event.save!
        if event.venue
          event.venue.duplicate_of_id = nil if event.venue.duplicate_of_id
          event.venue.save! if event.venue
        end
        events << event
      end
      self.save!
    end
    return events
  end

  # Normalize the URL.
  def url=(value)
    begin
      url = URI.parse(value.strip)
      url.scheme = 'http' unless ['http','https','ftp'].include?(url.scheme) || url.scheme.nil?
      write_attribute(:url, url.scheme.nil? ? 'http://'+value.strip : url.to_s)
    rescue URI::InvalidURIError => e
      false
    end
  end

  # Returns an Array of Event objects that were read from this source.
  #
  # Options:
  # * :url -- URL of data to import. Defaults to record's #url attribute.
  # * :skip_old -- Should old events be skipped? Default is true.
  def to_events(opts={})
    self.imported_at = Time.now
    if valid?
      opts[:url] ||= self.url
      returning([]) do |events|
        SourceParser.to_abstract_events(opts).each do |abstract_event|
          event = Event.from_abstract_event(abstract_event, self)

          events << event
        end
      end
{'fraction_non_alphanumeric': 0.08092230870906153, 'fraction_numerical': 0.006755764429431634, 'mean_word_length': 2.4550989345509895, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4969708', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1983, 'n_words': 656}