<commit_before># == Source::Parser::Ical
#
# Reads iCalendar events.
#
# Example:
#   events = Source::Parser::Ical.to_events('http://appendix.23ae.com/calendars/AlternateHolidays.ics')
#
# Sample sources:
#   webcal://appendix.23ae.com/calendars/AlternateHolidays.ics
#   http://appendix.23ae.com/calendars/AlternateHolidays.ics
module Calagator

class Source::Parser::Ical < Source::Parser
  self.label = :iCalendar

  # Override Source::Parser.read_url to handle "webcal" scheme addresses.
  def self.read_url(url)
    url.gsub!(/^webcal:/, 'http:')
    super
  end

  def to_events
    return false unless calendars
    events = calendars.flat_map(&:events).reject(&:old?).map(&:to_event).each do |event|
      event.source = source
    end
    dedup(events)
  end

  private

  def calendars
    @calendars ||= VCalendar.parse(content)
  end

  def content
    self.class.read_url(url).tap do |content|
      content.gsub! /\r\n/, "\n" # normalize line endings
      content.gsub! /;TZID=GMT:(.*)/, ':\1Z' # normalize timezones
    end
  end

  def dedup(events)
    events.map do |event|
      event = event_or_duplicate(event)
      event.venue = venue_or_duplicate(event.venue) if event.venue
      event
    end.uniq
  end

  class VCalendar < Struct.new(:calendar)
    def self.parse(content)
      RiCal.parse_string(content).map do |calendar|
        VCalendar.new(calendar)
      end
    rescue Exception => exception
      return false if exception.message =~ /Invalid icalendar file/ # Invalid data, give up.
      raise # Unknown error, reraise
    end

    def events
      calendar.events.map do |component|
        VEvent.new(component, venues)
      end
    end

    VENUE_CONTENT_RE = /^BEGIN:VVENUE$.*?^END:VVENUE$/m

    def venues
      calendar.to_s.scan(VENUE_CONTENT_RE).map do |venue_content|
        VVenue.new(venue_content)
      end
    end
  end

  class VEvent < Struct.new(:component, :venues)
    def old?
      cutoff = Time.now.yesterday
      (component.dtend || component.dtstart).to_time < cutoff
    end

    def to_event
      event = EventParser.new(self).to_event
      event.venue = VenueParser.new(vvenue, component.location).to_venue
      event
    end

    private

    def venue_uid
      component.location_property.try(:params).try(:[], "VVENUE")
    end

    def vvenue
      venues.find { |venue| venue.uid == venue_uid } if venue_uid
    end<commit_msg>rename variables to distinguish between e.g. vevents and events.<commit_after># == Source::Parser::Ical
#
# Reads iCalendar events.
#
# Example:
#   events = Source::Parser::Ical.to_events('http://appendix.23ae.com/calendars/AlternateHolidays.ics')
#
# Sample sources:
#   webcal://appendix.23ae.com/calendars/AlternateHolidays.ics
#   http://appendix.23ae.com/calendars/AlternateHolidays.ics
module Calagator

class Source::Parser::Ical < Source::Parser
  self.label = :iCalendar

  # Override Source::Parser.read_url to handle "webcal" scheme addresses.
  def self.read_url(url)
    url.gsub!(/^webcal:/, 'http:')
    super
  end

  def to_events
    return false unless vcalendars
    events = vcalendars.flat_map(&:vevents).reject(&:old?).map(&:to_event).each do |event|
      event.source = source
    end
    dedup(events)
  end

  private

  def vcalendars
    @vcalendars ||= VCalendar.parse(content)
  end

  def content
    self.class.read_url(url).tap do |content|
      content.gsub! /\r\n/, "\n" # normalize line endings
      content.gsub! /;TZID=GMT:(.*)/, ':\1Z' # normalize timezones
    end
  end

  def dedup(events)
    events.map do |event|
      event = event_or_duplicate(event)
      event.venue = venue_or_duplicate(event.venue) if event.venue
      event
    end.uniq
  end

  class VCalendar < Struct.new(:calendar)
    def self.parse(content)
      RiCal.parse_string(content).map do |calendar|
        VCalendar.new(calendar)
      end
    rescue Exception => exception
      return false if exception.message =~ /Invalid icalendar file/ # Invalid data, give up.
      raise # Unknown error, reraise
    end

    def vevents
      calendar.events.map do |component|
        VEvent.new(component, vvenues)
      end
    end

    VENUE_CONTENT_RE = /^BEGIN:VVENUE$.*?^END:VVENUE$/m

    def vvenues
      calendar.to_s.scan(VENUE_CONTENT_RE).map do |venue_content|
        VVenue.new(venue_content)
      end
    end
  end

  class VEvent < Struct.new(:component, :venues)
    def old?
      cutoff = Time.now.yesterday
      (component.dtend || component.dtstart).to_time < cutoff
    end

    def to_event
      event = EventParser.new(self).to_event
      event.venue = VenueParser.new(vvenue, component.location).to_venue
      event
    end

    private

    def venue_uid
      component.location_property.try(:params).try(:[], "VVENUE")
    end

    def vvenue
      venues.find { |venue| venue.uid == venue_uid } if venue_uid
    end
{'fraction_non_alphanumeric': 0.11625999589911831, 'fraction_numerical': 0.002870617182694279, 'mean_word_length': 2.9466019417475726, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5439251', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1627, 'n_words': 347}