<commit_before># == Source::Parser::Ical
#
# Reads iCalendar events.
#
# Example:
#   events = Source::Parser::Ical.to_events('http://appendix.23ae.com/calendars/AlternateHolidays.ics')
#
# Sample sources:
#   webcal://appendix.23ae.com/calendars/AlternateHolidays.ics
#   http://appendix.23ae.com/calendars/AlternateHolidays.ics
module Calagator

class Source::Parser::Ical < Source::Parser
  self.label = :iCalendar

  VENUE_CONTENT_RE = /^BEGIN:VVENUE$.*?^END:VVENUE$/m

  # Override Source::Parser.read_url to handle "webcal" scheme addresses.
  def self.read_url(url)
    url.gsub!(/^webcal:/, 'http:')
    super
  end

  def to_events
    return false unless calendars
    events = calendars.flat_map(&:events).each do |event|
      event.source = source
    end
    dedup(events)
  end

  private

  def calendars
    @calendars ||= RiCal.parse_string(content).map do |calendar|
      VCalendar.new(calendar)
    end
  rescue Exception => exception
    return false if exception.message =~ /Invalid icalendar file/ # Invalid data, give up.
    raise # Unknown error, reraise
  end

  def content
    self.class.read_url(url).tap do |content|
      content.gsub! /\r\n/, "\n" # normalize line endings
      content.gsub! /;TZID=GMT:(.*)/, ':\1Z' # normalize timezones
    end
  end

  def dedup(events)
    events.map do |event|
      event = event_or_duplicate(event)
      event.venue = venue_or_duplicate(event.venue) if event.venue
      event
    end.uniq
  end

  class VCalendar < Struct.new(:calendar)
    def events
      calendar.events.map do |component|
        VEvent.new(component, venues)
      end.reject(&:old?).map(&:to_event)
    end

    def venues
      calendar.to_s.scan(VENUE_CONTENT_RE).map do |venue_content|
        VVenue.new(venue_content)
      end
    end
  end

  class VEvent < Struct.new(:component, :venues)
    def old?
      cutoff = Time.now.yesterday
      (component.dtend || component.dtstart).to_time < cutoff
    end

    def to_event
      event = EventParser.new(component).to_event
      event.venue = VenueParser.new(vvenue, component.location).to_venue
      event
    end

    private

    def venue_uid
      component.location_property.try(:params).try(:[], "VVENUE")
    end

    def vvenue
      venues.find { |venue| venue.uid == venue_uid } if venue_uid
    end
  end

  class VVenue < Struct.new(:content)
    def uid
      content.match(/^UID:(?<uid>.+)$/)[:uid]
    end

    def vcard_hash
      return unless data = content.scan(VENUE_CONTENT_RE).first

      # Only use first vcard of a VVENUE
      vcard = RiCal.parse_string(data).first

      # Extract all properties, including non-standard ones, into an array of "KEY;meta-qualifier:value" strings
      vcard_lines = vcard.export_properties_to(StringIO.new(''))

      # Turn a String-like object into an Enumerable.
      vcard_lines = vcard_lines.respond_to?(:lines) ? vcard_lines.lines : vcard_lines

      hash_from_vcard_lines(vcard_lines)
    end

    private

    VCARD_LINES_RE = /^(?<key>[^;]+?)(?<qualifier>;[^:]*?)?:(?<value>.*)$/

    # Return hash parsed from VCARD lines.
    #
    # Arguments:
    # * vcard_lines - Array of "KEY;meta-qualifier:value" strings.
    def hash_from_vcard_lines(vcard_lines)
      vcard_lines.reduce({}) do |vcard_hash, vcard_line|
        vcard_line.match(VCARD_LINES_RE) do |match|
          vcard_hash[match[:key]] ||= match[:value]
        end
        vcard_hash
      end
    end
  end

  class EventParser < Struct.new(:component)
    def to_event
      Event.new({
        title:       component.summary,
        description: component.description,
        url:         component.url,
        start_time:  normalized_start_time(component),
        end_time:    normalized_end_time(component),
      })
    end
<commit_msg>now only VCalendar needs to know about this regex.<commit_after># == Source::Parser::Ical
#
# Reads iCalendar events.
#
# Example:
#   events = Source::Parser::Ical.to_events('http://appendix.23ae.com/calendars/AlternateHolidays.ics')
#
# Sample sources:
#   webcal://appendix.23ae.com/calendars/AlternateHolidays.ics
#   http://appendix.23ae.com/calendars/AlternateHolidays.ics
module Calagator

class Source::Parser::Ical < Source::Parser
  self.label = :iCalendar

  # Override Source::Parser.read_url to handle "webcal" scheme addresses.
  def self.read_url(url)
    url.gsub!(/^webcal:/, 'http:')
    super
  end

  def to_events
    return false unless calendars
    events = calendars.flat_map(&:events).each do |event|
      event.source = source
    end
    dedup(events)
  end

  private

  def calendars
    @calendars ||= RiCal.parse_string(content).map do |calendar|
      VCalendar.new(calendar)
    end
  rescue Exception => exception
    return false if exception.message =~ /Invalid icalendar file/ # Invalid data, give up.
    raise # Unknown error, reraise
  end

  def content
    self.class.read_url(url).tap do |content|
      content.gsub! /\r\n/, "\n" # normalize line endings
      content.gsub! /;TZID=GMT:(.*)/, ':\1Z' # normalize timezones
    end
  end

  def dedup(events)
    events.map do |event|
      event = event_or_duplicate(event)
      event.venue = venue_or_duplicate(event.venue) if event.venue
      event
    end.uniq
  end

  class VCalendar < Struct.new(:calendar)
    VENUE_CONTENT_RE = /^BEGIN:VVENUE$.*?^END:VVENUE$/m

    def events
      calendar.events.map do |component|
        VEvent.new(component, venues)
      end.reject(&:old?).map(&:to_event)
    end

    def venues
      calendar.to_s.scan(VENUE_CONTENT_RE).map do |venue_content|
        VVenue.new(venue_content)
      end
    end
  end

  class VEvent < Struct.new(:component, :venues)
    def old?
      cutoff = Time.now.yesterday
      (component.dtend || component.dtstart).to_time < cutoff
    end

    def to_event
      event = EventParser.new(component).to_event
      event.venue = VenueParser.new(vvenue, component.location).to_venue
      event
    end

    private

    def venue_uid
      component.location_property.try(:params).try(:[], "VVENUE")
    end

    def vvenue
      venues.find { |venue| venue.uid == venue_uid } if venue_uid
    end
  end

  class VVenue < Struct.new(:content)
    def uid
      content.match(/^UID:(?<uid>.+)$/)[:uid]
    end

    def vcard_hash
      # Only use first vcard of a VVENUE
      vcard = RiCal.parse_string(content).first

      # Extract all properties into an array of "KEY;meta-qualifier:value" strings
      vcard_lines = vcard.export_properties_to(StringIO.new(''))

      hash_from_vcard_lines(vcard_lines)
    end

    private

    VCARD_LINES_RE = /^(?<key>[^;]+?)(?<qualifier>;[^:]*?)?:(?<value>.*)$/

    def hash_from_vcard_lines(vcard_lines)
      vcard_lines.reduce({}) do |vcard_hash, vcard_line|
        vcard_line.match(VCARD_LINES_RE) do |match|
          vcard_hash[match[:key]] ||= match[:value]
        end
        vcard_hash
      end
    end
  end

  class EventParser < Struct.new(:component)
    def to_event
      Event.new({
        title:       component.summary,
        description: component.description,
        url:         component.url,
        start_time:  normalized_start_time(component),
        end_time:    normalized_end_time(component),
      })
    end

{'fraction_non_alphanumeric': 0.11827068704392124, 'fraction_numerical': 0.0019275781357565745, 'mean_word_length': 2.8844919786096255, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5888741', 'n_tokens_mistral': 2535, 'n_tokens_neox': 2417, 'n_words': 506}