<commit_before>class Feed < ActiveRecord::Base
  belongs_to :person
  scope_by_site_id
  validates_presence_of :person_id, :url, :name
  validates_uniqueness_of :name, :scope => :person_id
  validates_format_of :url, :with => /^https?\:\/\/.+/, :message => "has an incorrect format (are you missing 'http://' at the beginning?)"
  
  before_save :transform_url
  
  def transform_url
    self.url = self.class.transform_url(url)
  end
  
  IMPORT_LIMIT = 5
  
  after_create :import_content
  
  def import_content
    begin
      feed = Feedzirra::Feed.fetch_and_parse(url)
    rescue
      increment!(:error_count)
    else
      if feed
        feed.entries[0...IMPORT_LIMIT].reverse.each do |entry|
          if url.include?('flickr.com')
            import_picture(entry)
          else
            import_note(entry)
          end
        end
      else
        increment!(:error_count)
      end
    end
  end
  
  def import_note(entry)
    unless Note.exists?(['original_url = ?', entry.url])
      body = entry.content || entry.summary
      if url.include?('twitter.com') and url =~ /screen_name=(.+)/
        username = $1
        body.sub!(/^#{username}:\s/, '')
      end
      person.notes.create(
        :title        => url.include?('facebook.com') || url.include?('twitter.com') ? nil : entry.title,
        :body         => body,
        :created_at   => entry.published,
        :original_url => entry.url
      )
    end
  end
  
  def import_picture(entry)
    unless Picture.exists?(['original_url = ?', entry.url])
      album = person.albums.find_or_create_by_name('Flickr') do |a|
        a.description = 'Photos from my Flickr account.'
        a.is_public = false
      end
      picture = person.pictures.create(
        :album_id     => album.id,
        :original_url => entry.url,
        :created_at   => entry.published
      )
      if entry.content =~ /<img src="([^"]+_m\.jpg)/
        picture.photo = $1.sub(/_m\.jpg$/, '_b.jpg')
        unless picture.has_photo?
          picture.destroy
        end
      else
        picture.destroy
      end
    end
  end
  
  def self.import_all
    Feed.all.each do |feed|
      feed.import_content
    end
  end
  
  def self.transform_url(url)
    url = url.to_s
    if url.include?('facebook.com')<commit_msg>Allow duplicate note/pic alternate_urls per person.<commit_after>class Feed < ActiveRecord::Base
  belongs_to :person
  scope_by_site_id
  validates_presence_of :person_id, :url, :name
  validates_uniqueness_of :name, :scope => :person_id
  validates_format_of :url, :with => /^https?\:\/\/.+/, :message => "has an incorrect format (are you missing 'http://' at the beginning?)"
  
  before_save :transform_url
  
  def transform_url
    self.url = self.class.transform_url(url)
  end
  
  IMPORT_LIMIT = 5
  
  after_create :import_content
  
  def import_content
    begin
      feed = Feedzirra::Feed.fetch_and_parse(url)
    rescue
      increment!(:error_count)
    else
      if feed
        feed.entries[0...IMPORT_LIMIT].reverse.each do |entry|
          if url.include?('flickr.com')
            import_picture(entry)
          else
            import_note(entry)
          end
        end
      else
        increment!(:error_count)
      end
    end
  end
  
  def import_note(entry)
    unless Note.exists?(['original_url = ? and person_id = ?', entry.url, person_id])
      body = entry.content || entry.summary
      if url.include?('twitter.com') and url =~ /screen_name=(.+)/
        username = $1
        body.sub!(/^#{username}:\s/, '')
      end
      person.notes.create(
        :title        => url.include?('facebook.com') || url.include?('twitter.com') ? nil : entry.title,
        :body         => body,
        :created_at   => entry.published,
        :original_url => entry.url
      )
    end
  end
  
  def import_picture(entry)
    unless Picture.exists?(['original_url = ? and person_id = ?', entry.url, person_id])
      album = person.albums.find_or_create_by_name('Flickr') do |a|
        a.description = 'Photos from my Flickr account.'
        a.is_public = false
      end
      picture = person.pictures.create(
        :album_id     => album.id,
        :original_url => entry.url,
        :created_at   => entry.published
      )
      if entry.content =~ /<img src="([^"]+_m\.jpg)/
        picture.photo = $1.sub(/_m\.jpg$/, '_b.jpg')
        unless picture.has_photo?
          picture.destroy
        end
      else
        picture.destroy
      end
    end
  end
  
  def self.import_all
    Feed.all.each do |feed|
      feed.import_content
    end
  end
  
  def self.transform_url(url)
    url = url.to_s
    if url.include?('facebook.com')
{'fraction_non_alphanumeric': 0.12478632478632479, 'fraction_numerical': 0.0017094017094017094, 'mean_word_length': 2.2017783857729136, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5350935', 'n_tokens_mistral': 1538, 'n_tokens_neox': 1530, 'n_words': 316}