<commit_before>require 'open-uri'

class Link < ActiveRecord::Base
  scope :search, lambda { |term| where(arel_table[:line].matches("%#{term.strip}%")) }

  validates_presence_of :network, :channel, :sender, :line

  after_create :mirror_image!,   :if => :image?
  after_create :mirror_imgur!,   :if => :imgur?
  after_create :mirror_ninegag!, :if => :ninegag?

  def self.request_resource_mirroring(uri)
    RestClient.post(ENV['MIRROR_SERVICE_URI'], :uri => uri) rescue false
  end

  def uri
    line.match(/(http[s]?:\/\/[^\s]+)/)[0]
  end

  def image?
    uri.end_with? *%w(.png .gif .jpg .jpeg)
  end

  def imgur?
    uri.start_with?('http://imgur.com/')
  end

  def ninegag?
    uri.start_with?('http://9gag.com/gag/')
  end

  def imgur_image_uri
    @imgur_image_uri ||= get_imgur_image_uri
  end

  def ninegag_image_uri
    @ninegag_image_uri ||= get_ninegag_image_uri
  end

  # Tagged Not Safe For Work or Not Mind Safe
  def nws?
    nms? || line.split(' ').any? { |word| word.match(/^(nws|nsfw)[.?!:]*$/i) }
  end

  # Tagged Not Mind Safe
  def nms?
    line.split(' ').any? { |word| word.match(/^(nms|nsfm)[.?!:]*$/i) }
  end

  # Tagged as Spoiler
  def spoiler?
    line.split(' ').any? { |word| word.match(/^(spoil|spoiler)[.?!:]*$/i) }
  end

  private
  def mirror_image!
    mirror_uri = self.class.request_resource_mirroring(uri)
    update_column(:mirror_uri, mirror_uri) if mirror_uri
  end

  def mirror_imgur!
    if imgur_image_uri
      mirror_uri = self.class.request_resource_mirroring(imgur_image_uri)
      update_column(:mirror_uri, mirror_uri) if mirror_uri
    end
  end

  def mirror_ninegag!
    if ninegag_image_uri
      mirror_uri = self.class.request_resource_mirroring(ninegag_image_uri)
      update_column(:mirror_uri, mirror_uri) if mirror_uri
    end
  end
<commit_msg>Slightly better tag detection, allowing all non letter characters as punctuation<commit_after>require 'open-uri'

class Link < ActiveRecord::Base
  scope :search, lambda { |term| where(arel_table[:line].matches("%#{term.strip}%")) }

  validates_presence_of :network, :channel, :sender, :line

  after_create :mirror_image!,   :if => :image?
  after_create :mirror_imgur!,   :if => :imgur?
  after_create :mirror_ninegag!, :if => :ninegag?

  def self.request_resource_mirroring(uri)
    RestClient.post(ENV['MIRROR_SERVICE_URI'], :uri => uri) rescue false
  end

  def uri
    line.match(/(http[s]?:\/\/[^\s]+)/)[0]
  end

  def image?
    uri.end_with? *%w(.png .gif .jpg .jpeg)
  end

  def imgur?
    uri.start_with?('http://imgur.com/')
  end

  def ninegag?
    uri.start_with?('http://9gag.com/gag/')
  end

  def imgur_image_uri
    @imgur_image_uri ||= get_imgur_image_uri
  end

  def ninegag_image_uri
    @ninegag_image_uri ||= get_ninegag_image_uri
  end

  # Tagged Not Safe For Work or Not Mind Safe
  def nws?
    nms? || line.split(' ').any? { |word| word.match(/^(nws|nsfw)[^\w]*$/i) }
  end

  # Tagged Not Mind Safe
  def nms?
    line.split(' ').any? { |word| word.match(/^(nms|nsfm)[^\w]*$/i) }
  end

  # Tagged as Spoiler
  def spoiler?
    # line.split(' ').any? { |word| word.match(/^(spoil|spoiler)[.?!:]*$/i) }
    line.split(' ').any? { |word| word.match(/^(spoil|spoiler)[^\w]*$/i) }
  end

  private
  def mirror_image!
    mirror_uri = self.class.request_resource_mirroring(uri)
    update_column(:mirror_uri, mirror_uri) if mirror_uri
  end

  def mirror_imgur!
    if imgur_image_uri
      mirror_uri = self.class.request_resource_mirroring(imgur_image_uri)
      update_column(:mirror_uri, mirror_uri) if mirror_uri
    end
  end

  def mirror_ninegag!
    if ninegag_image_uri
      mirror_uri = self.class.request_resource_mirroring(ninegag_image_uri)
      update_column(:mirror_uri, mirror_uri) if mirror_uri
    end
  end

{'fraction_non_alphanumeric': 0.14648334214701217, 'fraction_numerical': 0.0010576414595452142, 'mean_word_length': 3.6819306930693068, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2531383', 'n_tokens_mistral': 1512, 'n_tokens_neox': 1447, 'n_words': 288}