<commit_before>class Story < ActiveRecord::Base
  belongs_to :user
  has_many :taggings,
    :autosave => true
  has_many :comments,
    :inverse_of => :story
  has_many :tags, :through => :taggings

  validates_length_of :title, :in => 3..150
  validates_length_of :description, :maximum => (64 * 1024)
  validates_presence_of :user_id

  DOWNVOTABLE_DAYS = 14

  # after this many minutes old, a story cannot be edited
  MAX_EDIT_MINS = 90

  # days a story is considered recent, for resubmitting
  RECENT_DAYS = 30

  attr_accessor :vote, :already_posted_story, :fetched_content, :previewing,
    :seen_previous
  attr_accessor :editor, :moderation_reason

  before_validation :assign_short_id_and_upvote,
    :on => :create
  before_save :log_moderation
  after_create :mark_submitter, :record_initial_upvote

  validate do
    if self.url.present?
      # URI.parse is not very lenient, so we can't use it

      if self.url.match(/\Ahttps?:\/\/([^\.]+\.)+[a-z]+(\/|\z)/)
        if self.new_record? && (s = Story.find_similar_by_url(self.url))
          self.already_posted_story = s
          if s.is_recent?
            errors.add(:url, "has already been submitted within the past " <<
              "#{RECENT_DAYS} days")
          end
        end
      else
        errors.add(:url, "is not valid")
      end
    elsif self.description.to_s.strip == ""
      errors.add(:description, "must contain text if no URL posted")
    end

    check_tags
  end

  def self.find_similar_by_url(url)
    urls = [ url.to_s ]
    urls2 = [ url.to_s ]

    # https
    urls.each do |u|
      urls2.push u.gsub(/^http:\/\//i, "https://")
      urls2.push u.gsub(/^https:\/\//i, "http://")
    end
    urls = urls2.clone

    # trailing slash
    urls.each do |u|
      urls2.push u.gsub(/\/+\z/, "")
      urls2.push (u + "/")
    end
    urls = urls2.clone

    # www prefix
    urls.each do |u|
      urls2.push u.gsub(/^(https?:\/\/)www\d*\./i) {|_| $1 }
      urls2.push u.gsub(/^(https?:\/\/)/i) {|_| "#{$1}www." }
    end
    urls = urls2.clone

    if s = Story.where(:url => urls, :is_expired => false).order("id DESC").first
      return s
    end

    false
  end

  def self.recalculate_all_hotnesses!
    Story.all.each do |s|
      s.recalculate_hotness!
    end
  end

  def as_json(options = {})
    h = super(:only => [
      :short_id,
      :created_at,
      :title,
      :url,
    ])
    h[:score] = score
    h[:comment_count] = comments_count
    h[:description] = markeddown_description
    h[:comments_url] = comments_url
    h[:submitter_user] = user

    if options && options[:with_comments]
      h[:comments] = options[:with_comments]
    end

    h
  end

  def assign_short_id_and_upvote
    self.short_id = ShortId.new(self.class).generate
    self.upvotes = 1
  end

  def calculated_hotness
    # don't immediately kill stories at 0 by bumping up score by one
    order = Math.log([ (score + 1).abs, 1 ].max, 10)
    if score > 0<commit_msg>Story: export tags in json view<commit_after>class Story < ActiveRecord::Base
  belongs_to :user
  has_many :taggings,
    :autosave => true
  has_many :comments,
    :inverse_of => :story
  has_many :tags, :through => :taggings

  validates_length_of :title, :in => 3..150
  validates_length_of :description, :maximum => (64 * 1024)
  validates_presence_of :user_id

  DOWNVOTABLE_DAYS = 14

  # after this many minutes old, a story cannot be edited
  MAX_EDIT_MINS = 90

  # days a story is considered recent, for resubmitting
  RECENT_DAYS = 30

  attr_accessor :vote, :already_posted_story, :fetched_content, :previewing,
    :seen_previous
  attr_accessor :editor, :moderation_reason

  before_validation :assign_short_id_and_upvote,
    :on => :create
  before_save :log_moderation
  after_create :mark_submitter, :record_initial_upvote

  validate do
    if self.url.present?
      # URI.parse is not very lenient, so we can't use it

      if self.url.match(/\Ahttps?:\/\/([^\.]+\.)+[a-z]+(\/|\z)/)
        if self.new_record? && (s = Story.find_similar_by_url(self.url))
          self.already_posted_story = s
          if s.is_recent?
            errors.add(:url, "has already been submitted within the past " <<
              "#{RECENT_DAYS} days")
          end
        end
      else
        errors.add(:url, "is not valid")
      end
    elsif self.description.to_s.strip == ""
      errors.add(:description, "must contain text if no URL posted")
    end

    check_tags
  end

  def self.find_similar_by_url(url)
    urls = [ url.to_s ]
    urls2 = [ url.to_s ]

    # https
    urls.each do |u|
      urls2.push u.gsub(/^http:\/\//i, "https://")
      urls2.push u.gsub(/^https:\/\//i, "http://")
    end
    urls = urls2.clone

    # trailing slash
    urls.each do |u|
      urls2.push u.gsub(/\/+\z/, "")
      urls2.push (u + "/")
    end
    urls = urls2.clone

    # www prefix
    urls.each do |u|
      urls2.push u.gsub(/^(https?:\/\/)www\d*\./i) {|_| $1 }
      urls2.push u.gsub(/^(https?:\/\/)/i) {|_| "#{$1}www." }
    end
    urls = urls2.clone

    if s = Story.where(:url => urls, :is_expired => false).order("id DESC").first
      return s
    end

    false
  end

  def self.recalculate_all_hotnesses!
    Story.all.each do |s|
      s.recalculate_hotness!
    end
  end

  def as_json(options = {})
    h = super(:only => [
      :short_id,
      :created_at,
      :title,
      :url,
    ])
    h[:score] = score
    h[:comment_count] = comments_count
    h[:description] = markeddown_description
    h[:comments_url] = comments_url
    h[:submitter_user] = user
    h[:tags] = self.tags.map{|t| t.tag }.sort

    if options && options[:with_comments]
      h[:comments] = options[:with_comments]
    end

    h
  end

  def assign_short_id_and_upvote
    self.short_id = ShortId.new(self.class).generate
    self.upvotes = 1
  end

  def calculated_hotness
    # don't immediately kill stories at 0 by bumping up score by one
    order = Math.log([ (score + 1).abs, 1 ].max, 10)
    if score > 0
{'fraction_non_alphanumeric': 0.1346986346986347, 'fraction_numerical': 0.011655011655011656, 'mean_word_length': 2.5067133683596032, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905000', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2153, 'n_words': 504}