<commit_before>module MediaPageItem
  extend ActiveSupport::Concern

  included do
    Media.declare('page_item', [/^.*$/])
  end

  def data_from_page_item
    self.doc ||= self.get_html
    handle_exceptions(RuntimeError) do
      self.data = self.page_get_data_from_url
    end

    if self.data[:picture].blank?
      generate_screenshot
    else
      self.data[:picture] = self.add_scheme(self.data[:picture])
    end
  end

  def page_get_data_from_url
    raise 'Could not parse this media' if self.doc.blank?
    data = self.data
    %w(basic oembed opengraph twitter).each do |meta|
      data.merge!(self.send("get_#{meta}_metadata")) { |_key, v1, v2| v2.blank? ? v1 : v2 }
    end
    data
  end

  def get_twitter_metadata
    metatags = { title: 'twitter:title', picture: 'twitter:image', description: 'twitter:description', username: 'twitter:creator' }
    data = get_html_metadata('name', metatags).with_indifferent_access
    data.merge!(get_html_metadata('property', metatags))
    data['author_url'] = 'https://twitter.com/' + data['username'] if data['username']
    data
  end

  def get_opengraph_metadata
    metatags = { title: 'og:title', picture: 'og:image', description: 'og:description', username: 'article:author', published_at: 'article:published_time' }
    get_html_metadata('property', metatags)
  end

  def get_oembed_metadata
    data = self.data_from_oembed_item
    self.provider = 'oembed' if data
    data || {}
  end

  def get_basic_metadata
    metatags = { title: 'title',  description: 'description', username: 'author' }
    data = get_html_metadata('name', metatags)
    title = self.doc.at_css("title")
    data[:title] ||= title.nil? ? '' : title.content
    data[:description] ||= data[:title]
    data[:username] ||= ''
    data[:published_at] = ''
    data[:picture] = ''

    data[:author_url] = top_url(self.url)<commit_msg>Ticket #5864: Return blank on description when not found on page<commit_after>module MediaPageItem
  extend ActiveSupport::Concern

  included do
    Media.declare('page_item', [/^.*$/])
  end

  def data_from_page_item
    self.doc ||= self.get_html
    handle_exceptions(RuntimeError) do
      self.data = self.page_get_data_from_url
    end

    if self.data[:picture].blank?
      generate_screenshot
    else
      self.data[:picture] = self.add_scheme(self.data[:picture])
    end
  end

  def page_get_data_from_url
    raise 'Could not parse this media' if self.doc.blank?
    data = self.data
    %w(basic oembed opengraph twitter).each do |meta|
      data.merge!(self.send("get_#{meta}_metadata")) { |_key, v1, v2| v2.blank? ? v1 : v2 }
    end
    data
  end

  def get_twitter_metadata
    metatags = { title: 'twitter:title', picture: 'twitter:image', description: 'twitter:description', username: 'twitter:creator' }
    data = get_html_metadata('name', metatags).with_indifferent_access
    data.merge!(get_html_metadata('property', metatags))
    data['author_url'] = 'https://twitter.com/' + data['username'] if data['username']
    data
  end

  def get_opengraph_metadata
    metatags = { title: 'og:title', picture: 'og:image', description: 'og:description', username: 'article:author', published_at: 'article:published_time' }
    get_html_metadata('property', metatags)
  end

  def get_oembed_metadata
    data = self.data_from_oembed_item
    self.provider = 'oembed' if data
    data || {}
  end

  def get_basic_metadata
    metatags = { title: 'title',  description: 'description', username: 'author' }
    data = get_html_metadata('name', metatags)
    title = self.doc.at_css("title")
    data[:title] ||= title.nil? ? '' : title.content
    data[:description] ||= ''
    data[:username] ||= ''
    data[:published_at] = ''
    data[:picture] = ''

    data[:author_url] = top_url(self.url)
{'fraction_non_alphanumeric': 0.13661489865754145, 'fraction_numerical': 0.0036851803106080546, 'mean_word_length': 3.8906048906048905, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '510103', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1246, 'n_words': 260}