<commit_before>
Dir["#{Rails.root}/lib/onebox/engine/*_onebox.rb"].each {|f|
  require_dependency(f.split('/')[-3..-1].join('/'))
}

module Oneboxer


  # keep reloaders happy
  unless defined? Oneboxer::Result
    Result = Struct.new(:doc, :changed) do
      def to_html
        doc.to_html
      end

      def changed?
        changed
      end
    end
  end

  def self.preview(url, options=nil)
    options ||= {}
    Oneboxer.invalidate(url) if options[:invalidate_oneboxes]
    onebox_raw(url).placeholder_html
  end

  def self.onebox(url, options=nil)
    options ||= {}
    Oneboxer.invalidate(url) if options[:invalidate_oneboxes]
    onebox_raw(url).to_s
  end

  def self.cached_onebox(url)
    Rails.cache.read(onebox_cache_key(url))
      .to_s
  end

  def self.cached_preview(url)
    Rails.cache.read(onebox_cache_key(url))
      .try(:placeholder_html)
      .to_s
  end

  def self.oneboxer_exists_for_url?(url)
    Onebox.has_matcher?(url)
  end

  def self.invalidate(url)
    Rails.cache.delete(onebox_cache_key(url))
  end

  # Parse URLs out of HTML, returning the document when finished.
  def self.each_onebox_link(string_or_doc)
    doc = string_or_doc
    doc = Nokogiri::HTML::fragment(doc) if doc.is_a?(String)

    onebox_links = doc.search("a.onebox")
    if onebox_links.present?
      onebox_links.each do |link|
        if link['href'].present?
          yield link['href'], link
        end
      end
    end

    doc
  end

  def self.apply(string_or_doc)
    doc = string_or_doc
    doc = Nokogiri::HTML::fragment(doc) if doc.is_a?(String)
    changed = false

    Oneboxer.each_onebox_link(doc) do |url, element|
      onebox, preview = yield(url,element)
      if onebox
        parsed_onebox = Nokogiri::HTML::fragment(onebox)
        next unless parsed_onebox.children.count > 0

        # special logic to strip empty p elements
        if  element.parent &&
            element.parent.node_name.downcase == "p" &&
            element.parent.children.count == 1 &&
            parsed_onebox.children.first.name.downcase == "div"
          element = element.parent
        end
        changed = true
        element.swap parsed_onebox.to_html
      end
    end

    Result.new(doc, changed)
  end

  private
  def self.onebox_cache_key(url)
    "onebox_#{url}"
  end

  def self.onebox_raw(url)
    Rails.cache.fetch(onebox_cache_key(url)){
      Onebox.preview(url, cache: {})
    }
  end

end<commit_msg>BUGFIX: protect ourselved against rogue onebox gem<commit_after>
Dir["#{Rails.root}/lib/onebox/engine/*_onebox.rb"].each {|f|
  require_dependency(f.split('/')[-3..-1].join('/'))
}

module Oneboxer


  # keep reloaders happy
  unless defined? Oneboxer::Result
    Result = Struct.new(:doc, :changed) do
      def to_html
        doc.to_html
      end

      def changed?
        changed
      end
    end
  end

  def self.preview(url, options=nil)
    options ||= {}
    Oneboxer.invalidate(url) if options[:invalidate_oneboxes]
    onebox_raw(url)[:preview]
  end

  def self.onebox(url, options=nil)
    options ||= {}
    Oneboxer.invalidate(url) if options[:invalidate_oneboxes]
    onebox_raw(url)[:onebox]
  end

  def self.cached_onebox(url)
    if c = Rails.cache.read(onebox_cache_key(url))
      c[:onebox]
    end
  end

  def self.cached_preview(url)
    if c = Rails.cache.read(onebox_cache_key(url))
      c[:preview]
    end
  end

  def self.oneboxer_exists_for_url?(url)
    Onebox.has_matcher?(url)
  end

  def self.invalidate(url)
    Rails.cache.delete(onebox_cache_key(url))
  end

  # Parse URLs out of HTML, returning the document when finished.
  def self.each_onebox_link(string_or_doc)
    doc = string_or_doc
    doc = Nokogiri::HTML::fragment(doc) if doc.is_a?(String)

    onebox_links = doc.search("a.onebox")
    if onebox_links.present?
      onebox_links.each do |link|
        if link['href'].present?
          yield link['href'], link
        end
      end
    end

    doc
  end

  def self.apply(string_or_doc)
    doc = string_or_doc
    doc = Nokogiri::HTML::fragment(doc) if doc.is_a?(String)
    changed = false

    Oneboxer.each_onebox_link(doc) do |url, element|
      onebox, preview = yield(url,element)
      if onebox
        parsed_onebox = Nokogiri::HTML::fragment(onebox)
        next unless parsed_onebox.children.count > 0

        # special logic to strip empty p elements
        if  element.parent &&
            element.parent.node_name.downcase == "p" &&
            element.parent.children.count == 1 &&
            parsed_onebox.children.first.name.downcase == "div"
          element = element.parent
        end
        changed = true
        element.swap parsed_onebox.to_html
      end
    end

    Result.new(doc, changed)
  end

  private
  def self.onebox_cache_key(url)
    "onebox__#{url}"
  end

  def self.onebox_raw(url)
    Rails.cache.fetch(onebox_cache_key(url)){
      begin
        r = Onebox.preview(url, cache: {})
        {
          onebox: r.to_s,
          preview: r.try(:placeholder_html).to_s
        }
      rescue => e
        Discourse.handle_exception(e, url: url)
      end
    }
  end

end
{'fraction_non_alphanumeric': 0.11221507890122735, 'fraction_numerical': 0.0015585427625170465, 'mean_word_length': 2.6180408738548273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58110', 'n_tokens_mistral': 1798, 'n_tokens_neox': 1768, 'n_words': 346}