<commit_before>require 'nokogiri'
require 'redirect_follower'
require "addressable/uri"
require 'uri'

class OpenGraph
  attr_accessor :src, :url, :type, :title, :description, :images, :metadata, :response, :original_images

  def initialize(src, fallback = true, options = {})
    if fallback.is_a? Hash
      options = fallback
      fallback = true
    end
    @src = src
    @body = nil
    @images = []
    @metadata = {}
    parse_opengraph(options)
    load_fallback if fallback
    check_images_path
  end

  private
  def parse_opengraph(options = {})
    begin
      if @src.include? '</html>'
        @body = @src
      else
        @body = RedirectFollower.new(@src, options).resolve.body
      end
    rescue
      @title = @url = @src
      return
    end

    if @body
      attrs_list = %w(title url type description)
      doc = Nokogiri.parse(@body)
      doc.css('meta').each do |m|
        if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
          m_content = m.attribute('content').to_s.strip
          metadata_name = m.attribute('property').to_s.gsub("og:", "")
          @metadata = add_metadata(@metadata, metadata_name, m_content)
          case metadata_name
            when *attrs_list
              self.instance_variable_set("@#{metadata_name}", m_content) unless m_content.empty?
            when "image"
              add_image(m_content)
          end<commit_msg>Add html_source?<commit_after>require 'nokogiri'
require 'redirect_follower'
require "addressable/uri"
require 'uri'

class OpenGraph
  attr_accessor :src, :url, :type, :title, :description, :images, :metadata, :response, :original_images

  def initialize(src, fallback = true, options = {})
    if fallback.is_a? Hash
      options = fallback
      fallback = true
    end
    @src = src
    @body = nil
    @images = []
    @metadata = {}
    parse_opengraph(options)
    load_fallback if fallback
    check_images_path unless html_source?
  end

  private
  def html_source?
    @src.include? '</html>'
  end

  def parse_opengraph(options = {})
    begin
      if html_source?
        @body = @src
      else
        @body = RedirectFollower.new(@src, options).resolve.body
      end
    rescue
      @title = @url = @src
      return
    end

    if @body
      attrs_list = %w(title url type description)
      doc = Nokogiri.parse(@body)
      doc.css('meta').each do |m|
        if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
          m_content = m.attribute('content').to_s.strip
          metadata_name = m.attribute('property').to_s.gsub("og:", "")
          @metadata = add_metadata(@metadata, metadata_name, m_content)
          case metadata_name
            when *attrs_list
              self.instance_variable_set("@#{metadata_name}", m_content) unless m_content.empty?
            when "image"
              add_image(m_content)
          end
{'fraction_non_alphanumeric': 0.1218343600273785, 'fraction_numerical': 0.0, 'mean_word_length': 2.5777233782129745, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '760560', 'n_tokens_mistral': 919, 'n_tokens_neox': 915, 'n_words': 206}