<commit_before>#Copyright (c) 2012, Kreeti Technologies All Rights Reserved.
require 'nokogiri'
require 'open-uri'
require 'parser/oembed_parser'
require 'parser/open_graph_parser'
require 'parser/html_parser'
require 'parser/twitter_parser'

module Widgetify

  class Base
    attr_reader :parse_result

    def initialize(url, options = {}, *args)
      @html_doc = Nokogiri::HTML(open(url))
      @parse_result = { }
      @options = options
      @options[:url] = url
      parse_all if args.length == 0
      args.each{ |arg| send(arg.to_sym)} if args.length > 0
    end

    def parse_open_graph
      @parse_result['open_graph'] = Widgetify::OpenGraphParser.new(@html_doc)
    end

    def parse_oembed
      @parse_result['oembed'] = Widgetify::OembedParser.new(@html_doc, @options)
    end

    def parse_html
      @parse_result['html'] = Widgetify::HTMLParser.new(@html_doc)
    end

    def parse_twitter
      @parse_result['twitter'] = Widgetify::TwitterParser.new(@html_doc)
    end

    def parse_all
      parse_open_graph
      parse_oembed
      parse_html
    end
  end<commit_msg>gem changed for image urls<commit_after>#Copyright (c) 2012, Kreeti Technologies All Rights Reserved.
require 'nokogiri'
require 'open-uri'
require 'parser/oembed_parser'
require 'parser/open_graph_parser'
require 'parser/html_parser'
require 'parser/twitter_parser'

module Widgetify

  class Base
    attr_reader :parse_result

    def initialize(url, options = {}, *args)
      @content_type = open(url){|u| u.content_type}
      @parse_result = { }
      @options = options
      @options[:url] = url
      if @content_type.include?("text")
        @html_doc = Nokogiri::HTML(open(url))
        parse_all if args.length == 0
        args.each{ |arg| send(arg.to_sym)} if args.length > 0
      elsif @content_type.include?("image")
        parse_image_url if args.include?("parse_image_url".to_sym)
      end  
    end

    def parse_open_graph
      @parse_result['open_graph'] = Widgetify::OpenGraphParser.new(@html_doc)
    end

    def parse_oembed
      @parse_result['oembed'] = Widgetify::OembedParser.new(@html_doc, @options)
    end

    def parse_html
      @parse_result['html'] = Widgetify::HTMLParser.new(@html_doc)
    end

    def parse_twitter
      @parse_result['twitter'] = Widgetify::TwitterParser.new(@html_doc)
    end

    def parse_all
      parse_open_graph
      parse_oembed
      parse_html
    end
  
    def parse_image_url
      @parse_result['image'] = {:url => @options[:url], :type => "photo"} if @content_type.include?("image")
    end
  end
{'fraction_non_alphanumeric': 0.1124031007751938, 'fraction_numerical': 0.004651162790697674, 'mean_word_length': 3.4271012006861064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1019097', 'n_tokens_mistral': 861, 'n_tokens_neox': 841, 'n_words': 176}