<commit_before>#!/usr/bin/env ruby
# -*- coding: utf-8 -*-
# Copyright (c) 2010 <NAME>
# This program is made available under the terms of the MIT License.

require 'cinch'
require 'nokogiri'
require 'curb'
require 'uri'
require 'cgi'

module Cinch
  module Plugins
    class Title
      include Cinch::Plugin
      
      match /(.*http.*)/, :use_prefix => false
      
      def execute m, message
        URI.extract(message, ["http", "https"]) do |uri|
          begin
            next if ignore uri
            
            m.reply parse(uri)
          rescue URI::InvalidURIError => e
            m.reply "Invalid url: #{uri}"
          end
        end
      end

      private

      def parse uri
        uri.gsub! "#!", "?_escaped_fragment_=" if uri.include? "#!"
        uri.gsub! /#.+$/, "" if uri.include? "#"
        
        call = Curl::Easy.perform(uri) do |easy| 
          easy.follow_location = true
          easy.max_redirects = config["max_redirects"]
        end
        html = Nokogiri::HTML(call.body_str)
        title = html.at_xpath('//title')
        
        return "No title" if title.nil?
        CGI.unescape_html title.text.gsub(/\s+/, ' ')
      end
      
      def ignore uri
        ignore = ["jpg$", "JPG$", "jpeg$", "gif$", "png$", "bmp$", "pdf$", "jpe$"]
        ignore.concat(config["ignore"]) if config.key? "ignore"
        
        ignore.each do |re|
          return true if uri =~ /#{re}/
        end
        
        false
      end
    end<commit_msg>Caught an exception and added the ability for custom responses<commit_after>#!/usr/bin/env ruby
# -*- coding: utf-8 -*-
# Copyright (c) 2010 <NAME>
# This program is made available under the terms of the MIT License.

require 'cinch'
require 'nokogiri'
require 'curb'
require 'uri'
require 'cgi'

module Cinch
  module Plugins
    class Title
      include Cinch::Plugin
      
      match /(.*http.*)/, :use_prefix => false
      
      def execute m, message
        URI.extract(message, ["http", "https"]) do |uri|
          begin
            next if ignore uri
            
            m.reply response(m, parse(uri))
          rescue URI::InvalidURIError => e
            m.reply response_invalid(m, uri)
          end
        end
      rescue
        puts "URL doesn't exist or can't be accessed."
      end
      
      def response m, uri
        uri
      end
      
      def response_invalid m, uri
        "Invalid url: #{uri}"
      end

      private

      def parse uri
        uri.gsub! "#!", "?_escaped_fragment_=" if uri.include? "#!"
        uri.gsub! /#.+$/, "" if uri.include? "#"
        
        call = Curl::Easy.perform(uri) do |easy| 
          easy.follow_location = true
          easy.max_redirects = config["max_redirects"]
        end
        
        html = Nokogiri::HTML(call.body_str)
        title = html.at_xpath('//title')
        
        return "No title" if title.nil?
        CGI.unescape_html title.text.gsub(/\s+/, ' ')
      end
      
      def ignore uri
        ignore = ["jpg$", "JPG$", "jpeg$", "gif$", "png$", "bmp$", "pdf$", "jpe$"]
        ignore.concat(config["ignore"]) if config.key? "ignore"
        
        ignore.each do |re|
          return true if uri =~ /#{re}/
        end
        
        false
      end
    end
{'fraction_non_alphanumeric': 0.13201219512195123, 'fraction_numerical': 0.003048780487804878, 'mean_word_length': 1.7318900915903415, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2820484', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1016, 'n_words': 287}