<commit_before>
#
# Terminus-Bot: An IRC bot to solve all of the problems with IRC bots.
# Copyright (C) 2012 Terminus-Bot Development Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#


require "net/http"
require "uri"
require "strscan"
require "htmlentities"
require 'rexml/document'

def initialize
  register_script("Fetches titles for URLs spoken in channels.")

  register_event("PRIVMSG", :on_message)
end

def on_message(msg)
  return if msg.silent? or msg.private?

  i = 0
  max = get_config("max", 3).to_i

  URI.extract(msg.text, ["http","https"]) { |match|
    return if i >= max

    $log.debug("title.on_message") { "#{i}/#{max}: #{match}" }

    match = URI(match)

    if match.host =~ /(www\.)?youtube\.com/ and not match.query == nil
      next if get_youtube(msg, match)
    elsif match.host =~ /(www\.)?youtu\.be/ and not match.path == nil
      next if get_youtube(msg, match)
    end
    
    get_title(msg, match)

    i += 1
  }
end

def get_youtube(msg, uri)
  $log.debug('title.get_title') { "Getting YouTube info for #{uri}" }

  # TODO: This whole function is really awful and needs cleaning up.

  link = ""
  vid = ""

  if uri.host == 'youtu.be'
    vid = uri.path[1..uri.path.length-1].split("&")[0]
  else
    query = uri.query.split("&").select {|a| a.start_with? "v="}[0]

    return false if query == nil

    vid = query.split("=")[1]

    link = " - http://youtu.be/#{vid}"
  end

  $log.debug('title.get_title') { "VID #{vid}" }

  vid = URI.escape(vid)
  api = URI("https://gdata.youtube.com/feeds/api/videos/#{vid}?v=2")

  response = get_page(api)

  return false if response == nil

  root = REXML::Document.new(response[0].body.force_encoding('UTF-8')).root

  return false unless root.get_elements("error").empty?

  title    = root.get_elements("title").first.text.to_s
  author   = root.get_elements("author/name").first.text.to_s
  views    = root.get_elements("yt:statistics").first.attribute("viewCount").to_s

  rating   = root.get_elements("yt:rating").first

  likes    = rating.attribute("numLikes").to_s
  dislikes = rating.attribute("numDislikes").to_s
  
  msg.reply("\02YouTube Video\02 #{title} \02Uploaded By:\02 #{author} \02Views:\02 #{views} \02Likes:\02 #{likes} \02Dislikes:\02 #{dislikes}#{link}")

  return true
end



def get_title(msg, uri)
  begin
    $log.debug('title.get_title') { "Getting title for #{uri}" }

    response = get_page(uri)

    return if response == nil

    page = StringScanner.new(response[0].body.force_encoding('UTF-8'))

    page.skip_until(/<title[^>]*>/ix)
    title = page.scan_until(/<\/title[^>]*>/ix)

    return if title == nil

    len = title.length - 9
    return if len <= 0

    title = title[0..len].strip.gsub(/[[[:cntrl:]]\s]+/, " ")
    title = HTMLEntities.new.decode(title)

    msg.reply("\02Title on #{response[1]}#{" (redirected)" if response[2]}:\02 " + title, false)
  rescue => e
    $log.debug('title.get_title') { "Error getting title for #{uri}: #{e}" }
    return
  end
end

def get_page(uri, limit = get_config("redirects", 10), redirected = false)
  return nil if limit == 0

  response = Net::HTTP.start(uri.host, uri.port,
    :use_ssl => uri.scheme == "https",
    :verify_mode => OpenSSL::SSL::VERIFY_NONE) do |http|

    http.request Net::HTTP::Get.new(uri.request_uri)
  end

  case response

  when Net::HTTPSuccess
    return [response, uri.hostname, redirected]

  when Net::HTTPRedirection
    location = URI(response['location'])

    $log.debug("title.get_page") { "Redirection: #{uri} -> #{location} (#{limit})" }

    return get_page(location, limit - 1, true)

  else
    return nil

  end


end
<commit_msg>Title script now shows info for videos which have never been rated.<commit_after>
#
# Terminus-Bot: An IRC bot to solve all of the problems with IRC bots.
# Copyright (C) 2012 Terminus-Bot Development Team
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#


require "net/http"
require "uri"
require "strscan"
require "htmlentities"
require 'rexml/document'

def initialize
  register_script("Fetches titles for URLs spoken in channels.")

  register_event("PRIVMSG", :on_message)
end

def on_message(msg)
  return if msg.silent? or msg.private?

  i = 0
  max = get_config("max", 3).to_i

  URI.extract(msg.text, ["http","https"]) { |match|
    return if i >= max

    $log.debug("title.on_message") { "#{i}/#{max}: #{match}" }

    match = URI(match)

    if match.host =~ /(www\.)?youtube\.com/ and not match.query == nil
      next if get_youtube(msg, match)
    elsif match.host =~ /(www\.)?youtu\.be/ and not match.path == nil
      next if get_youtube(msg, match)
    end
    
    get_title(msg, match)

    i += 1
  }
end

def get_youtube(msg, uri)
  $log.debug('title.get_title') { "Getting YouTube info for #{uri}" }

  # TODO: This whole function is really awful and needs cleaning up.

  link = ""
  vid = ""

  if uri.host == 'youtu.be'
    vid = uri.path[1..uri.path.length-1].split("&")[0]
  else
    query = uri.query.split("&").select {|a| a.start_with? "v="}[0]

    return false if query == nil

    vid = query.split("=")[1]

    link = " - http://youtu.be/#{vid}"
  end

  $log.debug('title.get_title') { "VID #{vid}" }

  vid = URI.escape(vid)
  api = URI("https://gdata.youtube.com/feeds/api/videos/#{vid}?v=2")

  response = get_page(api)

  return false if response == nil

  root = REXML::Document.new(response[0].body.force_encoding('UTF-8')).root

  return false unless root.get_elements("error").empty?

  title    = root.get_elements("title").first.text.to_s
  author   = root.get_elements("author/name").first.text.to_s
  views    = root.get_elements("yt:statistics").first.attribute("viewCount").to_s

  rating   = root.get_elements("yt:rating").first

  likes, dislikes = 0, 0

  unless rating == nil
    likes    = rating.attribute("numLikes").to_s
    dislikes = rating.attribute("numDislikes").to_s
  end
  
  msg.reply("\02YouTube Video\02 #{title} \02Uploaded By:\02 #{author} \02Views:\02 #{views} \02Likes:\02 #{likes} \02Dislikes:\02 #{dislikes}#{link}")

  return true
end



def get_title(msg, uri)
  begin
    $log.debug('title.get_title') { "Getting title for #{uri}" }

    response = get_page(uri)

    return if response == nil

    page = StringScanner.new(response[0].body.force_encoding('UTF-8'))

    page.skip_until(/<title[^>]*>/ix)
    title = page.scan_until(/<\/title[^>]*>/ix)

    return if title == nil

    len = title.length - 9
    return if len <= 0

    title = title[0..len].strip.gsub(/[[[:cntrl:]]\s]+/, " ")
    title = HTMLEntities.new.decode(title)

    msg.reply("\02Title on #{response[1]}#{" (redirected)" if response[2]}:\02 " + title, false)
  rescue => e
    $log.debug('title.get_title') { "Error getting title for #{uri}: #{e}" }
    return
  end
end

def get_page(uri, limit = get_config("redirects", 10), redirected = false)
  return nil if limit == 0

  response = Net::HTTP.start(uri.host, uri.port,
    :use_ssl => uri.scheme == "https",
    :verify_mode => OpenSSL::SSL::VERIFY_NONE) do |http|

    http.request Net::HTTP::Get.new(uri.request_uri)
  end

  case response

  when Net::HTTPSuccess
    return [response, uri.hostname, redirected]

  when Net::HTTPRedirection
    location = URI(response['location'])

    $log.debug("title.get_page") { "Redirection: #{uri} -> #{location} (#{limit})" }

    return get_page(location, limit - 1, true)

  else
    return nil

  end


end

{'fraction_non_alphanumeric': 0.15299155190371486, 'fraction_numerical': 0.01203564402268256, 'mean_word_length': 3.510438413361169, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3575696', 'n_tokens_mistral': 3017, 'n_tokens_neox': 2882, 'n_words': 805}