<commit_before># Description:
#   Webutility returns title of urls
#
# Dependencies:
#   "htmlparser": "1.7.6"
#   "soupselect": "0.2.0"
#   "jsdom": "0.2.14"
#
# Configuration:
#   HUBOT_BITLY_USERNAME
#   HUBOT_BITLY_API_KEY
#
# Commands:
#   None
#
# Author:
#   KevinTraver

Select     = require("soupselect").select
HtmlParser = require "htmlparser"
JSDom      = require "jsdom"

# Decode HTML entities
unEntity = (str) ->
    e = JSDom.jsdom().createElement("div")
    e.innerHTML = str
    if e.childNodes.length == 0 then "" else e.childNodes[0].nodeValue

module.exports = (robot) ->
  robot.hear /(http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?/i, (msg) ->
    url = msg.match[0]
    httpResponse = (url) ->
      msg
        .http(url)
        .get() (err, res, body) ->
          if res.statusCode is 301 or res.statusCode is 302
            httpResponse(res.headers.location)
          else if res.statusCode is 200
            if res.headers['content-type'] != 'text/html'
              return

            handler = new HtmlParser.DefaultHandler()
            parser  = new HtmlParser.Parser handler
            parser.parseComplete body

            # abort if soupselect runs out of stack space
            try
              results = (Select handler.dom, "head title")
            catch RangeError
              return

            processResult = (elem) ->
                unEntity(elem.children[0].data.replace(/(\r\n|\n|\r)/gm,"").trim())
            if results[0]
              msg.send processResult(results[0])
            else
              results = (Select handler.dom, "title")
              if results[0]
                msg.send processResult(results[0])
          else
            msg.send "Error " + res.statusCode

    httpBitlyResponse = (url) ->
      msg
        .http("http://api.bitly.com/v3/info")<commit_msg>[web.coffee] Check for that content-type _starts-with_ 'text/html'<commit_after># Description:
#   Webutility returns title of urls
#
# Dependencies:
#   "htmlparser": "1.7.6"
#   "soupselect": "0.2.0"
#   "jsdom": "0.2.14"
#
# Configuration:
#   HUBOT_BITLY_USERNAME
#   HUBOT_BITLY_API_KEY
#
# Commands:
#   None
#
# Author:
#   KevinTraver

Select     = require("soupselect").select
HtmlParser = require "htmlparser"
JSDom      = require "jsdom"

# Decode HTML entities
unEntity = (str) ->
    e = JSDom.jsdom().createElement("div")
    e.innerHTML = str
    if e.childNodes.length == 0 then "" else e.childNodes[0].nodeValue

module.exports = (robot) ->
  robot.hear /(http|ftp|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&amp;:/~\+#]*[\w\-\@?^=%&amp;/~\+#])?/i, (msg) ->
    url = msg.match[0]
    httpResponse = (url) ->
      msg
        .http(url)
        .get() (err, res, body) ->
          if res.statusCode is 301 or res.statusCode is 302
            httpResponse(res.headers.location)
          else if res.statusCode is 200
            if res.headers['content-type'].indexOf('text/html') != 0
              return

            handler = new HtmlParser.DefaultHandler()
            parser  = new HtmlParser.Parser handler
            parser.parseComplete body

            # abort if soupselect runs out of stack space
            try
              results = (Select handler.dom, "head title")
            catch RangeError
              return

            processResult = (elem) ->
                unEntity(elem.children[0].data.replace(/(\r\n|\n|\r)/gm,"").trim())
            if results[0]
              msg.send processResult(results[0])
            else
              results = (Select handler.dom, "title")
              if results[0]
                msg.send processResult(results[0])
          else
            msg.send "Error " + res.statusCode

    httpBitlyResponse = (url) ->
      msg
        .http("http://api.bitly.com/v3/info")
{'fraction_non_alphanumeric': 0.15602284527518173, 'fraction_numerical': 0.014797507788161994, 'mean_word_length': 1.968412942989214, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6307526', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1239, 'n_words': 259}