<commit_before>#
# TextMate related stuff.
#

require 'async'
require 'net/http'
require 'set'

class Textmate < PluginBase

  def phrase_to_keywords(phrase)
    phrase.gsub(/\b(\w+)s\b/, '\1').downcase.split(/\W/).to_set
  end

  def parse_toc(text)
    res = []
    text.grep(%r{<li>\s*([\d.]+)\s*<a href='(.*?)'>(.*?)</a>}) do |line|
      section = $1
      url = $2
      title = $3.gsub(%r{</?\w+>}, '')
      res << {
        :title    => "#{section} #{title}",
        :link     => "http://macromates.com/textmate/manual/#{url}",
        :keywords => phrase_to_keywords(title)
      }
    end
    res
  end

  def cmd_doc(irc, line)
    if line.to_s.empty?
      irc.reply 'USAGE: doc <search string or regex>'
    else
      Async.run(irc) do
        Net::HTTP.start('macromates.com') do |http|
          re = http.get('/textmate/manual/',  { 'User-Agent' => 'CyBrowser' })
          if re.code == '200'

            search_keywords = phrase_to_keywords(line)
            entries = parse_toc re.body
            matches = entries.find_all { |m| search_keywords.subset? m[:keywords] }
            if matches.empty?
              irc.reply 'No matches found.'
            else
              ranked = matches.map { |m| m.merge({ :rank => search_keywords.length.to_f / m[:keywords].length }) }
              hit = ranked.max { |a, b| a[:rank] <=> b[:rank] }
              irc.respond "\x02#{hit[:title]}\x0f #{hit[:link]}"
            end

          else
            irc.reply "Documentation site returned an error: #{re.code} #{re.message}"
          end
        end
      end
    end
  end
  help :doc, 'Searches the TextMate manual for the given string or regex.'

  def parse_faq(text)
    res = [ ]
    text.grep(%r{<a name=["'](.*?)["'][^>]*>.*?Keywords:(.*?)</span>}) do |line|
      res << {
        :link     => 'http://macromates.com/wiki/Main/FAQ#' + $1,
        :keywords => phrase_to_keywords($2)
      }
    end
    res
  end

  def cmd_faq(irc, line)
    if line.to_s.empty?
      irc.reply 'USAGE: faq <search keyword(s)>'
    else
      Async.run(irc) do
        Net::HTTP.start('macromates.com') do |http|
          re = http.get('/wiki/Main/FAQ',  { 'User-Agent' => 'CyBrowser' })
          if re.code == '200'

            search_keywords = phrase_to_keywords(line)<commit_msg>Update FAQ (wiki) and documentation URLs.<commit_after>#
# TextMate related stuff.
#

require 'async'
require 'net/http'
require 'set'

class Textmate < PluginBase

  def phrase_to_keywords(phrase)
    phrase.gsub(/\b(\w+)s\b/, '\1').downcase.split(/\W/).to_set
  end

  def parse_toc(text)
    res = []
    text.grep(%r{<li>\s*([\d.]+)\s*<a href='(.*?)'>(.*?)</a>}) do |line|
      section = $1
      url = $2
      title = $3.gsub(%r{</?\w+>}, '')
      res << {
        :title    => "#{section} #{title}",
        :link     => "http://manual.macromates.com/en/#{url}",
        :keywords => phrase_to_keywords(title)
      }
    end
    res
  end

  def cmd_doc(irc, line)
    if line.to_s.empty?
      irc.reply 'USAGE: doc <search string or regex>'
    else
      Async.run(irc) do
        Net::HTTP.start('manual.macromates.com') do |http|
          re = http.get('/en/',  { 'User-Agent' => 'CyBrowser' })
          if re.code == '200'

            search_keywords = phrase_to_keywords(line)
            entries = parse_toc re.body
            matches = entries.find_all { |m| search_keywords.subset? m[:keywords] }
            if matches.empty?
              irc.reply 'No matches found.'
            else
              ranked = matches.map { |m| m.merge({ :rank => search_keywords.length.to_f / m[:keywords].length }) }
              hit = ranked.max { |a, b| a[:rank] <=> b[:rank] }
              irc.respond "\x02#{hit[:title]}\x0f #{hit[:link]}"
            end

          else
            irc.reply "Documentation site returned an error: #{re.code} #{re.message}"
          end
        end
      end
    end
  end
  help :doc, 'Searches the TextMate manual for the given string or regex.'

  def parse_faq(text)
    res = [ ]
    text.grep(%r{<a name=["'](.*?)["'][^>]*>.*?Keywords:(.*?)</span>}) do |line|
      res << {
        :link     => 'http://wiki.macromates.com/Main/FAQ#' + $1,
        :keywords => phrase_to_keywords($2)
      }
    end
    res
  end

  def cmd_faq(irc, line)
    if line.to_s.empty?
      irc.reply 'USAGE: faq <search keyword(s)>'
    else
      Async.run(irc) do
        Net::HTTP.start('wiki.macromates.com') do |http|
          re = http.get('/Main/FAQ',  { 'User-Agent' => 'CyBrowser' })
          if re.code == '200'

            search_keywords = phrase_to_keywords(line)
{'fraction_non_alphanumeric': 0.17813852813852815, 'fraction_numerical': 0.006493506493506494, 'mean_word_length': 2.0622929092113984, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100958', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1602, 'n_words': 340}