<commit_before>﻿require 'json'
require 'openssl'
require 'open-uri'
require 'rest-client'
require 'pp' if __FILE__ == $PROGRAM_NAME

class WatsonAPIClient

  VERSION = '0.0.7'

  class Alchemy < self; end

  class << self

    private

    def retrieve_doc(doc_urls)
      apis  = {}

      # Watson API Explorer
      host1 = doc_urls[:doc_base1][/^https?:\/\/[^\/]+/]
      open(doc_urls[:doc_base1], Options, &:read).scan(/<a class="swagger-list--item-link" href="\/(.+?)".*?>\s*(.+?)\s*<\/a>/i) do
        begin
          api = {'path'=>doc_urls[:doc_base1] + $1, 'title'=>$2.sub(/\s*\(.+?\)$/,'')}
          open(api['path'], Options, &:read).scan(/url:\s*'(.+?)'/) do
            api['path'] = host1 + $1
          end
          apis[api['title']] = api
        rescue OpenURI::HTTPError
        end
      end

      # Watson Developercloud
      host2 = doc_urls[:doc_base2][/^https?:\/\/[^\/]+/]
      open(doc_urls[:doc_base2], Options, &:read).scan(/<li>\s*<img.+data-src=.+?>\s*<h2><a href="(.+?)".*?>\s*(.+?)\s*<\/a><\/h2>\s*<p>(.+?)<\/p>\s*<\/li>/) do
        api = {'path'=>$1, 'title'=>$2, 'description'=>$3}
        apis[api['title']]['description'] = api['description'] if api['path'] !~ /\.\./ && apis.key?(api['title'])
      end

      apis
    end

    # for Swagger 2.0
    def listings(apis)
      methods = Hash.new {|h,k| h[k] = {}}
      digest  = Hash.new {|h,k| h[k] = {}}
      apis['paths'].each_pair do |path, operations|
        operations.each_pair do |access, operation|
          body, query, min, max = nil, [], [], []
          if operation['parameters']
            (0...operation['parameters'].size).to_a.reverse.each do |index|
              parameter = operation['parameters'][index]
              operation['parameters'][index..index] = apis['parameters'][parameter[parameter.keys.first].split('/').last] if parameter.keys.first == '$ref'
            end
            operation['parameters'].each do |parameter|
              param  = parameter['name']
              body ||= param if parameter['in'] == 'body'
              query << param if parameter['in'] == 'query'
              min   << param if parameter['required']
              max   << param
            end<commit_msg>Followed changes in Conversation API's Swagger document<commit_after>﻿require 'json'
require 'openssl'
require 'open-uri'
require 'rest-client'
require 'pp' if __FILE__ == $PROGRAM_NAME

class WatsonAPIClient

  VERSION = '0.0.7'

  class Alchemy < self; end

  class << self

    private

    def retrieve_doc(doc_urls)
      apis  = {}

      # Watson API Explorer
      host1 = doc_urls[:doc_base1][/^https?:\/\/[^\/]+/]
      open(doc_urls[:doc_base1], Options, &:read).scan(/<a class="swagger-list--item-link" href="\/(.+?)".*?>\s*(.+?)\s*<\/a>/i) do
        begin
          api = {'path'=>doc_urls[:doc_base1] + $1, 'title'=>$2.sub(/\s*\(.+?\)$/,'')}
          open(api['path'], Options, &:read).scan(/url:\s*'(.+?)'/) do
            api['path'] = host1 + $1
          end
          apis[api['title']] = api
        rescue OpenURI::HTTPError
        end
      end

      # Watson Developercloud
      host2 = doc_urls[:doc_base2][/^https?:\/\/[^\/]+/]
      open(doc_urls[:doc_base2], Options, &:read).scan(/<li>\s*<img.+data-src=.+?>\s*<h2><a href="(.+?)".*?>\s*(.+?)\s*<\/a><\/h2>\s*<p>(.+?)<\/p>\s*<\/li>/) do
        api = {'path'=>$1, 'title'=>$2, 'description'=>$3}
        apis[api['title']]['description'] = api['description'] if api['path'] !~ /\.\./ && apis.key?(api['title'])
      end

      apis
    end

    # for Swagger 2.0
    def listings(apis)
      methods = Hash.new {|h,k| h[k] = {}}
      digest  = Hash.new {|h,k| h[k] = {}}
      apis['paths'].each_pair do |path, operations|
        common_parameters = nil
        operations.each_pair do |access, operation|
          if access == 'parameters'
            common_parameters = operation
            next
          end
          body, query, min, max = nil, [], [], []
          if operation['parameters']
            operation['parameters'] += common_parameters if common_parameters
            (0...operation['parameters'].size).to_a.reverse.each do |index|
              parameter = operation['parameters'][index]
              operation['parameters'][index..index] = apis['parameters'][parameter[parameter.keys.first].split('/').last] if parameter.keys.first == '$ref'
            end
            operation['parameters'].each do |parameter|
              param  = parameter['name']
              body ||= param if parameter['in'] == 'body'
              query << param if parameter['in'] == 'query'
              min   << param if parameter['required']
              max   << param
            end
{'fraction_non_alphanumeric': 0.2124176057835424, 'fraction_numerical': 0.009355730384860728, 'mean_word_length': 2.308016877637131, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1746341', 'n_tokens_mistral': 1586, 'n_tokens_neox': 1542, 'n_words': 296}