<commit_before>//
// Copyright (c) 2011 Mashery, Inc.
// Copyright (c) 2013-2014 InterMine and <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// 'Software'), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//

// This version has all its redis dependencies stripped.

//
// Module dependencies
//
var express     = require('express'),
    util        = require('util'),
    fs          = require('fs'),
    query       = require('querystring'),
    url         = require('url'),
    http        = require('http'),
    https       = require('https'),
    winston     = require('winston'),
    cmw         = require('coffee-middleware'),
    crypto      = require('crypto'),
    stylus      = require('stylus'),
    nib         = require('nib'),
    EventEmitter = require('events').EventEmitter,
    oauth       = require('./lib/oauth'),
    config      = {},
    redis, RedisStore;

try {
  redis       = require('redis'),
  RedisStore  = require('connect-redis')(express);
} catch (e) {
  // ignore.
}

// Load basic app Configuration
try {
  (function () {
    var configJSON = fs.readFileSync(__dirname + "/config.json");
    config = JSON.parse(configJSON.toString());
  })();
} catch(e) {
  console.error("File config.json not found or is invalid.  Try: `cp config.json.sample config.json`");
  process.exit(1);
}

var CAN_HAVE_BODY = function (method) {
  return ['POST','PUT'].indexOf(method) >= 0;
};

// Init logger.
var logger = new winston.Logger({
  transports: [
    new winston.transports.Console({
      colorize: true,
      level: 'info'
    }),
    new winston.transports.File({
      filename: "iodocs.log",
      timestamp: true,
      level: 'debug'
    })
  ]
});

// Init Redis connection (used for persisting OAuth connection details).
var defaultDB = '0';
var db;

// Read config from ENV, if provided.
if (process.env.REDISTOGO_URL) {
  (function () {
    var rtg = url.parse(process.env.REDISTOGO_URL);
    if (!config.redis) config.redis = {};
    config.redis.host = rtg.hostname;
    config.redis.port = rtg.port;
    config.redis.password = rtg.<PASSWORD>.split(":")[1];
  })();
}

if (redis) {
  if (config.redis && config.redis.port) {
    db = redis.createClient(config.redis.port, config.redis.host);
    db.auth(config.redis.password);
  }
} else if (config.redis && config.redis.port) {
  logger.warn("Redis connection configured, but not available. Install library with 'npm install redis'")
}

if (!db) {
  (function () {
    db = new EventEmitter();
    RedisStore = function MockRedisStore () {};

    // Stub out all db methods that we call.
    db.set = mockMethod;
    db.expire = mockMethod;
    db.mget = mockMethod;
    db.mset = mockMethod;

    function mockMethod () {
      logger.warn("Redis command called, but redis is not installed.");
    }
  })();
}

db.on("error", logger.error.bind(logger, '[REDIS] %s'));

//
// Load API Configs
//
var apisConfig = {};
var imAPIs = {};

function initAppConfig () {

  fs.readFile('public/data/apiconfig.json', 'utf-8', function(err, data) {
    var apiName, parsed, slug;
    if (err) throw err;

    parsed = JSON.parse(data);
    for (slug in parsed) {
      apisConfig[slug] = parsed[slug];
    }

    logger.debug('API CONFIG: %j', apisConfig)

    for (apiName in apisConfig) {
      fetchServiceListing(apiName);
    }

    function fetchServiceListing (name) {
      var req
        , api = apisConfig[name]
        , serviceListingURI = api.protocol + "://" + api.baseURL + api.publicPath;

      if (!api.intermine) return;

      logger.debug("Fetching service listing from " + serviceListingURI);

      var doget = (api.protocol === 'https') ? https.get : http.get;
      req = doget(serviceListingURI, function responseHandler (res) {
          var body = "";
          res.on('data', function(chunk) { body += chunk; });
          res.on('error', handleError);
          res.on('end', function done () {
              logger.log('debug', "Retrieved service listing for %s", name);
              try {
                if (api.excludeRequiresAuthn === true) {
                  imAPIs[name] = JSON.parse(body, reviver);
                } else {
                  imAPIs[name] = JSON.parse(body);
                }
              } catch (e) {
                var msg = "Error parsing service listing for " + name + ": " + e;
                handleError(new Error(msg));
              }
          });
      });
      req.on('error', handleError);
      function  handleError (e) {
        logger.log('warn', 'Could not fetch service listing for %s', name, e);
        delete apisConfig[name];
      }
    }

    /* exclude endpoints that require authentication
       set 'excludeRequiresAuthn': 'true' in apiconfig.json */
    function reviver(k, v) {
      if (k === 'methods' && v[0]['RequiresAuthentication'] === 'true') {
          return undefined;
      }
      return v;
    }
  });
}

// Init config now.
initAppConfig();

// Refresh config every configured interval, or every hour.
setInterval(initAppConfig, (config.refreshInterval || 60 * 60) * 1000);

var app = module.exports = express();
<commit_msg>improve logic used to filter/remove unwanted methods in api docs json<commit_after>//
// Copyright (c) 2011 Mashery, Inc.
// Copyright (c) 2013-2014 InterMine and <NAME>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// 'Software'), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//

// This version has all its redis dependencies stripped.

//
// Module dependencies
//
var express     = require('express'),
    util        = require('util'),
    fs          = require('fs'),
    query       = require('querystring'),
    url         = require('url'),
    http        = require('http'),
    https       = require('https'),
    winston     = require('winston'),
    cmw         = require('coffee-middleware'),
    crypto      = require('crypto'),
    stylus      = require('stylus'),
    nib         = require('nib'),
    EventEmitter = require('events').EventEmitter,
    oauth       = require('./lib/oauth'),
    config      = {},
    redis, RedisStore;

try {
  redis       = require('redis'),
  RedisStore  = require('connect-redis')(express);
} catch (e) {
  // ignore.
}

// Load basic app Configuration
try {
  (function () {
    var configJSON = fs.readFileSync(__dirname + "/config.json");
    config = JSON.parse(configJSON.toString());
  })();
} catch(e) {
  console.error("File config.json not found or is invalid.  Try: `cp config.json.sample config.json`");
  process.exit(1);
}

var CAN_HAVE_BODY = function (method) {
  return ['POST','PUT'].indexOf(method) >= 0;
};

// Init logger.
var logger = new winston.Logger({
  transports: [
    new winston.transports.Console({
      colorize: true,
      level: 'info'
    }),
    new winston.transports.File({
      filename: "iodocs.log",
      timestamp: true,
      level: 'debug'
    })
  ]
});

// Init Redis connection (used for persisting OAuth connection details).
var defaultDB = '0';
var db;

// Read config from ENV, if provided.
if (process.env.REDISTOGO_URL) {
  (function () {
    var rtg = url.parse(process.env.REDISTOGO_URL);
    if (!config.redis) config.redis = {};
    config.redis.host = rtg.hostname;
    config.redis.port = rtg.port;
    config.redis.password = rtg.auth.split(":")[1];
  })();
}

if (redis) {
  if (config.redis && config.redis.port) {
    db = redis.createClient(config.redis.port, config.redis.host);
    db.auth(config.redis.password);
  }
} else if (config.redis && config.redis.port) {
  logger.warn("Redis connection configured, but not available. Install library with 'npm install redis'")
}

if (!db) {
  (function () {
    db = new EventEmitter();
    RedisStore = function MockRedisStore () {};

    // Stub out all db methods that we call.
    db.set = mockMethod;
    db.expire = mockMethod;
    db.mget = mockMethod;
    db.mset = mockMethod;

    function mockMethod () {
      logger.warn("Redis command called, but redis is not installed.");
    }
  })();
}

db.on("error", logger.error.bind(logger, '[REDIS] %s'));

//
// Load API Configs
//
var apisConfig = {};
var imAPIs = {};

function initAppConfig () {

  fs.readFile('public/data/apiconfig.json', 'utf-8', function(err, data) {
    var apiName, parsed, slug;
    if (err) throw err;

    parsed = JSON.parse(data);
    for (slug in parsed) {
      apisConfig[slug] = parsed[slug];
    }

    logger.debug('API CONFIG: %j', apisConfig)

    for (apiName in apisConfig) {
      fetchServiceListing(apiName);
    }

    function fetchServiceListing (name) {
      var req
        , api = apisConfig[name]
        , serviceListingURI = api.protocol + "://" + api.baseURL + api.publicPath;

      if (!api.intermine) return;

      logger.debug("Fetching service listing from " + serviceListingURI);

      var doget = (api.protocol === 'https') ? https.get : http.get;
      req = doget(serviceListingURI, function responseHandler (res) {
          var body = "";
          res.on('data', function(chunk) { body += chunk; });
          res.on('error', handleError);
          res.on('end', function done () {
              logger.log('debug', "Retrieved service listing for %s", name);
              try {
                imAPIs[name] = JSON.parse(body);
                if (api.excludeRequiresAuthn === true) {
                  imAPIs[name] = excludeRequiresAuthn(imAPIs[name]);
                }
              } catch (e) {
                var msg = "Error parsing service listing for " + name + ": " + e;
                handleError(new Error(msg));
              }
          });
      });
      req.on('error', handleError);
      function  handleError (e) {
        logger.log('warn', 'Could not fetch service listing for %s', name, e);
        delete apisConfig[name];
      }
    }

    /* set 'excludeRequiresAuthn': 'true' in apiconfig.json
       - exclude endpoints that require authentication
       - disable login capabilities on mine-docs page */
    function excludeRequiresAuthn(imAPI) {
      var imAPIflt = {};
      for (var k1 in imAPI) {
        imAPIflt[k1] = [];
        if (k1 === 'endpoints') {
          for(var i = 0; i < imAPI[k1].length; i++) {
            var e = {}, m = [];
            for (var k2 in imAPI[k1][i]) {
              if (k2 === 'methods') {
                for (var j = 0; j < imAPI[k1][i][k2].length; j++) {
                  if (imAPI[k1][i][k2][j]['RequiresAuthentication'] === 'false') {
                    m.push(imAPI[k1][i][k2][j]);
                  }
                }
                e[k2] = m;
              } else {
                e[k2] = imAPI[k1][i][k2];
              }
            }
            if (m.length !== 0) {
              imAPIflt[k1].push(e);
            }
          }
        } else if (k1 !== 'auth') {
          imAPIflt[k1].push(imAPI[k1]);
        }
      }
      return imAPIflt;
    }
  });
}

// Init config now.
initAppConfig();

// Refresh config every configured interval, or every hour.
setInterval(initAppConfig, (config.refreshInterval || 60 * 60) * 1000);

var app = module.exports = express();

{'fraction_non_alphanumeric': 0.13811645414042095, 'fraction_numerical': 0.005761253648793978, 'mean_word_length': 2.479155531801176, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1471159', 'n_tokens_mistral': 4019, 'n_tokens_neox': 3887, 'n_words': 1149}