<commit_before>//-------------------------------------------------------------------------------
// Copyright IBM Corp. 2015
//
// Licensed under the Apache License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//-------------------------------------------------------------------------------

'use strict';

var request = require("request");
var _ = require("lodash");
var async = require("async");
var pipesSDK = require('simple-data-pipe-sdk');
var pipesDb = pipesSDK.pipesDb;
var connectorExt = pipesSDK.connectorExt;
var cloudant = pipesSDK.cloudant;
var moment = require("moment");
var flightAccess = require("./flightAccess");
var weatherAccess = require("./weatherAccess");

var concurrency = 20;	//Async queue concurrency
/**
 * Pipes Connector for flightStats (part of flight predictor app)
 */
function flightStatsConnector( parentDirPath ){
	//Call constructor from super class
	connectorExt.call(this, "flightStats", "Flight Stats", {
		copyToDashDb: false,
		extraRequiredFields: null,
		useOAuth: false,
		useCustomTables: true,
		recreateTargetDb: false	//Preserve target db as we want to accumulate training data
	});
	
	this.getTablePrefix = function(){
		return "flightStats";
	}
	
	this.getCloudantDbName = function(pipe, table){
		return pipe.name + "_" + table.name;
	}
	
	var designFlightMetadataName = '_design/flightMetadata';
	var USAirportViewName = 'US Airports';
	var AllAirportsViewName = "airports";
	var AllAirlinesViewName = "airlines";
	var genViewsForFlightMetadata = function(){
		var manager = new cloudant.views( designFlightMetadataName );
		
		manager.addView(
			AllAirportsViewName,
			JSON.parse("{"+
					"\"map\": \"function(doc){" +
					"if ( doc.metadata_type === '" + AllAirportsViewName + "' && doc.fs != null){" +
					"emit( doc.fs, {'_id': doc._id, 'rev': doc._rev, 'name':doc.name } );" +
					"}" +
					"}\"" +
					"}"
			), 2 //Version
		)
		.addView(
			AllAirlinesViewName,
			JSON.parse("{"+
					"\"map\": \"function(doc){" +
					"if ( doc.metadata_type === '" + AllAirlinesViewName + "'){" +
					"emit( doc._id, {'_id': doc._id, 'rev': doc._rev } );" +
					"}" +
					"}\"" +
					"}"
			), 2 //Version
		)
		.addView(
			USAirportViewName,
			JSON.parse("{"+
				"\"map\": \"function(doc){" +
           			"if ( doc.metadata_type === '" + AllAirportsViewName + "' && doc.countryCode==='US' && doc.fs != null){" +
           				"emit( doc.fs, {'_id': doc._id, 'rev': doc._rev, 'name':doc.name } );" +
           			"}" +
       			"}\""+
			"}"
			), 3 //Version
		);

		return [manager];
	}
	
	var metadataDb = null;
	var loadMetadataDb = function(callback,appId, appKey){
		if ( metadataDb ){
			return callback();
		}
		
		//Check if the flightMetadata database exists
		var dbName = "flight-metadata";
		metadataDb = new cloudant.db(dbName, genViewsForFlightMetadata() );
		metadataDb.on( "cloudant_ready", function(){
			if ( metadataDb.dbCreated ){
				//Load the airlines and airports info
				console.log("Loading airlines and airports info");
				
				if ( !appId || !appKey ){
					return callback("Unable to load flight metadata db: No appId or appKey provided");
				}
				
				async.parallel([flightAccess.loadAirports.bind(metadataDb, appId, appKey), flightAccess.loadAirlines.bind(metadataDb, appId, appKey)],function(err, results){
					if ( err ){
						metadataDb = null;
					}
					
					return callback(err);
				});
			}else{
				return callback();
			}
		});
	
		metadataDb.on("cloudant_error", function(){
			var message = "Fatal error from Cloudant database: unable to initialize " + dbName;
			console.log( message );
			metadataDb = null;
			return callback( message );
		});
	}
	
	var getAirportInfo = function( airportFsCode, callback ){
		loadMetadataDb( function(err){
			if ( err ){
				return callback(err);
			}
			metadataDb.run( function(err, db){
				if ( err ){
					return callback(err);
				}
				db.view('flightMetadata', AllAirportsViewName, {key: airportFsCode, include_docs: true}, function(err, body){
					if ( err || body.rows.length <= 0){
						return callback(err || "Unable to find record for airport " + airportFsCode);
					}
					return callback( null, body.rows[0].doc );
				});
			})
		});
	}
	
	var superConnectDataSource = this.connectDataSource;
	this.connectDataSource = function( req, res, pipeId, url, callback ){
		pipesDb.getPipe( pipeId, function( err, pipe ){
			if ( err ){
				return callback(err);
			}
			loadMetadataDb(function(err){
				if ( err ){
					return callback(err);
				}
				return superConnectDataSource.call(this, req, res, pipeId, url, callback);
			}.bind(this),pipe.appId, pipe.appKey);
		}.bind(this));
	};
	
	/**
	 * getTables: return Array of table objects: {name:'XXXX', labelPlural: 'XXXX}
	 */
	var busiestAirports = ['ATL','LAX','ORD','DFW','JFK','DEN','SFO','CLT','LAS','PHX','IAH','MIA','SEA','EWR','MCO','MSP','DTW',
	                       'BOS','PHL','LGA','FLL','BWI','IAD','MDW','SLC','DCA','HNL','SAN','TPA','PDX'];
	var tenbusiestAirports = ['ATL','LAX','ORD','JFK','SFO','LAS','BOS','PHL','SAN','PDX'];
	this.getTables = function(){
		return [
	        {name : null, labelPlural : 'All Sets'},
	        {name : 'mega_set', labelPlural : 'Mega Set from 30 busiest airports', CLIENT_airportCodes:busiestAirports},
	        {name : 'mega_subset', labelPlural : 'Mega SubSet from 10 busiest airports', CLIENT_airportCodes:tenbusiestAirports},<commit_msg>change training/test ration to 20/10 airports<commit_after>//-------------------------------------------------------------------------------
// Copyright IBM Corp. 2015
//
// Licensed under the Apache License, Version 2.0 (the 'License');
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an 'AS IS' BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//-------------------------------------------------------------------------------

'use strict';

var request = require("request");
var _ = require("lodash");
var async = require("async");
var pipesSDK = require('simple-data-pipe-sdk');
var pipesDb = pipesSDK.pipesDb;
var connectorExt = pipesSDK.connectorExt;
var cloudant = pipesSDK.cloudant;
var moment = require("moment");
var flightAccess = require("./flightAccess");
var weatherAccess = require("./weatherAccess");

var concurrency = 20;	//Async queue concurrency
/**
 * Pipes Connector for flightStats (part of flight predictor app)
 */
function flightStatsConnector( parentDirPath ){
	//Call constructor from super class
	connectorExt.call(this, "flightStats", "Flight Stats", {
		copyToDashDb: false,
		extraRequiredFields: null,
		useOAuth: false,
		useCustomTables: true,
		recreateTargetDb: false	//Preserve target db as we want to accumulate training data
	});
	
	this.getTablePrefix = function(){
		return "flightStats";
	}
	
	this.getCloudantDbName = function(pipe, table){
		return pipe.name + "_" + table.name;
	}
	
	var designFlightMetadataName = '_design/flightMetadata';
	var USAirportViewName = 'US Airports';
	var AllAirportsViewName = "airports";
	var AllAirlinesViewName = "airlines";
	var genViewsForFlightMetadata = function(){
		var manager = new cloudant.views( designFlightMetadataName );
		
		manager.addView(
			AllAirportsViewName,
			JSON.parse("{"+
					"\"map\": \"function(doc){" +
					"if ( doc.metadata_type === '" + AllAirportsViewName + "' && doc.fs != null){" +
					"emit( doc.fs, {'_id': doc._id, 'rev': doc._rev, 'name':doc.name } );" +
					"}" +
					"}\"" +
					"}"
			), 2 //Version
		)
		.addView(
			AllAirlinesViewName,
			JSON.parse("{"+
					"\"map\": \"function(doc){" +
					"if ( doc.metadata_type === '" + AllAirlinesViewName + "'){" +
					"emit( doc._id, {'_id': doc._id, 'rev': doc._rev } );" +
					"}" +
					"}\"" +
					"}"
			), 2 //Version
		)
		.addView(
			USAirportViewName,
			JSON.parse("{"+
				"\"map\": \"function(doc){" +
           			"if ( doc.metadata_type === '" + AllAirportsViewName + "' && doc.countryCode==='US' && doc.fs != null){" +
           				"emit( doc.fs, {'_id': doc._id, 'rev': doc._rev, 'name':doc.name } );" +
           			"}" +
       			"}\""+
			"}"
			), 3 //Version
		);

		return [manager];
	}
	
	var metadataDb = null;
	var loadMetadataDb = function(callback,appId, appKey){
		if ( metadataDb ){
			return callback();
		}
		
		//Check if the flightMetadata database exists
		var dbName = "flight-metadata";
		metadataDb = new cloudant.db(dbName, genViewsForFlightMetadata() );
		metadataDb.on( "cloudant_ready", function(){
			if ( metadataDb.dbCreated ){
				//Load the airlines and airports info
				console.log("Loading airlines and airports info");
				
				if ( !appId || !appKey ){
					return callback("Unable to load flight metadata db: No appId or appKey provided");
				}
				
				async.parallel([flightAccess.loadAirports.bind(metadataDb, appId, appKey), flightAccess.loadAirlines.bind(metadataDb, appId, appKey)],function(err, results){
					if ( err ){
						metadataDb = null;
					}
					
					return callback(err);
				});
			}else{
				return callback();
			}
		});
	
		metadataDb.on("cloudant_error", function(){
			var message = "Fatal error from Cloudant database: unable to initialize " + dbName;
			console.log( message );
			metadataDb = null;
			return callback( message );
		});
	}
	
	var getAirportInfo = function( airportFsCode, callback ){
		loadMetadataDb( function(err){
			if ( err ){
				return callback(err);
			}
			metadataDb.run( function(err, db){
				if ( err ){
					return callback(err);
				}
				db.view('flightMetadata', AllAirportsViewName, {key: airportFsCode, include_docs: true}, function(err, body){
					if ( err || body.rows.length <= 0){
						return callback(err || "Unable to find record for airport " + airportFsCode);
					}
					return callback( null, body.rows[0].doc );
				});
			})
		});
	}
	
	var superConnectDataSource = this.connectDataSource;
	this.connectDataSource = function( req, res, pipeId, url, callback ){
		pipesDb.getPipe( pipeId, function( err, pipe ){
			if ( err ){
				return callback(err);
			}
			loadMetadataDb(function(err){
				if ( err ){
					return callback(err);
				}
				return superConnectDataSource.call(this, req, res, pipeId, url, callback);
			}.bind(this),pipe.appId, pipe.appKey);
		}.bind(this));
	};
	
	/**
	 * getTables: return Array of table objects: {name:'XXXX', labelPlural: 'XXXX}
	 */
	var busiestAirports = ['DFW','DEN','CLT','PHX','IAH','MIA','SEA','EWR','MCO','MSP','DTW',
	                       'LGA','FLL','BWI','IAD','MDW','SLC','DCA','HNL','TPA'];
	var tenbusiestAirports = ['ATL','LAX','ORD','JFK','SFO','LAS','BOS','PHL','SAN','PDX'];
	this.getTables = function(){
		return [
	        {name : null, labelPlural : 'All Sets'},
	        {name : 'mega_set', labelPlural : 'Mega Set from 20 busiest airports', CLIENT_airportCodes:busiestAirports},
	        {name : 'mega_subset', labelPlural : 'Mega SubSet from 10 busiest airports', CLIENT_airportCodes:tenbusiestAirports},
{'fraction_non_alphanumeric': 0.18186457978904388, 'fraction_numerical': 0.003572643756379721, 'mean_word_length': 3.082291666666667, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3248950', 'n_tokens_mistral': 4259, 'n_tokens_neox': 3605, 'n_words': 926}