<commit_before>/*
 * vrunner
 * http://vrest.io
 *
 * Copyright (c) 2015 vREST Team
 * Licensed under the MIT license.
 */

'use strict';

var request = require('request').defaults({jar: true, json: true}),
    zSchemaValidator = require('z-schema'),
    events = require('events'),
    jsonSchemaFiles = require('./lib/schemaFiles'),
    util = require('./lib/util'),
    runner = require('./lib/testRunner'),
    jquery = require('jquery'),
    OAuth1 = require('./lib/oauth-1_0'),
    loggers = ['console','json','xunit'],
    JSONPath = require('./lib/jsonpath'),
    btoa = require('btoa'),
    V_BASE_URL = 'https://vrest.io/',
    RUNNER_LIMIT = 5000,
    EMAIL_REGEX = /^\S+@\S+\.\S+$/,
    ORG_URL_PREFIX = 'i/',
    START_VAR_EXPR = '{{',
    END_VAR_EXPR = '}}',
    TRTC_BATCH = 5,
    MONGO_REGEX = /^[0-9a-fA-F]{24}$/,
    pages = [false],
    options = {
      credentials : {},
      logger : 'console',
      varColMap : {},
      exitOnDone : true,
      pageSize : 100,
      authorizations : {},
      validatorIdCodeMap : {},
      variables : {}
    };

var getInstanceName = function(url){
  var instanceName = null, prefixIndex = url.indexOf(ORG_URL_PREFIX), prefixLength = ORG_URL_PREFIX.length, index;
  if(prefixIndex > -1){
    index = url.indexOf("/", prefixIndex + prefixLength);
    if(index > -1) instanceName = url.substring(prefixIndex + prefixLength, index);
    else instanceName = url.substring(prefixIndex + prefixLength);
  }
  return String(instanceName);
};

var fetchSinglePage = function(url, page, pageSize, cb, next, vrunner){
  if(page===vrunner.totalPages) next();
  else if(Array.isArray(pages[page])) {
    afterFetch(pages[page], cb, function(err){
      if(err) next(err);
      else fetchSinglePage(url, page+1, pageSize, cb, next, vrunner);
    });
    if(pages[page+1] === false) fetchSinglePage(url, page+1, pageSize, cb, next, vrunner);
  } else if(typeof pages[page] === 'string') next(pages[page]);
  else if(pages[page] === false) {
    pages[page] = true;
    request(url + '&pageSize=' + pageSize + '&currentPage=' + page, function(err, res, body){
      if(err || body.error) pages[page] = util.stringify(['Error found while fetching test cases at page '+page+' :', body]);
      else if(!util.isNumber(body.total) || body.total > RUNNER_LIMIT)
        pages[page] = 'More than '+RUNNER_LIMIT+ ' test cases can not be executed in one go.';
      else if(!page){
        oneTimeCache(vrunner,body.output,body.total);
        fetchSinglePage(url, page, pageSize, cb, next, vrunner);
        vrunner.on('new_page', function(npage){
          if(vrunner.pageLoading){
            fetchSinglePage(url,npage,pageSize,cb,next,vrunner);
            vrunner.pageLoading = false;
          }
        });
      } else {
        pages[page] = body.output;
        vrunner.emit('new_page', page);
      }
    });
  } else {
    vrunner.pageLoading = true;
    vrunner.emit('log', 'Fetching page ' + (page+1) + ' (upto ' + pageSize + ' testcases) ...');
  }
};

var afterFetch = function(body, cb, next){
  util.recForEach({ ar : body, ec : cb, finishOnError : true, cb : next });
};

var oneTimeCache = function(vrunner,records,total){
  vrunner.emit("log", "Executing test cases ... (Please wait, it may take some time.)");
  vrunner.totalRecords = total;
  vrunner.totalPages = Math.ceil(total/vrunner.pageSize);
  pages[0] = records;
  for(var z=1;z<vrunner.totalPages;z++){
    pages[z] = false;
  }
};

var fetchAndServe = function(url, pageSize, cb, next, vrunner){
  fetchSinglePage(url, 0, pageSize, cb, next, vrunner);
};

var hasRunPermission = function(instance, project, next){
  request(V_BASE_URL+'user/hasPermission?permission=RUN_TEST_CASES&project='+project+'&instance='+instance,
  function(err,res,body){
    if(err || body.error) next(['Error while checking execute permission  :', err||body], 'VRUN_OVER');
    else if(!body.output) next('Internal permission error.', 'VRUN_OVER');
    else if(body.output.permit !== true) next('NO_PERMISSION_TO_RUN_TESTCASE_IN_PROJECT', 'VRUN_OVER');
    else next(null,body.output.project);
  });
};

var findHelpers = function(vrunner, what, next){
  vrunner.emit('log', 'Finding '+what+'s ...');
  var instanceURL = vrunner.instanceURL,
    projectId = vrunner.projectId;

  request(instanceURL+'/g/'+what+'?&projectId='+projectId,
    function(err,res,body){
      if(err || body.error) next(['Error while fetching '+what+'s :', err||body]);
      else {
        if(!Array.isArray(body.output)) body.output = [];
        next(null,body.output);
      }
  });
};

var createTestRun = function(instanceURL, filterData, next){
  var filters = filterData;
  filters.currentPage = 0;
  filters.pageSize = 100;
  request({ method: 'POST', uri: instanceURL+'/g/testrun',
    body: { name : util.getReadableDate(), projectId : true, filterData : filters } }, function(err,res,body){
      if(err || body.error) next(['Error while creating test run : ',err||body]);
      else next(null,body.output);
  });
};

var getBasicAuthHeader = function(ath){
  var authConfig = ath.authConfig || '', token = authConfig.username + ':' + authConfig.password;
  return 'Basic ' + btoa(token);
};

var getOAuthTwoHeader = function(ath){
  var authConfig = ath.authConfig || {};
  return (authConfig.accessTokenType || 'OAuth') + ' ' + authConfig.accessToken;
};

var getAuthHeader = function(ath){
  var authType = ath.authType,
      authConfig = ath.authConfig || {};
  if(authType === 'basic'){
    return getBasicAuthHeader(ath);
  } else if(authType === 'raw'){
    return authConfig.authHeader || '';
  } else if(authType === 'oauth1.0'){
    return function(tc) {
      var params = util.extractParamters(tc.params);
      new OAuth1(authConfig, tc.method, util.completeURL(tc.url, params.query)).getAuthHeader();
    };
  } else if(authType === 'oauth2.0'){
    return getOAuthTwoHeader(ath);
  }
};

var fireRequest = function(tc, trtc, callback){
  runner({ testcase : tc },function(result){
    var afterWait = function(){
      if(!result || result.err) {
        //console.log(result);
      }
      if(result.runnerCase) trtc.runnerCase = result.runnerCase;
      callback(result);
    };
    trtc.executionTime = new Date().getTime() - trtc.executionTime;
    if(tc.waitFor) setTimeout(afterWait, tc.waitFor*1000);
    else afterWait();
  });
};

var getContentType = function(responseHeaders){
  if(responseHeaders){
    return responseHeaders['Content-Type'] || responseHeaders['content-type'];
  }
  return null;
};

var getResultType = function(response) {
  var rType = 'text';
  var contentType = getContentType(response.headers);
  if(contentType){
    if(contentType.indexOf('json') != -1) {
      rType = 'json';
    } else if(contentType.indexOf('html') != -1) {
      rType = 'html';
    } else if(contentType.indexOf('xml') != -1) {
      rType = 'xml';
    }
  }
  return rType;
};

var getActualResults = function(response) {
  return {
    statusCode : response.statusCode,
    headers : util.mapToArray(response.headers),
    content: util.stringify(response.body, null, true),
    resultType: getResultType(response)
  };
};

var extractVarsFrom = function(tc, result, tcVar) {
  if(result && result.resultType){
    var opts = { startVarExpr : START_VAR_EXPR, endVarExpr : END_VAR_EXPR, prefs : [{$:jquery},''] };
    tc.tcVariables.forEach(function(vr){
      if(vr.name && vr.path && util.isValidPathVar({ meta : opts },vr.path)){
        if(vr.path.indexOf(opts.startVarExpr) === 0 && vr.path.indexOf(opts.endVarExpr) !== -1){
          opts.prefs[1] = result.content;
          opts.varkey = vr.name;
          util.funcVarReplace([vr.path], opts, tcVar);
        } else if(result.resultType === 'json') {
          var jsonData = util.getJsonOrString(result.content), tp;
          if(typeof(jsonData) != 'object') return;
          tcVar[vr.name] = JSONPath(jsonData, vr.path);
          if(tcVar[vr.name] === 'TC_VAR_NOT_RESOLVED') {
            tp = util.searchAndReplaceString(vr.path, tcVar, opts);
            if(tp !== vr.path) tcVar[vr.name] = JSONPath(jsonData, tp);
          }
          if(Array.isArray(tcVar[vr.name]) && tcVar[vr.name].length === 1)
            tcVar[vr.name] = tcVar[vr.name][0]; // TODO : how it was working earlier? jsonpath return array.
        }
      }
    });
  }
  return;
};

var setFinalExpContent = function(er,ar,curVars){
  var toSet = false, opts = { startVarExpr : START_VAR_EXPR, endVarExpr : END_VAR_EXPR, prefs : [{$:jquery},''] };
  if(util.isWithVars(er.content, opts)){
    if(er.resultType === 'json'){
      toSet = true;
      var spcl = opts.startVarExpr + '*' + opts.endVarExpr, isSpcl = (er.content.indexOf('"'+spcl+'"') !== -1),
        exCont = util.getJsonOrString(er.content);
      if(typeof exCont === 'object'){
        util.walkInto(function(valn, key, root){
          if(typeof root === 'object' && root && root.hasOwnProperty(key)){
            var val = root[key], tmpKy = null;
            if(util.isWithVars(key, opts) && key !== spcl){
              tmpKy = util.searchAndReplaceString(key, curVars, opts);
              if(tmpKy !== key){
                val = root[tmpKy] = root[key];
                delete root[key];
              }
            }
            if(typeof val === 'string' && val && val !== spcl){
              if(util.isWithVars(val, opts)){
                var newValue = curVars[val.substring(opts.startVarExpr.length, val.length - opts.endVarExpr.length)];
                root[tmpKy || key] = newValue || util.searchAndReplaceString(val, curVars, opts);
              }
            }
          }
        }, null, exCont);
        if(isSpcl) exCont = util.mergeObjects(exCont, util.getJsonOrString(ar.content), { spcl : spcl });
        er.content = util.stringify(exCont);
      }
    } else {
      er.content = util.searchAndReplaceString(er.content, curVars, opts);
    }
  }
  return toSet;
};<commit_msg>special var<commit_after>/*
 * vrunner
 * http://vrest.io
 *
 * Copyright (c) 2015 vREST Team
 * Licensed under the MIT license.
 */

'use strict';

var request = require('request').defaults({jar: true, json: true}),
    zSchemaValidator = require('z-schema'),
    events = require('events'),
    jsonSchemaFiles = require('./lib/schemaFiles'),
    util = require('./lib/util'),
    runner = require('./lib/testRunner'),
    jquery = require('jquery'),
    OAuth1 = require('./lib/oauth-1_0'),
    loggers = ['console','json','xunit'],
    JSONPath = require('./lib/jsonpath'),
    btoa = require('btoa'),
    V_BASE_URL = 'https://vrest.io/',
    RUNNER_LIMIT = 5000,
    EMAIL_REGEX = /^\S+@\S+\.\S+$/,
    ORG_URL_PREFIX = 'i/',
    START_VAR_EXPR = '{{',
    END_VAR_EXPR = '}}',
    TRTC_BATCH = 5,
    MONGO_REGEX = /^[0-9a-fA-F]{24}$/,
    pages = [false],
    options = {
      credentials : {},
      logger : 'console',
      varColMap : {},
      exitOnDone : true,
      pageSize : 100,
      authorizations : {},
      validatorIdCodeMap : {},
      variables : {}
    };

var getInstanceName = function(url){
  var instanceName = null, prefixIndex = url.indexOf(ORG_URL_PREFIX), prefixLength = ORG_URL_PREFIX.length, index;
  if(prefixIndex > -1){
    index = url.indexOf("/", prefixIndex + prefixLength);
    if(index > -1) instanceName = url.substring(prefixIndex + prefixLength, index);
    else instanceName = url.substring(prefixIndex + prefixLength);
  }
  return String(instanceName);
};

var fetchSinglePage = function(url, page, pageSize, cb, next, vrunner){
  if(page===vrunner.totalPages) next();
  else if(Array.isArray(pages[page])) {
    afterFetch(pages[page], cb, function(err){
      if(err) next(err);
      else fetchSinglePage(url, page+1, pageSize, cb, next, vrunner);
    });
    if(pages[page+1] === false) fetchSinglePage(url, page+1, pageSize, cb, next, vrunner);
  } else if(typeof pages[page] === 'string') next(pages[page]);
  else if(pages[page] === false) {
    pages[page] = true;
    request(url + '&pageSize=' + pageSize + '&currentPage=' + page, function(err, res, body){
      if(err || body.error) pages[page] = util.stringify(['Error found while fetching test cases at page '+page+' :', body]);
      else if(!util.isNumber(body.total) || body.total > RUNNER_LIMIT)
        pages[page] = 'More than '+RUNNER_LIMIT+ ' test cases can not be executed in one go.';
      else if(!page){
        oneTimeCache(vrunner,body.output,body.total);
        fetchSinglePage(url, page, pageSize, cb, next, vrunner);
        vrunner.on('new_page', function(npage){
          if(vrunner.pageLoading){
            fetchSinglePage(url,npage,pageSize,cb,next,vrunner);
            vrunner.pageLoading = false;
          }
        });
      } else {
        pages[page] = body.output;
        vrunner.emit('new_page', page);
      }
    });
  } else {
    vrunner.pageLoading = true;
    vrunner.emit('log', 'Fetching page ' + (page+1) + ' (upto ' + pageSize + ' testcases) ...');
  }
};

var afterFetch = function(body, cb, next){
  util.recForEach({ ar : body, ec : cb, finishOnError : true, cb : next });
};

var oneTimeCache = function(vrunner,records,total){
  vrunner.emit("log", "Executing test cases ... (Please wait, it may take some time.)");
  vrunner.totalRecords = total;
  vrunner.totalPages = Math.ceil(total/vrunner.pageSize);
  pages[0] = records;
  for(var z=1;z<vrunner.totalPages;z++){
    pages[z] = false;
  }
};

var fetchAndServe = function(url, pageSize, cb, next, vrunner){
  fetchSinglePage(url, 0, pageSize, cb, next, vrunner);
};

var hasRunPermission = function(instance, project, next){
  request(V_BASE_URL+'user/hasPermission?permission=RUN_TEST_CASES&project='+project+'&instance='+instance,
  function(err,res,body){
    if(err || body.error) next(['Error while checking execute permission  :', err||body], 'VRUN_OVER');
    else if(!body.output) next('Internal permission error.', 'VRUN_OVER');
    else if(body.output.permit !== true) next('NO_PERMISSION_TO_RUN_TESTCASE_IN_PROJECT', 'VRUN_OVER');
    else next(null,body.output.project);
  });
};

var findHelpers = function(vrunner, what, next){
  vrunner.emit('log', 'Finding '+what+'s ...');
  var instanceURL = vrunner.instanceURL,
    projectId = vrunner.projectId;

  request(instanceURL+'/g/'+what+'?&projectId='+projectId,
    function(err,res,body){
      if(err || body.error) next(['Error while fetching '+what+'s :', err||body]);
      else {
        if(!Array.isArray(body.output)) body.output = [];
        next(null,body.output);
      }
  });
};

var createTestRun = function(instanceURL, filterData, next){
  var filters = filterData;
  filters.currentPage = 0;
  filters.pageSize = 100;
  request({ method: 'POST', uri: instanceURL+'/g/testrun',
    body: { name : util.getReadableDate(), projectId : true, filterData : filters } }, function(err,res,body){
      if(err || body.error) next(['Error while creating test run : ',err||body]);
      else next(null,body.output);
  });
};

var getBasicAuthHeader = function(ath){
  var authConfig = ath.authConfig || '', token = authConfig.username + ':' + authConfig.password;
  return 'Basic ' + btoa(token);
};

var getOAuthTwoHeader = function(ath){
  var authConfig = ath.authConfig || {};
  return (authConfig.accessTokenType || 'OAuth') + ' ' + authConfig.accessToken;
};

var getAuthHeader = function(ath){
  var authType = ath.authType,
      authConfig = ath.authConfig || {};
  if(authType === 'basic'){
    return getBasicAuthHeader(ath);
  } else if(authType === 'raw'){
    return authConfig.authHeader || '';
  } else if(authType === 'oauth1.0'){
    return function(tc) {
      var params = util.extractParamters(tc.params);
      new OAuth1(authConfig, tc.method, util.completeURL(tc.url, params.query)).getAuthHeader();
    };
  } else if(authType === 'oauth2.0'){
    return getOAuthTwoHeader(ath);
  }
};

var fireRequest = function(tc, trtc, callback){
  runner({ testcase : tc },function(result){
    var afterWait = function(){
      if(!result || result.err) {
        //console.log(result);
      }
      if(result.runnerCase) trtc.runnerCase = result.runnerCase;
      callback(result);
    };
    trtc.executionTime = new Date().getTime() - trtc.executionTime;
    if(tc.waitFor) setTimeout(afterWait, tc.waitFor*1000);
    else afterWait();
  });
};

var getContentType = function(responseHeaders){
  if(responseHeaders){
    return responseHeaders['Content-Type'] || responseHeaders['content-type'];
  }
  return null;
};

var getResultType = function(response) {
  var rType = 'text';
  var contentType = getContentType(response.headers);
  if(contentType){
    if(contentType.indexOf('json') != -1) {
      rType = 'json';
    } else if(contentType.indexOf('html') != -1) {
      rType = 'html';
    } else if(contentType.indexOf('xml') != -1) {
      rType = 'xml';
    }
  }
  return rType;
};

var getActualResults = function(response) {
  return {
    statusCode : response.statusCode,
    headers : util.mapToArray(response.headers),
    content: util.stringify(response.body, null, true),
    resultType: getResultType(response)
  };
};

var extractVarsFrom = function(tc, result, tcVar) {
  if(result && result.resultType){
    var opts = { startVarExpr : START_VAR_EXPR, endVarExpr : END_VAR_EXPR, prefs : [{$:jquery},''] };
    tc.tcVariables.forEach(function(vr){
      if(vr.name && vr.path && util.isValidPathVar({ meta : opts },vr.path)){
        if(vr.path.indexOf(opts.startVarExpr) === 0 && vr.path.indexOf(opts.endVarExpr) !== -1){
          opts.prefs[1] = result.content;
          opts.varkey = vr.name;
          util.funcVarReplace([vr.path], opts, tcVar);
        } else if(result.resultType === 'json') {
          var jsonData = util.getJsonOrString(result.content), tp;
          if(typeof(jsonData) != 'object') return;
          tcVar[vr.name] = JSONPath(jsonData, vr.path);
          if(tcVar[vr.name] === 'TC_VAR_NOT_RESOLVED') {
            tp = util.searchAndReplaceString(vr.path, tcVar, opts);
            if(tp !== vr.path) tcVar[vr.name] = JSONPath(jsonData, tp);
          }
          if(Array.isArray(tcVar[vr.name]) && tcVar[vr.name].length === 1)
            tcVar[vr.name] = tcVar[vr.name][0]; // TODO : how it was working earlier? jsonpath return array.
        }
      }
    });
  }
  return;
};

var setFinalExpContent = function(er,ar,curVars){
  var toSet = false, opts = { startVarExpr : START_VAR_EXPR, endVarExpr : END_VAR_EXPR, prefs : [{$:jquery},''] };
  if(util.isWithVars(er.content, opts)){
    var spcl = START_VAR_EXPR + '*' + END_VAR_EXPR, spclFl = '"'+spcl+'"';
    toSet = true;
    if(er.content === spclFl) {
      er.content = ar.content;
    } else if(er.resultType === 'json'){
      var spclIn = er.content.indexOf(spclFl), isSpcl = (spclIn !== -1), exCont = this.getJsonOrString(er.content);
      if(typeof exCont === 'object'){
        util.walkInto(function(valn, key, root){
          if(typeof root === 'object' && root && root.hasOwnProperty(key)){
            var val = root[key], tmpKy = null;
            if(util.isWithVars(key, opts) && key !== spcl){
              tmpKy = util.searchAndReplaceString(key, curVars, opts);
              if(tmpKy !== key){
                val = root[tmpKy] = root[key];
                delete root[key];
              }
            }
            if(typeof val === 'string' && val && val !== spcl){
              if(util.isWithVars(val, opts)){
                var newValue = curVars[val.substring(opts.startVarExpr.length, val.length - opts.endVarExpr.length)];
                root[tmpKy || key] = newValue || util.searchAndReplaceString(val, curVars, opts);
              }
            }
          }
        }, null, exCont);
        if(isSpcl) exCont = util.mergeObjects(exCont, util.getJsonOrString(ar.content), { spcl : spcl });
        er.content = util.stringify(exCont);
      }
    } else {
      er.content = util.searchAndReplaceString(er.content, curVars, opts);
    }
  }
  return toSet;
};
{'fraction_non_alphanumeric': 0.16636600521146522, 'fraction_numerical': 0.0050110242533573865, 'mean_word_length': 3.2201311059420594, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '840331', 'n_tokens_mistral': 6608, 'n_tokens_neox': 6583, 'n_words': 1311}