<commit_before>#!/usr/bin/env node

require('dotenv').config({silent: true});

var program = require('commander'),
    request = require('request-promise'),
    throat = require('throat'),
    fs = require('fs'),
    path = require('path'),
    dateFormat = require('dateformat'),
    urljoin = require('url-join'),
    chalk = require('chalk'),
    curl = require('curl-cmd'),
    qs = require('qs'),
    mask = require('json-mask'),
    CronJob = require('cron').CronJob;

program
    .description('Export data from the management API')
    .option("-D, --dimension <dimension>", "The traffic dimension to collect. Valid dimensions: apiproducts, devs, apps, apis(default)", /^(apiproducts|devs|apps|apis)$/i, 'apis')
    .option("-d, --days <days>", "The number of days to collect in retrograde. 3 by default", 3, parseInt)

    // added back
    // it was removed because of bug in windows bigger than 24 hours in stats api, which can be fixed by giving large limit. e.g. 1'000'000 records
    .option("-w, --window <window>", 'The number days to collect per request.  For example, you can collect a month ' +
                                     'of traffic one day at a time, 3 days at a time or \'N\' days at a time.  Using this ' +
                                     'results in shorter-lived AX requests and can be used to reduce timeouts from AX API. 3 by default', 3, parseInt)
    .option("-m, --apigee_mgmt_api_uri <apigee_mgmt_api_uri>", "URL to management API")
    .option("-u, --apigee_mgmt_api_email <apigee_mgmt_api_email>", "Email registered on the Management API. See .env file to setup default value")
    .option("-p, --apigee_mgmt_api_password <apigee_mgmt_api_password>", "Password associated to the management api email account")
    .option("-i, --include_orgs <items>", 'Include orgs from this list (comma separated)', list)
    .option("-x, --exclude_orgs <items>", 'Exclude orgs from this list (comma separated)', list)
    .option("-n, --include_envs <items>", 'Include environments from this list (comma separated)',list)
    .option("-e, --exclude_envs <items>", 'Exclude envs from this list (comma separated)', list)
    .option("-o, --output <path>", "Path and filename to save output")
    .option("-s, --time_range_start <time_range_start>", 'Time range start for querying traffic stats e.g. "03/01/2016 00:00"')
    .option("-z, --time_range_end <time_range_end>", 'Time range end for querying traffic stats e.g. "04/01/2016 24:00"')
    .option("-t, --time_unit <time_unit>", 'Time unit for traffic stats. Default week. Default units by hour. Valid time units: second, minute, hour, day, week', /^(second|minute|hour|day|week)$/i, 'hour')
    .option("-U, --apigee_analytics_api_url <apigee_analytics_api_url>", "apigee analytics URL to submit the traffic output. Send a request to <EMAIL> to request credentials.", "https://nucleus-api-test.apigee.com/v1/apigee-analytics-cli-api/traffic/orgs") //"http://localhost:10010/traffic/orgs")
    .option("-S, --standard_output", "output through the terminal (stdout).")
    .option("-c, --apigee_analytics_client_id <apigee_analytics_client_id>", "cliend_id used to authenticate against apigee analytics api")
    .option("-r, --apigee_analytics_secret <apigee_analytics_secret>", "secret used to authenticate againts apigee analytics api")
    .option("-R, --include_curl_commands", "include sample cURL commands for debugging")
    .option("-v, --verbose","make the operation more talkative")
    //.option("-N, --run_as_standalone_cronjob","indicate to run as a standalone job in background")
    //.option("-E, --cronjob_schedule <cronjob schedule>","cronjob schedule. Default schedule as \"30 2 * * *\", everyday at 2:30 am. Requires --run_as_standalone_flag","30 2 * * *")
    .parse(process.argv);

// this is required to enable debug as a flag instead of as an environment variable
if( program.verbose ) process.env.DEBUG = '*';
var debug = require('debug')('apigee-nucleus');

if( !program.run_as_standalone_cronjob ) extract_traffic(program);
else {
  debug('executing job with program.cronjob_schedule', program.cronjob_schedule);
  var job = new CronJob({<commit_msg>Changed the URL to prod<commit_after>#!/usr/bin/env node

require('dotenv').config({silent: true});

var program = require('commander'),
    request = require('request-promise'),
    throat = require('throat'),
    fs = require('fs'),
    path = require('path'),
    dateFormat = require('dateformat'),
    urljoin = require('url-join'),
    chalk = require('chalk'),
    curl = require('curl-cmd'),
    qs = require('qs'),
    mask = require('json-mask'),
    CronJob = require('cron').CronJob;

program
    .description('Export data from the management API')
    .option("-D, --dimension <dimension>", "The traffic dimension to collect. Valid dimensions: apiproducts, devs, apps, apis(default)", /^(apiproducts|devs|apps|apis)$/i, 'apis')
    .option("-d, --days <days>", "The number of days to collect in retrograde. 3 by default", 3, parseInt)

    // added back
    // it was removed because of bug in windows bigger than 24 hours in stats api, which can be fixed by giving large limit. e.g. 1'000'000 records
    .option("-w, --window <window>", 'The number days to collect per request.  For example, you can collect a month ' +
                                     'of traffic one day at a time, 3 days at a time or \'N\' days at a time.  Using this ' +
                                     'results in shorter-lived AX requests and can be used to reduce timeouts from AX API. 3 by default', 3, parseInt)
    .option("-m, --apigee_mgmt_api_uri <apigee_mgmt_api_uri>", "URL to management API")
    .option("-u, --apigee_mgmt_api_email <apigee_mgmt_api_email>", "Email registered on the Management API. See .env file to setup default value")
    .option("-p, --apigee_mgmt_api_password <apigee_mgmt_api_password>", "Password associated to the management api email account")
    .option("-i, --include_orgs <items>", 'Include orgs from this list (comma separated)', list)
    .option("-x, --exclude_orgs <items>", 'Exclude orgs from this list (comma separated)', list)
    .option("-n, --include_envs <items>", 'Include environments from this list (comma separated)',list)
    .option("-e, --exclude_envs <items>", 'Exclude envs from this list (comma separated)', list)
    .option("-o, --output <path>", "Path and filename to save output")
    .option("-s, --time_range_start <time_range_start>", 'Time range start for querying traffic stats e.g. "03/01/2016 00:00"')
    .option("-z, --time_range_end <time_range_end>", 'Time range end for querying traffic stats e.g. "04/01/2016 24:00"')
    .option("-t, --time_unit <time_unit>", 'Time unit for traffic stats. Default week. Default units by hour. Valid time units: second, minute, hour, day, week', /^(second|minute|hour|day|week)$/i, 'hour')
    .option("-U, --apigee_analytics_api_url <apigee_analytics_api_url>", "apigee analytics URL to submit the traffic output. Send a request to <EMAIL> to request credentials.", "https://nucleus-api-prod.apigee.com/v1/apigee-analytics-cli-api/traffic/orgs") //"http://localhost:10010/traffic/orgs")
    .option("-S, --standard_output", "output through the terminal (stdout).")
    .option("-c, --apigee_analytics_client_id <apigee_analytics_client_id>", "cliend_id used to authenticate against apigee analytics api")
    .option("-r, --apigee_analytics_secret <apigee_analytics_secret>", "secret used to authenticate againts apigee analytics api")
    .option("-R, --include_curl_commands", "include sample cURL commands for debugging")
    .option("-v, --verbose","make the operation more talkative")
    //.option("-N, --run_as_standalone_cronjob","indicate to run as a standalone job in background")
    //.option("-E, --cronjob_schedule <cronjob schedule>","cronjob schedule. Default schedule as \"30 2 * * *\", everyday at 2:30 am. Requires --run_as_standalone_flag","30 2 * * *")
    .parse(process.argv);

// this is required to enable debug as a flag instead of as an environment variable
if( program.verbose ) process.env.DEBUG = '*';
var debug = require('debug')('apigee-nucleus');

if( !program.run_as_standalone_cronjob ) extract_traffic(program);
else {
  debug('executing job with program.cronjob_schedule', program.cronjob_schedule);
  var job = new CronJob({
{'fraction_non_alphanumeric': 0.1455984648596786, 'fraction_numerical': 0.012712880786759414, 'mean_word_length': 3.873758036236119, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4111851', 'n_tokens_mistral': 2667, 'n_tokens_neox': 2582, 'n_words': 833}