<commit_before>/* jshint node:true, unused:true */
'use strict';

var url = require('url');

var _ = require('lodash'),
    Q = require('q'),
    log = require('log4js').getLogger('schoolnet'),
    moment = require('moment'),
    rest = require('restler-q');

// This shouldn't be necessary as it should be the default form log4js but it isn't.
log.setLevel('WARN');

var DEFAULTS = {
    limit: 500,
    offset: 0,
};

var MAX_RETRIES = 4;

function SchoolnetApi(config) {
    config = config || {};
    var creds = {
        client_id: config.clientId || config.client_id,
        client_secret: config.clientSecret || config.client_secret,
        grant_type: 'client_credentials',
        scope: config.scope && 'default_tenant_path:' + config.scope
    };
    var baseUrl = url.parse(config.baseUrl || config.url || config.baseURL);
    this.baseURL = baseUrl.resolve('/api/v1/');
    this.tokenUrl = baseUrl.resolve('/api/oauth/token');
    this.token = void(0);
    this.expires = Date.now();
    this.omissions = ['links', 'institutionType'];
    this.oauthCreds = _.omit(creds, _.isEmpty);
}

var AUG = 7;

function schoolYearStart(date) {
    if (date) {
        date = moment(date);
    } else {
        date = moment();
        if (date.month() >= AUG) {
            date = moment(date.year() + '-08-01');
        } else {
            date = moment((date.year() - 1) + '-08-01');
        }
    }
    return date.format('MM-DD-YYYY');
}

function schoolYearEnd(date) {
    if (date) {
        date = moment(date);
    } else {
        date = moment();
        if (date.month() >= AUG) {
            date = moment((date.year() + 1) + '-07-31');
        } else {
            date = moment(date.year() + '-07-31');
        }
    }
    return date.format('MM-DD-YYYY');
}

// var BACKOFF = [3000, 1000, 500];
var BACKOFF = [5 * 60 * 1000, 60 * 1000, 3 * 1000];

function backoff(idx) {
    var timeout = BACKOFF[idx];
    var dfd = Q.defer();
    setTimeout(function() {
        dfd.resolve(timeout);
    }, timeout);
    return dfd.promise;
}

var RETRY_CODES = ['ETIMEDOUT', 'ECONNRESET'];


module.exports = rest.service(SchoolnetApi, {}, {
    accessToken: function(creds) {
        var dfd = Q.defer();
        var self = this;
        var now = Date.now();
        if (self.expires < now) {
            log.debug('Obtain new token.');
            rest.post(self.tokenUrl, {data: creds}).then(function(data) {
                self.token = data.access_token;
                self.expires = now + data.expires_in * 1000;
                dfd.resolve(self.token);
            });
        } else {
            dfd.resolve(self.token);
        }
        return dfd.promise;
    },
    _get: function _get(uri, options, retries) {
        var dfd = Q.defer();
        var self = this;
        var result = self.get(uri, options);
        result.then(function(data) {
            dfd.resolve(data);
        }, function(err) {
            var retryCode = _.indexOf(RETRY_CODES, err.code) !== -1;
            if (!(retryCode && retries)) {
                dfd.reject(err);
                return;
            }
            retries--;
            backoff(retries).then(function() {
                dfd.resolve(self._get(uri, options, retries));
            });
        });
        return dfd.promise;
    },
    apiGet: function(path, options, recursive) {
        var self = this;
        options = _.extend({query: {}}, options || {});
        log.info('apiGet:', {path: path, options: options, recursive: recursive});
        if (recursive === void(0)) {
            if (options.limit || options.offset >= 0) {
                log.debug('limit or offset provided');
                recursive = options.recursive;
                options.query.limit = options.limit || DEFAULTS.limit;
                options.query.offset = options.offset || DEFAULTS.offset;
                delete options.limit;
                delete options.offset;
                delete options.recursive;
            } else {
                log.debug('neither limit nor offset provided');
                recursive = true;
                options.query.limit = DEFAULTS.limit;
                options.query.offset = DEFAULTS.offset;
            }
        }
        return self.accessToken(self.oauthCreds).then(function(token) {
            var opts = _.extend({}, options, {accessToken: token});
            log.info('requesting:', {path: path, opts: opts});
            return self._get(path, opts, MAX_RETRIES).then(function(data) {
                data = data.data || {};
                if (_.isArray(data)) {
                    data = _.map(data, function(obj) {
                        return self.trimObj(obj, self.omissions);
                    });
                } else {
                    data = self.trimObj(data, self.omissions);
                }
                return data;
            });
        }).then(function(data) {
            if (!recursive || !_.isArray(data) || data.length < options.query.limit) {
                return data;
            }
            var limit = options.query.limit;
            var offset = options.query.offset += limit;<commit_msg>Simplify and clarify retry logic.<commit_after>/* jshint node:true, unused:true */
'use strict';

var url = require('url');

var _ = require('lodash'),
    Q = require('q'),
    log = require('log4js').getLogger('schoolnet'),
    moment = require('moment'),
    rest = require('restler-q');

// This shouldn't be necessary as it should be the default form log4js but it isn't.
log.setLevel('WARN');

var DEFAULTS = {
    limit: 500,
    offset: 0,
};

var MAX_ATTEMPTS = 3;

function SchoolnetApi(config) {
    config = config || {};
    var creds = {
        client_id: config.clientId || config.client_id,
        client_secret: config.clientSecret || config.client_secret,
        grant_type: 'client_credentials',
        scope: config.scope && 'default_tenant_path:' + config.scope
    };
    var baseUrl = url.parse(config.baseUrl || config.url || config.baseURL);
    this.baseURL = baseUrl.resolve('/api/v1/');
    this.tokenUrl = baseUrl.resolve('/api/oauth/token');
    this.token = void(0);
    this.expires = Date.now();
    this.omissions = ['links', 'institutionType'];
    this.oauthCreds = _.omit(creds, _.isEmpty);
}

var AUG = 7;

function schoolYearStart(date) {
    if (date) {
        date = moment(date);
    } else {
        date = moment();
        if (date.month() >= AUG) {
            date = moment(date.year() + '-08-01');
        } else {
            date = moment((date.year() - 1) + '-08-01');
        }
    }
    return date.format('MM-DD-YYYY');
}

function schoolYearEnd(date) {
    if (date) {
        date = moment(date);
    } else {
        date = moment();
        if (date.month() >= AUG) {
            date = moment((date.year() + 1) + '-07-31');
        } else {
            date = moment(date.year() + '-07-31');
        }
    }
    return date.format('MM-DD-YYYY');
}

// var BACKOFF = [3000, 1000, 500];
var BACKOFF = [5 * 60 * 1000, 60 * 1000, 3 * 1000];

function backoff(idx) {
    var timeout = BACKOFF[idx];
    var dfd = Q.defer();
    setTimeout(function() {
        dfd.resolve(timeout);
    }, timeout);
    return dfd.promise;
}

var RETRY_CODES = ['ETIMEDOUT', 'ECONNRESET'];


module.exports = rest.service(SchoolnetApi, {}, {
    accessToken: function(creds) {
        var dfd = Q.defer();
        var self = this;
        var now = Date.now();
        if (self.expires < now) {
            log.debug('Obtain new token.');
            rest.post(self.tokenUrl, {data: creds}).then(function(data) {
                self.token = data.access_token;
                self.expires = now + data.expires_in * 1000;
                dfd.resolve(self.token);
            });
        } else {
            dfd.resolve(self.token);
        }
        return dfd.promise;
    },
    _get: function _get(uri, options, attemptsRemaining) {
        var self = this;
        return self.get(uri, options).fail(function(err) {
            var retryCode = _.contains(RETRY_CODES, err.code);
            if (!(retryCode && attemptsRemaining)) {
                return Q.reject(err);
            }
            attemptsRemaining--;
            backoff(attemptsRemaining).then(function() {
                return self._get(uri, options, attemptsRemaining);
            });
        });
    },
    apiGet: function(path, options, recursive) {
        var self = this;
        options = _.extend({query: {}}, options || {});
        log.info('apiGet:', {path: path, options: options, recursive: recursive});
        if (recursive === void(0)) {
            if (options.limit || options.offset >= 0) {
                log.debug('limit or offset provided');
                recursive = options.recursive;
                options.query.limit = options.limit || DEFAULTS.limit;
                options.query.offset = options.offset || DEFAULTS.offset;
                delete options.limit;
                delete options.offset;
                delete options.recursive;
            } else {
                log.debug('neither limit nor offset provided');
                recursive = true;
                options.query.limit = DEFAULTS.limit;
                options.query.offset = DEFAULTS.offset;
            }
        }
        return self.accessToken(self.oauthCreds).then(function(token) {
            var opts = _.extend({}, options, {accessToken: token});
            log.info('requesting:', {path: path, opts: opts});
            return self._get(path, opts, MAX_ATTEMPTS).then(function(data) {
                data = data.data || {};
                if (_.isArray(data)) {
                    data = _.map(data, function(obj) {
                        return self.trimObj(obj, self.omissions);
                    });
                } else {
                    data = self.trimObj(data, self.omissions);
                }
                return data;
            });
        }).then(function(data) {
            if (!recursive || !_.isArray(data) || data.length < options.query.limit) {
                return data;
            }
            var limit = options.query.limit;
            var offset = options.query.offset += limit;
{'fraction_non_alphanumeric': 0.14595757161012807, 'fraction_numerical': 0.012415680907224558, 'mean_word_length': 1.839300582847627, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1253908', 'n_tokens_mistral': 3164, 'n_tokens_neox': 3008, 'n_words': 652}