<commit_before>//
// exports
//
var forecasts = module.exports = {};


//
// required packages
//
var fs = require('fs');
var winston = require('winston');
var request = require('request');
var moment = require('moment');
var underscore = require('underscore');
var async = require('async');
var xml2js = require('xml2js');
var cheerio = require('cheerio');


// avi danger levels
forecasts.AVI_LEVEL_UNKNOWN = 0;
forecasts.AVI_LEVEL_LOW = 1;
forecasts.AVI_LEVEL_MODERATE = 2;
forecasts.AVI_LEVEL_CONSIDERABLE = 3;
forecasts.AVI_LEVEL_HIGH = 4;
forecasts.AVI_LEVEL_EXTREME = 5;

// time intervals
// NOTE to ensure forecasts get generated, ensure FORECAST_GEN_INTERVAL_SECONDS >> DATA_REQUEST_TIMEOUT_SECONDS
// NOTE the total delay that a client might see from forecast issued to available at client is the sum
// of FORECAST_GEN_INTERVAL_SECONDS + CACHE_MAX_AGE_SECONDS
forecasts.DATA_REQUESTS_IN_PARALLEL = 10;
forecasts.DATA_REQUEST_TIMEOUT_SECONDS = 15;
forecasts.FORECAST_GEN_INTERVAL_SECONDS = 300;
forecasts.CACHE_MAX_AGE_SECONDS = 60;

// filepaths
forecasts.STATIC_FILES_DIR_PATH  = __dirname + '/public';
forecasts.REGIONS_PATH = __dirname + '/public/v1/regions.json';
forecasts.FORECASTS_DATA_PATH = __dirname + '/public/v1/forecasts.json';
forecasts.FORECASTS_DATA_TEMP_PATH = __dirname + '/public/v1/forecasts_TEMP.json';
forecasts.TEMPLATE_FILES_DIR_PATH  = __dirname + '/templates';


// generated forecast tracking
// NOTE these are only kept for the life of a single process, not across process restarts
forecasts.forecastGenerationCount = 0;
forecasts.mostRecentForecasts = [];


forecasts.aggregateForecasts = function(regions) {

    winston.info('aggregateForecasts: initiated');

    var startTime = new Date();
    var forecastsArray = [];
    var invalidCount = 0;

    async.forEachLimit(
        regions,
        forecasts.DATA_REQUESTS_IN_PARALLEL,
        function(region, callback) {
            var regionId = region.regionId;
            forecasts.forecastForRegion(region, function(forecast) {

                // sanity check the forecast
                var valid = forecasts.validateForecast(region, forecast, true);
                if (!valid) {
                    invalidCount++;
                }

                // add the forecast to the forecasts array
                // NOTE the order of forecast generation completion is not deterministic, so they may end up in any
                // order in the array
                forecastsArray.push({regionId:regionId, forecast:forecast});

                winston.info('generated forecast for regionId: ' + regionId + '; count generated so far: ' + forecastsArray.length);

                // continue even on error
                callback(null);
            });
        },
        function() {
            var endTime = new Date();
            var elapsedTime = endTime.getTime() - startTime.getTime();
            if (regions.length !== forecastsArray.length) {
                winston.warn('aggregateForecasts: forecast generation error, expected count of forecasts: ' + regions.length + '; actual: ' + forecastsArray.length);
            }
            winston.info('aggregateForecasts: all forecasts processed, region count: ' + forecastsArray.length + '; elapsed time (ms): ' + elapsedTime);
            if (invalidCount > 0) {
                winston.warn('aggregateForecasts: there were invalid forecasts; invalid forecast count: ' + invalidCount);
            } else {
                winston.info('aggregateForecasts: all forecasts valid');
            }

            // write the forecasts out to a static json file, that can be served by the HTTP server

            // NOTE to ensure atomicity at the filesystem level, we write out to a temporary file, and
            // then move it into place, overwriting the old file

            fs.writeFile(forecasts.FORECASTS_DATA_TEMP_PATH, JSON.stringify(forecastsArray, null, 4), 'utf8',
                function() {
                    fs.rename(forecasts.FORECASTS_DATA_TEMP_PATH, forecasts.FORECASTS_DATA_PATH,
                        function() {
                            winston.info('aggregateForecasts: forecast data file updated; path: ' + forecasts.FORECASTS_DATA_PATH);
                            forecasts.forecastGenerationCount++;
                        }
                    );
                }
            );
        }
    );
};

forecasts.validateForecast = function(region, forecast, validateForCurrentDay) {

    // BUGBUG how do we deal with centers shutting down for the season???

    var validForecast = true;
    var regionId = region.regionId;
    var centerId = region.centerId;
    if (!forecast) {
        // check for null forecast

        // NOTE known exceptions: these regions currently do not provide any danger level ratings
        if (regionId === 'ca_north-rockies' ||
            regionId === 'nac_187' ||
            regionId === 'nac_190' ||
            regionId === 'nac_202' ||
            regionId === 'nac_252' ||
            regionId === 'nac_259_1' ||
            regionId === 'nac_259_2' ||
            regionId === 'nac_259_3' ||
            regionId === 'nac_259_4' ||
            regionId === 'nac_259_5' ||
            regionId === 'nac_183' ||
            regionId === 'nac_189' ||
            regionId === 'nac_197' ||
            regionId === 'nac_193' ||
            regionId === 'nac_186' ||
            regionId === 'nac_194' ||
            regionId === 'nac_191' ||
            regionId === 'nac_121' ||
            regionId === 'nac_203' ||
            regionId === 'nac_282' ||
            regionId === 'nac_261' ||
            centerId === 'caac' ||
            centerId === 'juak' ||
            centerId === 'wcmac' ||
            centerId === 'avalanche-quebec') {
            winston.info('forecast validation: as expected, got null forecast; regionId: ' + regionId);
        } else {
            validForecast = false;<commit_msg>restoring check for ca_north-rockies<commit_after>//
// exports
//
var forecasts = module.exports = {};


//
// required packages
//
var fs = require('fs');
var winston = require('winston');
var request = require('request');
var moment = require('moment');
var underscore = require('underscore');
var async = require('async');
var xml2js = require('xml2js');
var cheerio = require('cheerio');


// avi danger levels
forecasts.AVI_LEVEL_UNKNOWN = 0;
forecasts.AVI_LEVEL_LOW = 1;
forecasts.AVI_LEVEL_MODERATE = 2;
forecasts.AVI_LEVEL_CONSIDERABLE = 3;
forecasts.AVI_LEVEL_HIGH = 4;
forecasts.AVI_LEVEL_EXTREME = 5;

// time intervals
// NOTE to ensure forecasts get generated, ensure FORECAST_GEN_INTERVAL_SECONDS >> DATA_REQUEST_TIMEOUT_SECONDS
// NOTE the total delay that a client might see from forecast issued to available at client is the sum
// of FORECAST_GEN_INTERVAL_SECONDS + CACHE_MAX_AGE_SECONDS
forecasts.DATA_REQUESTS_IN_PARALLEL = 10;
forecasts.DATA_REQUEST_TIMEOUT_SECONDS = 15;
forecasts.FORECAST_GEN_INTERVAL_SECONDS = 300;
forecasts.CACHE_MAX_AGE_SECONDS = 60;

// filepaths
forecasts.STATIC_FILES_DIR_PATH  = __dirname + '/public';
forecasts.REGIONS_PATH = __dirname + '/public/v1/regions.json';
forecasts.FORECASTS_DATA_PATH = __dirname + '/public/v1/forecasts.json';
forecasts.FORECASTS_DATA_TEMP_PATH = __dirname + '/public/v1/forecasts_TEMP.json';
forecasts.TEMPLATE_FILES_DIR_PATH  = __dirname + '/templates';


// generated forecast tracking
// NOTE these are only kept for the life of a single process, not across process restarts
forecasts.forecastGenerationCount = 0;
forecasts.mostRecentForecasts = [];


forecasts.aggregateForecasts = function(regions) {

    winston.info('aggregateForecasts: initiated');

    var startTime = new Date();
    var forecastsArray = [];
    var invalidCount = 0;

    async.forEachLimit(
        regions,
        forecasts.DATA_REQUESTS_IN_PARALLEL,
        function(region, callback) {
            var regionId = region.regionId;
            forecasts.forecastForRegion(region, function(forecast) {

                // sanity check the forecast
                var valid = forecasts.validateForecast(region, forecast, true);
                if (!valid) {
                    invalidCount++;
                }

                // add the forecast to the forecasts array
                // NOTE the order of forecast generation completion is not deterministic, so they may end up in any
                // order in the array
                forecastsArray.push({regionId:regionId, forecast:forecast});

                winston.info('generated forecast for regionId: ' + regionId + '; count generated so far: ' + forecastsArray.length);

                // continue even on error
                callback(null);
            });
        },
        function() {
            var endTime = new Date();
            var elapsedTime = endTime.getTime() - startTime.getTime();
            if (regions.length !== forecastsArray.length) {
                winston.warn('aggregateForecasts: forecast generation error, expected count of forecasts: ' + regions.length + '; actual: ' + forecastsArray.length);
            }
            winston.info('aggregateForecasts: all forecasts processed, region count: ' + forecastsArray.length + '; elapsed time (ms): ' + elapsedTime);
            if (invalidCount > 0) {
                winston.warn('aggregateForecasts: there were invalid forecasts; invalid forecast count: ' + invalidCount);
            } else {
                winston.info('aggregateForecasts: all forecasts valid');
            }

            // write the forecasts out to a static json file, that can be served by the HTTP server

            // NOTE to ensure atomicity at the filesystem level, we write out to a temporary file, and
            // then move it into place, overwriting the old file

            fs.writeFile(forecasts.FORECASTS_DATA_TEMP_PATH, JSON.stringify(forecastsArray, null, 4), 'utf8',
                function() {
                    fs.rename(forecasts.FORECASTS_DATA_TEMP_PATH, forecasts.FORECASTS_DATA_PATH,
                        function() {
                            winston.info('aggregateForecasts: forecast data file updated; path: ' + forecasts.FORECASTS_DATA_PATH);
                            forecasts.forecastGenerationCount++;
                        }
                    );
                }
            );
        }
    );
};

forecasts.validateForecast = function(region, forecast, validateForCurrentDay) {

    // BUGBUG how do we deal with centers shutting down for the season???

    var validForecast = true;
    var regionId = region.regionId;
    var centerId = region.centerId;
    if (!forecast) {
        // check for null forecast

        // NOTE known exceptions: these regions currently do not provide any danger level ratings
        if (regionId === 'nac_187' ||
            regionId === 'nac_190' ||
            regionId === 'nac_202' ||
            regionId === 'nac_252' ||
            regionId === 'nac_259_1' ||
            regionId === 'nac_259_2' ||
            regionId === 'nac_259_3' ||
            regionId === 'nac_259_4' ||
            regionId === 'nac_259_5' ||
            regionId === 'nac_183' ||
            regionId === 'nac_189' ||
            regionId === 'nac_197' ||
            regionId === 'nac_193' ||
            regionId === 'nac_186' ||
            regionId === 'nac_194' ||
            regionId === 'nac_191' ||
            regionId === 'nac_121' ||
            regionId === 'nac_203' ||
            regionId === 'nac_282' ||
            regionId === 'nac_261' ||
            centerId === 'caac' ||
            centerId === 'juak' ||
            centerId === 'wcmac' ||
            centerId === 'avalanche-quebec') {
            winston.info('forecast validation: as expected, got null forecast; regionId: ' + regionId);
        } else {
            validForecast = false;
{'fraction_non_alphanumeric': 0.107040592892033, 'fraction_numerical': 0.015159171298635674, 'mean_word_length': 2.324468085106383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1114572', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3250, 'n_words': 844}