<commit_before>'use strict';

const _ = require('lodash');
const domain = require('domain');
const kue = require('kue');
const mustache = require('mustache');
const should = require('should');
const util = require('util');

const queue = kue.createQueue();


/**
 * BatchEvaluator
 * ==============
 *
 * This class implements a Batch Evaluator, i.e. a worker meant to evaluate
 * submissions tested against a known set of testcases.
 *
 * The evaluation of the testcases is delegated to specialized workers
 * (see BatchTestcaseEvaluator), the results being aggregated by this class
 * according to some function.
 *
 *
 * Evaluation Structure
 * --------------------
 *
 * We expect testcases for a task requiring Batch correction to be partitioned
 * into M subtasks. Subtask i (1 <= i <= M) contains N_i testcases.
 *
 * Let T_{i,j} be j-th testcase of the i-th subtask, and let S_{i,j} be its
 * score. The final score for the i-th subtask is calculated as
 *
 *     X_i = intraSubtaskAggregation(S_{i,1}, S_{i,2}, ..., S_{i,N_i}).
 *
 * In turn, the scores of the subtasks are aggregated into a final score for the
 * submission, namely
 *
 *     Score = interSubtaskAggregation(X_1, X_2, ..., X_M).
 *
 *
 * Evaluator Configuration
 * -----------------------
 *
 * These are the fields currently expected by BatchEvaluator:
 *
 * +-------------------------+-------------------------------------+-----------+
 * | Field name              | Description                         | Mandatory |
 * +-------------------------+-------------------------------------+-----------+
 * | tcInputFileUriSchema    | A string containing two `%d` wild-  |     Y     |
 * |                         | wildcards. This represent the URI   |           |
 * |                         | schema for the testcase input       |           |
 * |                         | files. The first wildcard will be   |           |
 * |                         | substituted by the subtask index,   |           |
 * |                         | the second one by the testcase      |           |
 * |                         | index.                              |           |
 * +-------------------------+-------------------------------------+-----------+
 * | tcOutputFileUriSchema   | A string containing two `%d` wild-  |     Y     |
 * |                         | wildcards. This represent the URI   |           |
 * |                         | schema for the testcase output      |           |
 * |                         | files. The first wildcard will be   |           |
 * |                         | substituted by the subtask index,   |           |
 * |                         | the second one by the testcase      |           |
 * |                         | index.                              |           |
 * +-------------------------+-------------------------------------+-----------+
 * | timeLimit               | A real number indicating how many   |     Y     |
 * |                         | seconds the submissionFile may be   |           |
 * |                         | executed for.                       |           |
 * +-------------------------+-------------------------------------+-----------+
 * | memoryLimit             | A real number indicating how many   |     Y     |
 * |                         | MiB are available for the execution |           |
 * |                         | of submissionFile.                  |           |
 * +-------------------------+-------------------------------------+-----------+
 * | evaluationStructure     | A list of subtask objects. The      |     Y     |
 * |                         | i-th of these objects represents    |           |
 * |                         | the i-th subtask.                   |           |
 * +-------------------------+-------------------------------------+-----------+
 * | submissionFileUri       | Forwarded field, see:               |     Y     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | submissionLanguage      | Forwarded field, see:               |     Y     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | checkerSourceUri        | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | checkerLanguage         | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | graderSourceUri         | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | intraSubtaskAggregation | A string representing the aggrega-  |     N     |
 * |                         | tion function inside subtasks (see  |           |
 * |                         | above). Currently, only the         |           |
 * |                         | following functions are accepted:   |           |
 * |                         |   - sum                             |           |
 * |                         |   - max                             |           |
 * |                         |   - min                             |           |
 * |                         | If null or not present, the default |           |
 * |                         | is chosen.                          |           |
 * +-------------------------+-------------------------------------+-----------+
 * | interSubtaskAggregation | A string representing the aggrega-  |     N     |
 * |                         | tion function for subtasks (see     |           |
 * |                         | above). Currently, only the         |           |
 * |                         | following functions are accepted:   |           |
 * |                         |   - sum                             |           |
 * |                         |   - max                             |           |
 * |                         |   - min                             |           |
 * |                         | If null or not present, the default |           |
 * |                         | is chosen.                          |           |
 * +-------------------------+-------------------------------------+-----------+
 *
 * Subtask Configuration
 * ---------------------
 *
 * These are the fields currently expected by each subtask object:
 *
 * +-------------------------+-------------------------------------+-----------+
 * | Field name              | Description                         | Mandatory |
 * +-------------------------+-------------------------------------+-----------+
 * | nTestcases              | An integer number representing how  |     Y     |
 * |                         | many testcases there are.           |           |
 * +-------------------------+-------------------------------------+-----------+
 * | scoreMultiplier         | Multiplier to be applied to the     |     N     |
 * |                         | subtask score (e.g. if the subtask  |           |
 * |                         | has 10 TCs and the scoreMultiplier  |           |
 * |                         | is 1.5, then solving all 10 TCs     |           |
 * |                         | will yield a score of 15 instead of |           |
 * |                         | the normal score of 10).            |           |
 * +-------------------------+-------------------------------------+-----------+
 *
 * Published result
 * ----------------
 *
 * When the evaluation job is finished a score and a message are published.
 * They are published in the form of an object having the following fields:
 * - score: a positive number,
 * - maxScore: the maximum achievable score.
 *
 * @todo Provide a way to specify a baseuri corresponding to phony protocol
 *           shuriken://.
 * @todo Accept timeLimit and memoryLimit as config parameters for the subtasks.
 * @todo Provide a way to configure the connection to Redis (e.g.
 *           authentication).
 * @todo Provide a way to specify a global timeLimit multiplier (via command
 *           line). This is necessary to compensate for different cpu speeds
 *           among the machines running the evaluators, making evaluations
 *           fairer.
 */
class BatchEvaluator {
  /**
   * Receive a link to the Kue Job and the callback to inform the queue manager
   * that the evaluation has finished.
   *
   * @param {!Object} job The current Kue Job.
   * @param {function} doneCallback Callback to inform the queue manager that
   *                       the evaluation has finished.
   * @constructor
   */
  constructor(job, doneCallback) {
    this._kueJob = job;
    this._doneCallback = doneCallback;

    // Parse the configuration for this job (found in job.data()).
    // Step 0. jobConfig must be an Object.
    this._config = job.data;
    should(this._config).be.Object();

    // Step 1. Check that all mandatory fields are there.
    should(this._config).have.properties(['submissionFileUri',
        'tcInputFileUriSchema', 'tcOutputFileUriSchema', 'evaluationStructure',
        'timeLimit', 'memoryLimit']);

    // Step 2. Set all non mandatory fields to the default values.
    this._config.checkerSourceUri =
        _.get(this._config, 'checkerSourceUri', null);
    this._config.intraSubtaskAggregation =
        _.get(this._config, 'intraSubtaskAggregation', 'sum');
    this._config.interSubtaskAggregation =
        _.get(this._config, 'interSubtaskAggregation', 'sum');

    // Step 3. For each field, check values are feasible.
    should(this._config.tcInputFileUriSchema).be.String();
    (this._config.tcInputFileUriSchema.match(/%s/g) || []).length.should.be.aboveOrEqual(2);
    should(this._config.tcOutputFileUriSchema).be.String();
    (this._config.tcOutputFileUriSchema.match(/%s/g) || []).length.should.be.aboveOrEqual(2);

    should(this._config.timeLimit)
        .be.Number()
        .and.not.be.Infinity()
        .and.be.above(0);

    should(this._config.memoryLimit)
        .be.Number()
        .and.not.be.Infinity()
        .and.be.above(0);
<commit_msg>Fix wrong check on the input/output URI schema<commit_after>'use strict';

const _ = require('lodash');
const domain = require('domain');
const kue = require('kue');
const mustache = require('mustache');
const should = require('should');
const util = require('util');

const queue = kue.createQueue();


/**
 * BatchEvaluator
 * ==============
 *
 * This class implements a Batch Evaluator, i.e. a worker meant to evaluate
 * submissions tested against a known set of testcases.
 *
 * The evaluation of the testcases is delegated to specialized workers
 * (see BatchTestcaseEvaluator), the results being aggregated by this class
 * according to some function.
 *
 *
 * Evaluation Structure
 * --------------------
 *
 * We expect testcases for a task requiring Batch correction to be partitioned
 * into M subtasks. Subtask i (1 <= i <= M) contains N_i testcases.
 *
 * Let T_{i,j} be j-th testcase of the i-th subtask, and let S_{i,j} be its
 * score. The final score for the i-th subtask is calculated as
 *
 *     X_i = intraSubtaskAggregation(S_{i,1}, S_{i,2}, ..., S_{i,N_i}).
 *
 * In turn, the scores of the subtasks are aggregated into a final score for the
 * submission, namely
 *
 *     Score = interSubtaskAggregation(X_1, X_2, ..., X_M).
 *
 *
 * Evaluator Configuration
 * -----------------------
 *
 * These are the fields currently expected by BatchEvaluator:
 *
 * +-------------------------+-------------------------------------+-----------+
 * | Field name              | Description                         | Mandatory |
 * +-------------------------+-------------------------------------+-----------+
 * | tcInputFileUriSchema    | A string containing two `%d` wild-  |     Y     |
 * |                         | wildcards. This represent the URI   |           |
 * |                         | schema for the testcase input       |           |
 * |                         | files. The first wildcard will be   |           |
 * |                         | substituted by the subtask index,   |           |
 * |                         | the second one by the testcase      |           |
 * |                         | index.                              |           |
 * +-------------------------+-------------------------------------+-----------+
 * | tcOutputFileUriSchema   | A string containing two `%d` wild-  |     Y     |
 * |                         | wildcards. This represent the URI   |           |
 * |                         | schema for the testcase output      |           |
 * |                         | files. The first wildcard will be   |           |
 * |                         | substituted by the subtask index,   |           |
 * |                         | the second one by the testcase      |           |
 * |                         | index.                              |           |
 * +-------------------------+-------------------------------------+-----------+
 * | timeLimit               | A real number indicating how many   |     Y     |
 * |                         | seconds the submissionFile may be   |           |
 * |                         | executed for.                       |           |
 * +-------------------------+-------------------------------------+-----------+
 * | memoryLimit             | A real number indicating how many   |     Y     |
 * |                         | MiB are available for the execution |           |
 * |                         | of submissionFile.                  |           |
 * +-------------------------+-------------------------------------+-----------+
 * | evaluationStructure     | A list of subtask objects. The      |     Y     |
 * |                         | i-th of these objects represents    |           |
 * |                         | the i-th subtask.                   |           |
 * +-------------------------+-------------------------------------+-----------+
 * | submissionFileUri       | Forwarded field, see:               |     Y     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | submissionLanguage      | Forwarded field, see:               |     Y     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | checkerSourceUri        | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | checkerLanguage         | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | graderSourceUri         | Forwarded field, see:               |     N     |
 * |                         |   BatchTestcaseEvaluator            |           |
 * +-------------------------+-------------------------------------+-----------+
 * | intraSubtaskAggregation | A string representing the aggrega-  |     N     |
 * |                         | tion function inside subtasks (see  |           |
 * |                         | above). Currently, only the         |           |
 * |                         | following functions are accepted:   |           |
 * |                         |   - sum                             |           |
 * |                         |   - max                             |           |
 * |                         |   - min                             |           |
 * |                         | If null or not present, the default |           |
 * |                         | is chosen.                          |           |
 * +-------------------------+-------------------------------------+-----------+
 * | interSubtaskAggregation | A string representing the aggrega-  |     N     |
 * |                         | tion function for subtasks (see     |           |
 * |                         | above). Currently, only the         |           |
 * |                         | following functions are accepted:   |           |
 * |                         |   - sum                             |           |
 * |                         |   - max                             |           |
 * |                         |   - min                             |           |
 * |                         | If null or not present, the default |           |
 * |                         | is chosen.                          |           |
 * +-------------------------+-------------------------------------+-----------+
 *
 * Subtask Configuration
 * ---------------------
 *
 * These are the fields currently expected by each subtask object:
 *
 * +-------------------------+-------------------------------------+-----------+
 * | Field name              | Description                         | Mandatory |
 * +-------------------------+-------------------------------------+-----------+
 * | nTestcases              | An integer number representing how  |     Y     |
 * |                         | many testcases there are.           |           |
 * +-------------------------+-------------------------------------+-----------+
 * | scoreMultiplier         | Multiplier to be applied to the     |     N     |
 * |                         | subtask score (e.g. if the subtask  |           |
 * |                         | has 10 TCs and the scoreMultiplier  |           |
 * |                         | is 1.5, then solving all 10 TCs     |           |
 * |                         | will yield a score of 15 instead of |           |
 * |                         | the normal score of 10).            |           |
 * +-------------------------+-------------------------------------+-----------+
 *
 * Published result
 * ----------------
 *
 * When the evaluation job is finished a score and a message are published.
 * They are published in the form of an object having the following fields:
 * - score: a positive number,
 * - maxScore: the maximum achievable score.
 *
 * @todo Provide a way to specify a baseuri corresponding to phony protocol
 *           shuriken://.
 * @todo Accept timeLimit and memoryLimit as config parameters for the subtasks.
 * @todo Provide a way to configure the connection to Redis (e.g.
 *           authentication).
 * @todo Provide a way to specify a global timeLimit multiplier (via command
 *           line). This is necessary to compensate for different cpu speeds
 *           among the machines running the evaluators, making evaluations
 *           fairer.
 */
class BatchEvaluator {
  /**
   * Receive a link to the Kue Job and the callback to inform the queue manager
   * that the evaluation has finished.
   *
   * @param {!Object} job The current Kue Job.
   * @param {function} doneCallback Callback to inform the queue manager that
   *                       the evaluation has finished.
   * @constructor
   */
  constructor(job, doneCallback) {
    this._kueJob = job;
    this._doneCallback = doneCallback;

    // Parse the configuration for this job (found in job.data()).
    // Step 0. jobConfig must be an Object.
    this._config = job.data;
    should(this._config).be.Object();

    // Step 1. Check that all mandatory fields are there.
    should(this._config).have.properties(['submissionFileUri',
        'tcInputFileUriSchema', 'tcOutputFileUriSchema', 'evaluationStructure',
        'timeLimit', 'memoryLimit']);

    // Step 2. Set all non mandatory fields to the default values.
    this._config.checkerSourceUri =
        _.get(this._config, 'checkerSourceUri', null);
    this._config.intraSubtaskAggregation =
        _.get(this._config, 'intraSubtaskAggregation', 'sum');
    this._config.interSubtaskAggregation =
        _.get(this._config, 'interSubtaskAggregation', 'sum');

    // Step 3. For each field, check values are feasible.
    should(this._config.tcInputFileUriSchema).be.String();
    (this._config.tcInputFileUriSchema.match(/%d/g) || []).length.should.equal(2);
    should(this._config.tcOutputFileUriSchema).be.String();
    (this._config.tcOutputFileUriSchema.match(/%d/g) || []).length.should.equal(2);

    should(this._config.timeLimit)
        .be.Number()
        .and.not.be.Infinity()
        .and.be.above(0);

    should(this._config.memoryLimit)
        .be.Number()
        .and.not.be.Infinity()
        .and.be.above(0);

{'fraction_non_alphanumeric': 0.21831220021845468, 'fraction_numerical': 0.002184546706558389, 'mean_word_length': 0.9176759857936435, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 68, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2749019', 'n_tokens_mistral': 4798, 'n_tokens_neox': 4565, 'n_words': 1345}