<commit_before>
'use strict';

console.log('Launching…');

// Pseudo-constants:
var PREFIX = 'stubs/'
,   SUFFIX = '.sh'
,   STEPS = [
      'retrieve-resources'
    , 'third-party-checker'
    , 'specberus'
    , 'link-checker'
    , 'parse-metadata'
    , 'publish'
    ]
;

var Promise = require('promise');

var meta = require('./package.json')
,   express = require('express')
,   ejs = require('ejs')
,   spawn = require('child_process').spawn
,   app = express()
,   port = 3000
,   requests = {}
;

app.use(express.compress());
app.use(express.bodyParser({}));

app.set('view engine', 'ejs');
app.engine('.html', ejs.renderFile);

if (process.env.NODE_ENV === 'production') {
  app.set('views', __dirname + '/dist/views');
  app.use(express.static(__dirname + '/dist/assets'));
} else {
  app.set('views', __dirname + '/views');
  app.use(express.static(__dirname + '/assets'));
}

// Index Page
app.get('/', function(request, response, next) {
  response.render('index.html');
});

// API methods

app.get('/api/version', function(req, res) {
  res.send(meta.name +
    ' version ' + meta.version +
    ' running on ' + process.platform +
    ' and listening on port ' + port +
    '. The server time is ' + new Date().toLocaleTimeString() + '.');
});

app.get('/api/status', function(req, res) {
  var result
  ,   url = req.query ? req.query.url : null
  ,   entry
  ;
  if (url) {
    if (requests[url]) {
      res.json({ 'request': requests[url] });
    }
    else {
      res.send(500, { error: 'Request of URL ' + url + ' does not exist.' });
    }
  }
  else {
    res.json({ 'requests': requests });
  }
});

app.post('/api/request', function(req, res) {
  var url = req.body ? req.body.url : null
  ,   decision = req.body ? req.body.decision : null
  ;
  if (!url || !decision) {
    res.send(500, {error: 'Missing parameters {url, decision}.'});
  }
  else {
    if (requests[url]) {
      res.send('Spec at ' + url + ' is yet pending validation OR has been already submitted. Check “/api/status”.');
    }
    else {
      requests[url] = {'url': url, 'decision': decision, 'jobs': {}};
      orchestrate(requests[url]).then(function () {
        console.log('Spec at ' + url + ' (decision: ' + decision + ') has FINISHED.');
      }, function (err) {
        console.log('Spec at ' + url + ' (decision: ' + decision + ') has FAILED.');
      });
      res.send(200, 'Spec at ' + url + ' (decision: ' + decision + ') added to the queue.');
    }
  }
});

function dumpLine(data) {
  console.log(data);
}


function retrieve(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/retrieve-resources.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve(url);
      }
      else {
        reject(new Error("retrieve-resources step failed with code " + innerCode));
      }
    });
  });
}

function specberus(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/specberus.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve(url);
      }
      else {
        reject(new Error("specberus step failed with code " + innerCode));
      }
    });
  });
}

function publish(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/publish.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve(url);
      }
      else {
        reject(new Error("publish step failed with code " + innerCode));
      }
    });
  });
}

function Job () {
  if (typeof this !== 'object') throw new TypeError('Jobs must be constructed via new');

  this.status = '';
  this.errors = [];
  this.time = '';
}

function orchestrate(spec) {
  spec.jobs['retrieve-resources'] = new Job();
  spec.jobs['specberus'] = new Job();
  spec.jobs['publish'] = new Job();

  spec.jobs['retrieve-resources'].status = 'pending';

  return retrieve(spec.url).then(
    function (url) {
      spec.jobs['retrieve-resources'].status = 'ok';
      spec.jobs['retrieve-resources'].time = new Date().toLocaleTimeString('en-GB');

      spec.jobs['specberus'].status = 'pending';
      return specberus(url).then(
        function (url) {
          spec.jobs['specberus'].status = 'ok';
          spec.jobs['specberus'].time = new Date().toLocaleTimeString('en-GB');

          spec.jobs['publish'].status = 'pending';
          return publish(url).then(
            function (url) {
              spec.jobs['publish'].status = 'ok';
              spec.jobs['publish'].time = new Date().toLocaleTimeString('en-GB');
              return Promise.resolve("finished");
            },
            function (err) {
              spec.jobs['publish'].status = 'failure';
              spec.jobs['publish'].errors.push(err.toString());
              return Promise.reject(err);
            }
          );
        },
        function (err) {
          spec.jobs['specberus'].status = 'failure';
          spec.jobs['specberus'].errors.push(err.toString());
          return Promise.reject(err);
        }
      );
    },
    function (err) {
      spec.jobs['retrieve-resources'].status = 'failure';
      spec.jobs['retrieve-resources'].errors.push(err.toString());
      return Promise.reject(err);
    }
  ).catch(function (err) {
    return Promise.reject(new Error('Orchestrator has failed.'));
  });
}<commit_msg>Add metadata step and set arguments<commit_after>
'use strict';

console.log('Launching…');

// Pseudo-constants:
var PREFIX = 'stubs/'
,   SUFFIX = '.sh'
,   STEPS = [
      'retrieve-resources'
    , 'third-party-checker'
    , 'specberus'
    , 'link-checker'
    , 'parse-metadata'
    , 'publish'
    ]
;

var Promise = require('promise');

var meta = require('./package.json')
,   express = require('express')
,   ejs = require('ejs')
,   spawn = require('child_process').spawn
,   app = express()
,   port = 3000
,   requests = {}
;

app.use(express.compress());
app.use(express.bodyParser({}));

app.set('view engine', 'ejs');
app.engine('.html', ejs.renderFile);

if (process.env.NODE_ENV === 'production') {
  app.set('views', __dirname + '/dist/views');
  app.use(express.static(__dirname + '/dist/assets'));
} else {
  app.set('views', __dirname + '/views');
  app.use(express.static(__dirname + '/assets'));
}

// Index Page
app.get('/', function(request, response, next) {
  response.render('index.html');
});

// API methods

app.get('/api/version', function(req, res) {
  res.send(meta.name +
    ' version ' + meta.version +
    ' running on ' + process.platform +
    ' and listening on port ' + port +
    '. The server time is ' + new Date().toLocaleTimeString() + '.');
});

app.get('/api/status', function(req, res) {
  var result
  ,   url = req.query ? req.query.url : null
  ,   entry
  ;
  if (url) {
    if (requests[url]) {
      res.json({ 'request': requests[url] });
    }
    else {
      res.send(500, { error: 'Request of URL ' + url + ' does not exist.' });
    }
  }
  else {
    res.json({ 'requests': requests });
  }
});

app.post('/api/request', function(req, res) {
  var url = req.body ? req.body.url : null
  ,   decision = req.body ? req.body.decision : null
  ;
  if (!url || !decision) {
    res.send(500, {error: 'Missing parameters {url, decision}.'});
  }
  else {
    if (requests[url]) {
      res.send('Spec at ' + url + ' is yet pending validation OR has been already submitted. Check “/api/status”.');
    }
    else {
      requests[url] = {'url': url, 'decision': decision, 'jobs': {}};
      orchestrate(requests[url]).then(function () {
        console.log('Spec at ' + url + ' (decision: ' + decision + ') has FINISHED.');
      }, function (err) {
        console.log('Spec at ' + url + ' (decision: ' + decision + ') has FAILED.');
      });
      res.send(200, 'Spec at ' + url + ' (decision: ' + decision + ') added to the queue.');
    }
  }
});

function dumpLine(data) {
  console.log(data);
}


function retrieve(from, to) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/retrieve-resources.sh', [from]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve();
      }
      else {
        reject(new Error("retrieve-resources step failed with code " + innerCode));
      }
    });
  });
}

function specberus(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/specberus.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve();
      }
      else {
        reject(new Error("specberus step failed with code " + innerCode));
      }
    });
  });
}

function parseMetadata(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/parse-metadata.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        var metadata = {};
        resolve();
      }
      else {
        reject(new Error("parse-metadata step failed with code " + innerCode));
      }
    });
  });
}

function publish(url) {
  return new Promise(function (resolve, reject) {
    var job = spawn('stubs/publish.sh', [url]);

    job.on('exit', function (innerCode, innerSignal) {
      if (innerCode === 0) {
        resolve();
      }
      else {
        reject(new Error("publish step failed with code " + innerCode));
      }
    });
  });
}

function Job () {
  if (typeof this !== 'object') throw new TypeError('Jobs must be constructed via new');

  this.status = '';
  this.errors = [];
  this.time = '';
}

function orchestrate(spec) {
  spec.jobs['retrieve-resources'] = new Job();
  spec.jobs['specberus'] = new Job();
  spec.jobs['parse-metadata'] = new Job();
  spec.jobs['publish'] = new Job();

  spec.jobs['retrieve-resources'].status = 'pending';

  var tempLocation = 'foo';
  var finalLocation = 'bar';

  return retrieve(spec.url, tempLocation).then(
    function () {
      spec.jobs['retrieve-resources'].status = 'ok';
      spec.jobs['retrieve-resources'].time = new Date().toLocaleTimeString('en-GB');

      spec.jobs['specberus'].status = 'pending';
      return specberus(tempLocation).then(
        function () {
          spec.jobs['specberus'].status = 'ok';
          spec.jobs['specberus'].time = new Date().toLocaleTimeString('en-GB');

          spec.jobs['parse-metadata'].status = 'pending';
          return parseMetadata(tempLocation).then(
            function (metadata) {
              spec.jobs['parse-metadata'].status = 'ok';
              spec.jobs['parse-metadata'].time = new Date().toLocaleTimeString('en-GB');

              spec.jobs['publish'].status = 'pending';
              return publish(tempLocation, finalLocation).then(
                function () {
                  spec.jobs['publish'].status = 'ok';
                  spec.jobs['publish'].time = new Date().toLocaleTimeString('en-GB');
                  return Promise.resolve("finished");
                },
                function (err) {
                  spec.jobs['publish'].status = 'failure';
                  spec.jobs['publish'].errors.push(err.toString());
                  return Promise.reject(err);
                }
              );
            },
            function (err) {
              spec.jobs['parse-metadata'].status = 'failure';
              spec.jobs['parse-metadata'].errors.push(err.toString());
              return Promise.reject(err);
            }
          );
        },
        function (err) {
          spec.jobs['specberus'].status = 'failure';
          spec.jobs['specberus'].errors.push(err.toString());
          return Promise.reject(err);
        }
      );
    },
    function (err) {
      spec.jobs['retrieve-resources'].status = 'failure';
      spec.jobs['retrieve-resources'].errors.push(err.toString());
      return Promise.reject(err);
    }
  ).catch(function (err) {
    return Promise.reject(new Error('Orchestrator has failed.'));
  });
}
{'fraction_non_alphanumeric': 0.18001329124439275, 'fraction_numerical': 0.002741319156005981, 'mean_word_length': 2.548187444739169, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1278723', 'n_tokens_mistral': 3839, 'n_tokens_neox': 3731, 'n_words': 785}