<commit_before>var async = require('async');
var Step = require('step');
var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var db = require('./db');
var Migration = require('./migration');

module.exports = function MigrationRunner(options){
  var self = this;

  var migrations = [];
  loadMigrationFiles();

  db.init(options.database_url);

  self.run = function(callback){
    var current_index, migrate_to_index;
    Step(
      function(){
        ensureMigrationTableExists(this);
      },
      function fetchCompletedMigrations(err){
        db.query("SELECT * FROM "+options.migrations_table+" ORDER BY run_on", this)
      },
      function determineMigrationsToRun(err, result){
        if (err) return callback(err);
        if (!result || !result.rows) return callback(new Error('Unable to fetch migrations from '+options.migrations_table+' table'));

        // TODO: handle merging of migrations that are out of order better
        // show a warning and add an option to apply missing migraitons

        // go through each existing migration and verify the file exists
        // store the index so we know where we are starting
        for (var i=0; i<result.rows.length; i++){
          // console.log( '"' + result.rows[i].name + '" -- "' + migrations[i].name + '"' );
          if ( result.rows[i].name !== migrations[i].name ){
            return callback(new Error('Migration file missing for already ran migration: '+result.rows[i].name))
          }
          if (options.file && options.file == migrations[i].name){
            migrate_to_index = i;
          }
        }
        current_index = result.rows.length;

        if (options.count){
          // infinity is set in the bin file
          if (options.count == Infinity){
            migrate_to_index = migrations.length;
          } else {
            // users can also specify the number of migrations to move up and down
            migrate_to_index = current_index + options.count * (options.direction=='up' ?1:-1);
          }
        }

        // 0 is a clean slate (no migrations run)
        if (migrate_to_index < 0) return callback(new Error('cannot migrate past the starting position'));
        if (migrate_to_index > migrations.length ) return callback(new Error('Cannot migrate past the last migration defined'));
        if (current_index == migrate_to_index) {
          console.log('No migrations to run!');
          return callback();
        }

        var to_run;
        if (options.direction == 'up'){
          to_run = migrations.slice(current_index, migrate_to_index);
        } else {
          to_run = migrations.slice(migrate_to_index, current_index);
          to_run = to_run.reverse();
        }

        // TODO: add some fancy colors to logging
        var current_state_name = current_index == 0 ? 'CLEAN SLATE' : migrations[current_index-1].name;
        var new_state_name = migrate_to_index == 0 ? 'CLEAN SLATE' : migrations[migrate_to_index-1].name;
        console.log('> Migrating from '+current_state_name + ' >to> ' + new_state_name );

        this(null, to_run);
      },
      function runMigrations(err, migrations_to_run){
        if (err) return callback(err);
        async.eachSeries(migrations_to_run, function(m, next){
          if (options.direction == 'up') m.applyUp(next);
          else m.applyDown(next);<commit_msg>Changed the migration files selection method to allow migrating older files, if they are not migrated yet<commit_after>var async = require('async');
var Step = require('step');
var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var db = require('./db');
var Migration = require('./migration');

module.exports = function MigrationRunner(options){
  var self = this;

  var migrations = [];
  loadMigrationFiles();

  db.init(options.database_url);

  self.run = function(callback){
    var current_index, migrate_to_index;
    Step(
      function(){
        ensureMigrationTableExists(this);
      },
      function fetchCompletedMigrations(err){
        db.query("SELECT * FROM "+options.migrations_table+" ORDER BY run_on", this)
      },
	    function determineMigrationsToRun(err, result){
		    if (err) return callback(err);
		    if (!result || !result.rows) return callback(new Error('Unable to fetch migrations from pgmigrations table'));

		    //store done migration names
		    var runNames = result.rows.map(function(row){
			    return row.name;
		    });

		    var doneMigrations = [];
		    var runMigrations = [];

				//filter out undone and done migrations from list of files
		    for (var i=0; i<migrations.length; i++){
			    if ( runNames.indexOf(migrations[i].name) < 0 ){

				    //if specific migration file is requested
				    if (options.file && options.file == migrations[i].name){
					    runMigrations = [migrations[i]];
					    break;
				    }
				    runMigrations.push(migrations[i]);
			    }else{
				    doneMigrations.push(migrations[i])
			    }

		    }

				//final selection will go here
		    var to_run;

		    //down gets by default one migration at the time, if not set otherwise
		    if(options.direction=='down'){
			    to_run = doneMigrations.slice(-1);
			  //up gets all undone migrations by default
		    }else{
			    to_run = runMigrations;
		    }

		    //if specific count of migrations is requested
		    if (options.count){
			    // infinity is set in the bin file
			    if (options.count !== Infinity){
				    if(options.direction=='up'){
					    to_run = runMigrations.slice(0, options.count);
				    }else{
					    to_run = doneMigrations.slice(options.count * -1);
				    }
			    }
		    }

		    if (!to_run.length) {
			    console.log('No migrations to run!');
			    return callback();
		    }

		    // TODO: add some fancy colors to logging
		    console.log('> Migrating files:' );
		    for(var i in to_run){
			    console.log('> - ' + to_run[i].name );
		    }

		    this(null, to_run);
	    },
      function runMigrations(err, migrations_to_run){
        if (err) return callback(err);
        async.eachSeries(migrations_to_run, function(m, next){
          if (options.direction == 'up') m.applyUp(next);
          else m.applyDown(next);
{'fraction_non_alphanumeric': 0.1221227621483376, 'fraction_numerical': 0.002237851662404092, 'mean_word_length': 2.5652421652421653, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793275', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1787, 'n_words': 499}