<commit_before>var path = require('path');
var fs = require('fs');
var _ = require ('lodash');
var fileUtils = require('../utils/filepath-utils.js');

var appConfigUtils = require('../utils/module-config-init.js');
var directoriesUtil = require('../tools/directories-utils.js');

var logUtils = require('../utils/log-utils.js');
var log = logUtils.log;
var warn = logUtils.warn;

var DEFAULT_MODE = 'extended';

function readDir(dir, list, options){

	var files = fs.readdirSync(dir);
	var dirs = [];

	//sort file list to prioritize 'dialog.xml' and 'input.xml' (i.e. put at the beginning of the list)
	var re = /^(dialog|input)\.xml$/i
	files.sort(function(f1, f2){
		if(f1 === f2) return 0;
		if(re.test(f1)) return -1;
		if(re.test(f2)) return 1;
		return f1.localeCompare(f2);
	});

	// log('read dir "'+dir+'" -> ', files);//DEBU

	files.forEach(function(p){
		var absPath = path.join(dir, p);
		if(fileUtils.isDirectory(absPath)){
			dirs.push(absPath);
			return false;
		} else if(/\.xml$/i.test(p)) {

			var normalized = fileUtils.normalizePath(absPath);

			var defModel, id, modId;
			if(/^(dialog|input)(DescriptionSCXML)?\.xml$/i.test(p)){// BACKWARDS COMPATIBILITY: do also accept old/deprecated file names ...DescriptionSCXML.xml
				// -> state models for input- or dialog-manager
				defModel = /^(dialog|input)/i.exec(p);
				id = defModel[1].toLowerCase();
				modId = id === 'dialog'? 'mmirf/dialogManager' : 'mmirf/inputManager';
			} else {
				// -> custom state models
				id = opt && opt.id? opt.id : path.basename(p, path.extname(p));
			}

			var opt = options && options[id];
			if(opt && (opt.exclude || opt.file)){
				//-> ignore/exclude this scxml!
				log('scxml-utils.addFromDirectory(): excluding scxml file for '+id+' model at "'+normalized+'"!');//DEBUG
			}<commit_msg>FIX do prioritize state-model setting from options over parsed/found resources<commit_after>var path = require('path');
var fs = require('fs');
var _ = require ('lodash');
var fileUtils = require('../utils/filepath-utils.js');

var appConfigUtils = require('../utils/module-config-init.js');
var directoriesUtil = require('../tools/directories-utils.js');

var logUtils = require('../utils/log-utils.js');
var log = logUtils.log;
var warn = logUtils.warn;

var DEFAULT_MODE = 'extended';

function readDir(dir, list, options){

	var files = fs.readdirSync(dir);
	var dirs = [];

	//sort file list to prioritize 'dialog.xml' and 'input.xml' (i.e. put at the beginning of the list)
	var re = /^(dialog|input)\.xml$/i
	files.sort(function(f1, f2){
		if(f1 === f2) return 0;
		if(re.test(f1)) return -1;
		if(re.test(f2)) return 1;
		return f1.localeCompare(f2);
	});

	// log('read dir "'+dir+'" -> ', files);//DEBU

	files.forEach(function(p){
		var absPath = path.join(dir, p);
		if(fileUtils.isDirectory(absPath)){
			dirs.push(absPath);
			return false;
		} else if(/\.xml$/i.test(p)) {

			var normalized = fileUtils.normalizePath(absPath);

			var defModel, id, modId;
			if(/^(dialog|input)(DescriptionSCXML)?\.xml$/i.test(p)){// BACKWARDS COMPATIBILITY: do also accept old/deprecated file names ...DescriptionSCXML.xml
				// -> state models for input- or dialog-manager
				defModel = /^(dialog|input)/i.exec(p);
				id = defModel[1].toLowerCase();
				modId = id === 'dialog'? 'mmirf/dialogManager' : 'mmirf/inputManager';
			} else {
				// -> custom state models
				id = opt && opt.id? opt.id : path.basename(p, path.extname(p));
			}

			var opt = options && options[id];
			if(opt && (opt.exclude || opt.file)){
				//-> ignore/exclude this scxml!
				//  (if was not exlcuded, but file was specified -> will be added later in addFromOptions(..))
				log('scxml-utils.addFromDirectory(): excluding scxml file for '+id+' model at "'+normalized+'"!');//DEBUG
				return;//////////////////// EARLY EXIT //////////////////
			}
{'fraction_non_alphanumeric': 0.1914672216441207, 'fraction_numerical': 0.006243496357960458, 'mean_word_length': 3.502341920374707, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584213', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1309, 'n_words': 322}