<commit_before>/**
 * Module that retrives the files to use
 */
const glob = require('glob-promise');
const Immutable = require('immutable');
const FileRecord = require('./FileRecord');
const path = require('path');

const absPath = config =>
                    file => file.set('absolutePath', path.resolve(config.get('contents'), file.get('relativePath')));

/**
 * Function that takes the engine's configuration and returns a
 * list of file records.
 *
 * @param  {Immutable.Map}  config  Engine configuration
 * @return {Promise<Immutable.List<FileRecord>>}
 */
module.exports = config => glob(`${config.get('contents')}/**/*.md`)
                            .then( files => files.map( file => new FileRecord({ relativePath: file }) ) )
                            .then( files => files.map( absPath(config) ))
                            .then( files => Immutable.List(files) );<commit_msg>Process absolute paths instead of relative paths<commit_after>/**
 * Module that retrives the files to use
 */
const glob = require('glob-promise');
const Immutable = require('immutable');
const FileRecord = require('./FileRecord');
const path = require('path');

const relPath = config =>
                    file => file.set('relativePath',
                                     file.get('absolutePath').replace( config.get('contents') + path.sep, '' ) );

/**
 * Function that takes the engine's configuration and returns a
 * list of file records.
 *
 * @param  {Immutable.Map}  config  Engine configuration
 * @return {Promise<Immutable.List<FileRecord>>}
 */
module.exports = config => glob(`${config.get('contents')}/**/*.md`)
                            .then( files => files.map( file => new FileRecord({ absolutePath: file }) ) )
                            .then( files => files.map( relPath(config) ))
                            .then( files => Immutable.List(files) );
{'fraction_non_alphanumeric': 0.15335463258785942, 'fraction_numerical': 0.0, 'mean_word_length': 2.641472868217054, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5239216', 'n_tokens_mistral': 494, 'n_tokens_neox': 488, 'n_words': 137}