<commit_before>var fs = require('fs');
var path = require('path');
var yaml = require('yaml-front-matter');
var _ = require('lodash');

/*
 * export
 */
module.exports = function(collectionRoot, playlistFolder){
  // return playlists found in collectionRoot/playlistFolder
  var playlistRoot = path.join(collectionRoot, playlistFolder);

  if (!fs.existsSync(playlistRoot)) {
    return [];
  }

  // create a list of playlists
  var playlistFiles = fs.readdirSync(playlistRoot);
  playlistFiles = _.map(playlistFiles, withPath, {root: playlistRoot});
  playlistFiles = _.filter(playlistFiles, isPlaylist);

  var playlists = _.map(playlistFiles, getPlaylist, {root: collectionRoot});

  return playlists;
};


/*
 * helper functions
 */
function withPath(filename) {
  // add path to filename
  return path.join(this.root, filename);
}

function isPlaylist(filename) {
  // if not directory and ends with .txt
  return filename.match(/\.txt$/) &&
         !fs.statSync(filename).isDirectory();
}

function playlistId(name){
  // replace chars in playlist-name, so that it can be used as id or class
  var id = name.replace(/ /g, '_');
  id = id.replace(/[\,\.\-\?]/g, '');
  return id;
}

function getLink(root, filename) {
  var link = filename.replace('.md', '.html');
  link = path.relative(root, link);
  link = path.join(path.basename(root), link);
  return link;
}

function getPlaylist(filename) {
  var lessonFiles = fs.readFileSync(filename, {encoding: 'utf8'}).split('\n');
  lessonFiles = _.compact(lessonFiles); // omit empty lines
  lessonFiles = _.map(lessonFiles, withPath, {root: this.root});

  var playlist = {};
  playlist.name = path.basename(filename).replace('.txt', '').replace(/_/g, ' ');
  playlist.id = playlistId(playlist.name);
  playlist.lessons = _.map(lessonFiles, getFrontMatter, {root: this.root});

  return playlist;
}

function getFrontMatter(filename){
  // return front matter of file, its filename and link
  var content = fs.readFileSync(filename);

  var frontMatter = yaml.loadFront(content);
  frontMatter = _.omit(frontMatter, '__content');
  frontMatter.filename = filename;
  frontMatter.link = getLink(this.root, filename);
<commit_msg>closes #71<commit_after>var fs = require('fs');
var path = require('path');
var yaml = require('yaml-front-matter');
var _ = require('lodash');

/*
 * export
 */
module.exports = function(collectionRoot, playlistFolder){
  // return playlists found in collectionRoot/playlistFolder
  var playlistRoot = path.join(collectionRoot, playlistFolder);

  if (!fs.existsSync(playlistRoot)) {
    return [];
  }

  // create a list of playlists
  var playlistFiles = fs.readdirSync(playlistRoot);
  playlistFiles = _.map(playlistFiles, withPath, {root: playlistRoot});
  playlistFiles = _.filter(playlistFiles, isPlaylist);

  var playlists = _.map(playlistFiles, getPlaylist, {root: collectionRoot});

  return playlists;
};


/*
 * helper functions
 */
function withPath(filename) {
  // add path to filename
  return path.join(this.root, filename);
}

function isPlaylist(filename) {
  // if not directory and ends with .txt
  return filename.match(/\.txt$/) &&
         !fs.statSync(filename).isDirectory();
}

function playlistId(name){
  // replace chars in playlist-name, so that it can be used as id or class
  var id = name.replace(/ /g, '_');
  id = id.replace(/[\,\.\-\?]/g, '');
  return id;
}

function getLink(root, filename) {
  var link = filename.replace('.md', '.html');
  link = path.relative(root, link);
  link = path.join(path.basename(root), link);
  return link;
}

function getPlaylist(filename) {
  var lessonFiles = fs.readFileSync(filename, {encoding: 'utf8'})
                      .replace(/\r/g, '')
                      .split('\n');
  lessonFiles = _.compact(lessonFiles); // omit empty lines
  lessonFiles = _.map(lessonFiles, withPath, {root: this.root});

  var playlist = {};
  playlist.name = path.basename(filename).replace('.txt', '').replace(/_/g, ' ');
  playlist.id = playlistId(playlist.name);
  playlist.lessons = _.map(lessonFiles, getFrontMatter, {root: this.root});

  return playlist;
}

function getFrontMatter(filename){
  // return front matter of file, its filename and link
  var content = fs.readFileSync(filename);

  var frontMatter = yaml.loadFront(content);
  frontMatter = _.omit(frontMatter, '__content');
  frontMatter.filename = filename;
  frontMatter.link = getLink(this.root, filename);

{'fraction_non_alphanumeric': 0.1507900677200903, 'fraction_numerical': 0.0009029345372460496, 'mean_word_length': 4.029511918274688, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3039758', 'n_tokens_mistral': 1433, 'n_tokens_neox': 1403, 'n_words': 341}