<commit_before>'use strict'
import fs from 'fs';
import path from 'path';

import config from '../../config';
import {enforce, fail} from '../helpers';

/*
 *  data structure of FileSet
 *    a Map of families
 *      (no DSC IMG ...) , still important i.e. for unstarred deletion
 *      each Family consisting out of individual Members (Files)
 */
 class FileSet {

	constructor(dirs) {

		enforce( Array.isArray(dirs), 'not an array' );
		this._families = new Map();
		console.log('FileSet constructed');

		this._families.set('DSC1234',42);
		this._families.set('_IMG_5678','banana');


		// STEP 1   parse picasa ini to key-value array to obtained starred
		 /*
		 * STEP 2
		 *    iterate images and collect families and singles
		 *    (non-family-matchable) singles become important for i.e. unstarred
		 *    deletions...)
		 */

		/* STEP 3  perform action
		 *    actual delete/recycle or simulation of it
		 */



		// parsing each dir:
		dirs.forEach(function (dir) {

			enforce(fs.existsSync(dir), `no directory or file ${dir}`);
			enforce(fs.statSync(dir).isDirectory(), 'single File – not handled yet');

			let files = fs.readdirSync(dir);

			files
				// filtering for valid-extension files
				.filter((filepath) => {
					const p = path.parse(filepath);

					enforce(p.name.length > 0, 'sanity: no empty filenames');
					// skipping hidden
					if (p.name[0] === '.') return false;
					// remove leading dot on ext (always, unless extensionless)
					if (p.ext.length > 0) {
						enforce(p.ext[0] === '.', 'sanity');
						p.ext = p.ext.substr(1);
					}

					console.log("dir " + p.dir + "name " + p.name + "  p.ext: " + p.ext);
					// filter for supported filetypes
					return config.extensions.includes(p.ext.toLowerCase());
				})
				.map((filepath) => {
					console.log(filepath)
				});
		});

		console.log('dumping FileSet Map:');

		for(var [key, value] of this._families) {
			console.log(key + ' = ' + value);
		}
<commit_msg>tween<commit_after>'use strict'
import fs from 'fs';
import path from 'path';

import config from '../../config';
import {enforce, fail} from '../helpers';

/*
 *  data structure of FileSet
 *    a Map of families
 *      (no DSC IMG ...) , still important i.e. for unstarred deletion
 *      each Family consisting out of individual Members (Files)
 */
 class FileSet {

	constructor(dirs) {

		enforce( Array.isArray(dirs), 'not an array' );
		this._families = new Map();
		console.log('FileSet constructed');

		// TEMPTEMP
		this._families.set('DSC1234',42);
		this._families.set('_IMG_5678','banana');

		// TODO:
		// STEP 1   parse picasa ini to key-value array to obtained starred
		 /*
		 * STEP 2
		 *    iterate images and collect families and singles
		 *    (non-family-matchable) singles become important for i.e. unstarred
		 *    deletions...)
		 */

		/* STEP 3  perform action
		 *    actual delete/recycle or simulation of it
		 */


		// parsing each dir:
		dirs.forEach(function (dir) {

			enforce(fs.existsSync(dir), `no directory or file ${dir}`);
			enforce(fs.statSync(dir).isDirectory(), 'single File – not handled yet');

			let files = fs.readdirSync(dir);

			files.map((filepath) => {

				// ========================================================================
				// BIG LOOP
				// ========================================================================

				const p = path.parse(filepath);
				//console.log("dir " + p.dir + "name " + p.name + "  p.ext: " + p.ext);

				enforce(p.dir==='', 'no relative subDirs (for now)');
				enforce(p.name.length > 0, 'sanity: no empty filenames');
				// skipping hidden
				if (p.name[0] === '.')
					return;

				// remove leading dot on ext (always, unless extensionless)
				if (p.ext.length > 0) {
					enforce(p.ext[0] === '.', 'sanity');
					p.ext = p.ext.substr(1);
				}

				// filter for supported filetypes 
				// TODO supported sidecars, etc
				if ( !config.extensions.includes(p.ext.toLowerCase()) )
					return;

				// REF
				// { root: '',
				// dir: '',
				// base: 'PM5A2847.JPG',
				// ext: 'JPG',
				// name: 'PM5A2847' }

				console.dir("dir " + dir + "  p.dir "+ p.dir + "    name " + p.name + "  p.ext: " + p.ext);






				// ========================================================================
				// BIG LOOP
				// ========================================================================

				
			}) // files.map
		}); // forEach


		console.log('dumping FileSet Map: ===============================');

		for(var [key, value] of this._families) {
			console.log(key + ' = ' + value);
		}

{'fraction_non_alphanumeric': 0.21965065502183406, 'fraction_numerical': 0.010043668122270743, 'mean_word_length': 2.7426470588235294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2717672', 'n_tokens_mistral': 1771, 'n_tokens_neox': 1421, 'n_words': 384}