<commit_before>import store from '../store.js';

import AppConstants  from '../constants/AppConstants';
import AppActions    from './AppActions';

import app   from '../lib/app';
import utils from '../utils/utils';

import fs       from 'fs';
import path     from 'path';
import globby   from 'globby';
import Promise  from 'bluebird';
import queue    from 'queue';

const dialog = electron.remote.dialog;
const statAsync = Promise.promisify(fs.stat);

const load = async () => {
    const querySort = {
        'loweredMetas.artist': 1,
        'year': 1,
        'loweredMetas.album': 1,
        'disk.no': 1,
        'track.no': 1
    };

    try {
        const tracks = await app.models.Track.find().sort(querySort).execAsync();
        store.dispatch({
            type : AppConstants.APP_REFRESH_LIBRARY,
            tracks
        });
    } catch (err) {
        console.warn(err);
    }
};

const setTracksCursor = (cursor) => {
    store.dispatch({
        type : AppConstants.APP_LIBRARY_SET_TRACKSCURSOR,
        cursor
    });
};

const resetTracks = () => {
    store.dispatch({
        type : AppConstants.APP_REFRESH_LIBRARY,
        tracks : null
    });
};

const filterSearch = (search) => {
    store.dispatch({
        type : AppConstants.APP_FILTER_SEARCH,
        search
    });
};

const scan = {
    processed: 0,
    total: 0,
};

const endScan = () => {
    scan.processed = 0;
    scan.total = 0;

    store.dispatch({
        type : AppConstants.APP_LIBRARY_REFRESH_END
    });

    AppActions.library.load();
};

const scanConcurrency = 16;
const scanQueue = new queue();
scanQueue.concurrency = scanConcurrency;
scanQueue.on('end', endScan);
scanQueue.on('success', () => {
    refreshProgress(scan.processed, scan.total);
});

const add = (pathsToScan) => {
    store.dispatch({
        type : AppConstants.APP_LIBRARY_REFRESH_START
    });

    let rootFiles; // HACK Kind of hack, looking for a better solution

    // Scan folders and add files to library
    new Promise((resolve) => {
        const paths = Promise.map(pathsToScan, (path) => {
            return statAsync(path).then((stat) => {
                return {
                    path,
                    stat
                };
            });
        });

        resolve(paths);
    }).then((paths) => {
        const files = [];
        const folders = [];

        paths.forEach((elem) => {
            if(elem.stat.isFile()) files.push(elem.path);
            if(elem.stat.isDirectory() || elem.stat.isSymbolicLink()) folders.push(elem.path);
        });

        rootFiles = files;

        return Promise.map(folders, (folder) => {
            const pattern = path.join(folder, '**/*.*');
            return globby(pattern, { nodir: true, follow: true });
        });
    }).then((files) => {
        // Merge all path arrays together and filter them with the extensions we support
        const flatFiles = files.reduce((acc, array) => acc.concat(array), [])
            .concat(rootFiles)
            .filter((filePath) => {
                const extension = path.extname(filePath).toLowerCase();
                return app.supportedExtensions.includes(extension);
            }
        );

        return flatFiles;
    }).then((supportedFiles) => {
        if (supportedFiles.length === 0) {
            return;
        }

        // Add files to be processed to the scan object
        scan.total += supportedFiles.length;

        supportedFiles.forEach((filePath) => {
            scanQueue.push((callback) => {
                return app.models.Track.findAsync({ path: filePath }).then((docs) => {
                    if (docs.length === 0) {
                        return utils.getMetadata(filePath);
                    }
                    return null;
                }).then((track) => {
                    // If null, that means a track with the same absolute path already exists in the database
                    if(track === null) return;
                    // else, insert the new document in the database
                    return app.models.Track.insertAsync(track);
                }).then(() => {
                    scan.processed++;
                    callback();
                });
            });
        });

        scanQueue.start();
    }).catch((err) => {
        console.warn(err);
    });

    // TODO progressive loading in the store, don't freeze the app, able to add files/folders when scanning<commit_msg>Use autostart for scan queue<commit_after>import store from '../store.js';

import AppConstants  from '../constants/AppConstants';
import AppActions    from './AppActions';

import app   from '../lib/app';
import utils from '../utils/utils';

import fs       from 'fs';
import path     from 'path';
import globby   from 'globby';
import Promise  from 'bluebird';
import queue    from 'queue';

const dialog = electron.remote.dialog;
const statAsync = Promise.promisify(fs.stat);

const load = async () => {
    const querySort = {
        'loweredMetas.artist': 1,
        'year': 1,
        'loweredMetas.album': 1,
        'disk.no': 1,
        'track.no': 1
    };

    try {
        const tracks = await app.models.Track.find().sort(querySort).execAsync();
        store.dispatch({
            type : AppConstants.APP_REFRESH_LIBRARY,
            tracks
        });
    } catch (err) {
        console.warn(err);
    }
};

const setTracksCursor = (cursor) => {
    store.dispatch({
        type : AppConstants.APP_LIBRARY_SET_TRACKSCURSOR,
        cursor
    });
};

const resetTracks = () => {
    store.dispatch({
        type : AppConstants.APP_REFRESH_LIBRARY,
        tracks : null
    });
};

const filterSearch = (search) => {
    store.dispatch({
        type : AppConstants.APP_FILTER_SEARCH,
        search
    });
};

const scan = {
    processed: 0,
    total: 0,
};

const endScan = () => {
    scan.processed = 0;
    scan.total = 0;

    store.dispatch({
        type : AppConstants.APP_LIBRARY_REFRESH_END
    });

    AppActions.library.load();
};

const scanConcurrency = 16;
const scanQueue = new queue();
scanQueue.concurrency = scanConcurrency;
scanQueue.autostart = true;
scanQueue.on('end', endScan);
scanQueue.on('success', () => {
    refreshProgress(scan.processed, scan.total);
});

const add = (pathsToScan) => {
    store.dispatch({
        type : AppConstants.APP_LIBRARY_REFRESH_START
    });

    let rootFiles; // HACK Kind of hack, looking for a better solution

    // Scan folders and add files to library
    new Promise((resolve) => {
        const paths = Promise.map(pathsToScan, (path) => {
            return statAsync(path).then((stat) => {
                return {
                    path,
                    stat
                };
            });
        });

        resolve(paths);
    }).then((paths) => {
        const files = [];
        const folders = [];

        paths.forEach((elem) => {
            if(elem.stat.isFile()) files.push(elem.path);
            if(elem.stat.isDirectory() || elem.stat.isSymbolicLink()) folders.push(elem.path);
        });

        rootFiles = files;

        return Promise.map(folders, (folder) => {
            const pattern = path.join(folder, '**/*.*');
            return globby(pattern, { nodir: true, follow: true });
        });
    }).then((files) => {
        // Merge all path arrays together and filter them with the extensions we support
        const flatFiles = files.reduce((acc, array) => acc.concat(array), [])
            .concat(rootFiles)
            .filter((filePath) => {
                const extension = path.extname(filePath).toLowerCase();
                return app.supportedExtensions.includes(extension);
            }
        );

        return flatFiles;
    }).then((supportedFiles) => {
        if (supportedFiles.length === 0) {
            return;
        }

        // Add files to be processed to the scan object
        scan.total += supportedFiles.length;

        supportedFiles.forEach((filePath) => {
            scanQueue.push((callback) => {
                return app.models.Track.findAsync({ path: filePath }).then((docs) => {
                    if (docs.length === 0) {
                        return utils.getMetadata(filePath);
                    }
                    return null;
                }).then((track) => {
                    // If null, that means a track with the same absolute path already exists in the database
                    if(track === null) return;
                    // else, insert the new document in the database
                    return app.models.Track.insertAsync(track);
                }).then(() => {
                    scan.processed++;
                    callback();
                });
            });
        });
    }).catch((err) => {
        console.warn(err);
    });

    // TODO progressive loading in the store, don't freeze the app, able to add files/folders when scanning
{'fraction_non_alphanumeric': 0.13338559749132042, 'fraction_numerical': 0.002911860230708926, 'mean_word_length': 1.8585147247119078, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6624523', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2419, 'n_words': 594}