<commit_before>const r = require('../r');
const path = require('path');
const dbExec = require('./run');
const db = require('./db');
const model = require('../../../shared/model');
const files = require('./files');
const projects = require('./projects');
const renameHelper = require('./directory-rename');
const getSingle = require('./get-single');

function* get(projectID, directoryID) {
    if (directoryID === "top") {
        return topLevelDir(projectID);
    } else {
        return directoryByID(directoryID);
    }
}

function getByName(projectID, name) {
    let indexKeys = [projectID, name];
    let rql = r.table('datadirs').getAll(indexKeys, {index: 'datadir_project_name'})
        .merge(function (ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function* getAll(projectId) {
    return yield r.table('project2datadir')
        .getAll(projectId, {index: 'project_id'})
        .eqJoin('datadir_id', r.db('materialscommons').table('datadirs')).zip()
        .pluck('name', 'id').orderBy('name');
}

function topLevelDir(projectID) {
    let rql = r.table('projects').getAll(projectID)
        .eqJoin('name', r.table('datadirs'), {index: 'name'}).zip()
        .eqJoin('id', r.table('project2datadir'), {index: 'datadir_id'}).zip()
        .filter({'project_id': projectID})
        .merge(function(ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('datadir_id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('datadir_id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function directoryByID(directoryID) {
    let rql = r.table('project2datadir').getAll(directoryID, {index: 'datadir_id'})
        .eqJoin('datadir_id', r.table('datadirs')).zip()
        .merge(function(ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('datadir_id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('datadir_id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function toDir(results) {
    let dir = {
        otype: 'directory',
        id: results.datadir_id ? results.datadir_id : results.id,
        size: 0,
        name: path.basename(results.name),
        path: results.name,
        checksum: "",
        children: [],
        shortcut: results.shortcut,
    };

    dir.children = results.files.map(f => {
        return {
            otype: 'file',
            size: f.size,
            name: f.name,
            path: path.join(dir.path, f.name),
            mediatype: f.mediatype,
            checksum: f.checksum,
            id: f.id,
            usesid: f.usesid,
            shortcut: false,
        };
    });

    let childrenDirs = results.directories.map(d => {
        return {
            otype: 'directory',
            id: d.id,
            size: 0,
            name: path.basename(d.name),
            path: d.name,
            checksum: "",
            shortcut: d.shortcut
        };
    });

    dir.children = dir.children.concat(childrenDirs);

    return dir;
}<commit_msg>Add mtime back into output (needed for Python cli/api)<commit_after>const r = require('../r');
const path = require('path');
const dbExec = require('./run');
const db = require('./db');
const model = require('../../../shared/model');
const files = require('./files');
const projects = require('./projects');
const renameHelper = require('./directory-rename');
const getSingle = require('./get-single');

function* get(projectID, directoryID) {
    if (directoryID === "top") {
        return topLevelDir(projectID);
    } else {
        return directoryByID(directoryID);
    }
}

function getByName(projectID, name) {
    let indexKeys = [projectID, name];
    let rql = r.table('datadirs').getAll(indexKeys, {index: 'datadir_project_name'})
        .merge(function (ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function* getAll(projectId) {
    return yield r.table('project2datadir')
        .getAll(projectId, {index: 'project_id'})
        .eqJoin('datadir_id', r.db('materialscommons').table('datadirs')).zip()
        .pluck('name', 'id').orderBy('name');
}

function topLevelDir(projectID) {
    let rql = r.table('projects').getAll(projectID)
        .eqJoin('name', r.table('datadirs'), {index: 'name'}).zip()
        .eqJoin('id', r.table('project2datadir'), {index: 'datadir_id'}).zip()
        .filter({'project_id': projectID})
        .merge(function(ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('datadir_id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('datadir_id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function directoryByID(directoryID) {
    let rql = r.table('project2datadir').getAll(directoryID, {index: 'datadir_id'})
        .eqJoin('datadir_id', r.table('datadirs')).zip()
        .merge(function(ddir) {
            return {
                files: r.table('datadir2datafile').getAll(ddir('datadir_id'), {index: 'datadir_id'})
                    .eqJoin('datafile_id', r.table('datafiles')).zip()
                    .filter({current: true})
                    .coerceTo('array'),
                directories: r.table('datadirs').getAll(ddir('datadir_id'), {index: 'parent'}).coerceTo('array')
            }
        });
    return dbExec(rql).then(results => toDir(results[0]));
}

function toDir(results) {
    let dir = {
        otype: 'directory',
        id: results.datadir_id ? results.datadir_id : results.id,
        size: 0,
        name: path.basename(results.name),
        path: results.name,
        mtime: results.mtime,
        checksum: "",
        children: [],
        shortcut: results.shortcut,
    };

    dir.children = results.files.map(f => {
        return {
            otype: 'file',
            size: f.size,
            name: f.name,
            path: path.join(dir.path, f.name),
            mediatype: f.mediatype,
            checksum: f.checksum,
            mtime: f.mtime,
            id: f.id,
            usesid: f.usesid,
            shortcut: false,
        };
    });

    let childrenDirs = results.directories.map(d => {
        return {
            otype: 'directory',
            id: d.id,
            size: 0,
            name: path.basename(d.name),
            path: d.name,
            checksum: "",
            shortcut: d.shortcut
        };
    });

    dir.children = dir.children.concat(childrenDirs);

    return dir;
}
{'fraction_non_alphanumeric': 0.1713822759566247, 'fraction_numerical': 0.002742116415305995, 'mean_word_length': 2.2224899598393573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1360447', 'n_tokens_mistral': 2562, 'n_tokens_neox': 2506, 'n_words': 420}