function remove_from_removed_projects(cb) {
    console.log("remove_from_removed_projects");
    let month_ago = new Date();
    month_ago.setMonth(month_ago.getMonth() - 1);

    //TODO mark already handled project so that I don't have to keep querying them?
    db.Projects.find({
        removed: true,
    }).select('_id').lean().exec((err, projects)=>{
        if(err) return cb(err);
        if(!projects) return cb(); // no project removed?

        async.eachSeries(projects, (project, next_project)=>{
            //now query datasets that are not removed on removed projects
            console.log(".... querying for old data not removed in removed project", project._id);
            db.Datasets.find({
                removed: false,
                project,
                create_date: {$lt: month_ago }, //only remove if it's old enough
            })
            .select('_id')
            .sort('create_date') //oldest first
            .limit(limit) 
            .exec((err,datasets)=>{
                if(err) return next_project(err);
                if(!datasets) return next_project(); //no datasets to clean up
                //console.debug("orphaned datasets needs removed "+datasets.length);
                async.eachSeries(datasets, (dataset, next_dataset)=>{
                    console.debug("removing orphaned dataset "+dataset._id.toString());
                    dataset.remove_date = new Date();
                    dataset.removed = true;
                    dataset.save(next_dataset);
                }, next_project);
            });
            
        }, err=>{
            if(err) return cb(err);
            console.log("done processing removed projects");
            cb();
        });
    });
}
//remove datasets stored in removed projects
remove datasets stored in removed projects
remove datasets stored in removed projects
function remove_from_removed_projects(cb) {
    console.log("remove_from_removed_projects");
    let month_ago = new Date();
    month_ago.setMonth(month_ago.getMonth() - 1);
    db.Projects.find({
        removed: true,
    }).select('_id').lean().exec((err, projects)=>{
        if(err) return cb(err);
        if(!projects) return cb(); 
        async.eachSeries(projects, (project, next_project)=>{
            console.log(".... querying for old data not removed in removed project", project._id);
            db.Datasets.find({
                removed: false,
                project,
                create_date: {$lt: month_ago }, 
            })
            .select('_id')
            .sort('create_date') 
            .limit(limit) 
            .exec((err,datasets)=>{
                if(err) return next_project(err);
                if(!datasets) return next_project(); 
                async.eachSeries(datasets, (dataset, next_dataset)=>{
                    console.debug("removing orphaned dataset "+dataset._id.toString());
                    dataset.remove_date = new Date();
                    dataset.removed = true;
                    dataset.save(next_dataset);
                }, next_project);
            });
        }, err=>{
            if(err) return cb(err);
            console.log("done processing removed projects");
            cb();
        });
    });
}
function remove_from_removed_projects ( cb ) { console . log ( "remove_from_removed_projects" ) ; let month_ago = new Date ( ) ; month_ago . setMonth ( month_ago . getMonth ( ) - 1 ) ; db . Projects . find ( { removed : true , } ) . select ( '_id' ) . lean ( ) . exec ( ( err , projects ) => { if ( err ) return cb ( err ) ; if ( ! projects ) return cb ( ) ; async . eachSeries ( projects , ( project , next_project ) => { console . log ( ".... querying for old data not removed in removed project" , project . _id ) ; db . Datasets . find ( { removed : false , project , create_date : { $lt : month_ago } , } ) . select ( '_id' ) . sort ( 'create_date' ) . limit ( limit ) . exec ( ( err , datasets ) => { if ( err ) return next_project ( err ) ; if ( ! datasets ) return next_project ( ) ; async . eachSeries ( datasets , ( dataset , next_dataset ) => { console . debug ( "removing orphaned dataset " + dataset . _id . toString ( ) ) ; dataset . remove_date = new Date ( ) ; dataset . removed = true ; dataset . save ( next_dataset ) ; } , next_project ) ; } ) ; } , err => { if ( err ) return cb ( err ) ; console . log ( "done processing removed projects" ) ; cb ( ) ; } ) ; } ) ; }
remove datasets stored in removed projects
remove datasets stored in removed projects
//TODO mark already handled project so that I don't have to keep querying them? // no project removed? //now query datasets that are not removed on removed projects //only remove if it's old enough //oldest first //no datasets to clean up //console.debug("orphaned datasets needs removed "+datasets.length);