<commit_before>var f = require('fermata'),
    Q = require('queue-async');


function makeSet(arr) {
    return arr.reduce(function (obj, k) {
        obj[k] = true;
        return obj;
    }, Object.create(null))
}

var CONTENT = makeSet(['files', 'thumbnails', 'previews', 'files_put', 'chunked_upload', 'commit_chunked_upload']),
    NO_AUTO = makeSet(['account', 'delta', 'longpoll_delta', 'chunked_upload', 'fileops']);

f.registerPlugin('dropbox', function (transport, key) {
    this.base = "/1/";       // (placeholder)
    this.path = [];
    transport = transport.using('statusCheck').using('autoConvert', "application/json");
    
    return function (req, callback) {
        // TODO: this check is not production-grade, and there are a couple subdomains missing
        if (req.path[0] in CONTENT) req.base = "https://api-content.dropbox.com/1/";
        else req.base = "https://api.dropbox.com/1/";
        
        // HACK: allow elision of common "auto" path component
        if (req.path[0] in NO_AUTO || req.path[1] === 'auto') ;
        else req.path.splice(1, 0, 'auto');
        
        // via https://www.dropbox.com/developers/blog/45/using-oauth-20-with-the-core-api
        req.headers['Authorization'] = "Bearer "+key;
        
        return transport(req, callback);
    };
});


var api = f.dropbox(process.env.API_TOKEN),
    db = f.json(process.env.DB_URL);

function logback(e,d) {
    if (e) throw e;
    else console.log(d);
}

//api.account.info.get(logback);


// WORKAROUND: https://github.com/natevw/fermata/issues/28
// NOTE: breaks unless run with `node --harmony_proxies`
function _apply(fn) {
    var args = Array.prototype.slice.call(arguments, 1);
    return Function.prototype.apply.call(fn, this, args);
}

function fetchPhoto(path, cb) {
    var doc = {},
        q = Q();
    q.defer(_apply, api.metadata([path], {include_media_info:true}).get);
    q.defer(_apply, api.thumbnails([path], {size:'s'}).get, {Accept:"image/jpeg"}, Buffer(0));
    q.defer(_apply, api.thumbnails([path], {size:'l'}).get, {Accept:"image/jpeg"}, Buffer(0));
    //q.defer(_apply, api.files([path]).head, {Accept:"*/*"}, Buffer(0));
    
    q.await(function (e,d,sm,lg) {
        if (e) cb(e);
        else if (typeof d.photo_info !== 'object') cb(new Error("Photo metadata not available: "+meta.photo_info));  // e.g. 'pending'
        else cb(null, {
            "com.shutterstem.image": true,
            identifiers: {
                path: {
                    source: {_id:"imgsrc-DROPBOX_DEMO","net.stemstorage.reference":true},
                    name: d.path.slice(1), dbg_size: d.bytes
                }
            },
            timestamp: new Date(d.photo_info.time_taken).toISOString(),
            geotag: (d.photo_info.lat_long) ? {lat:d.photo_info.lat_long[0], lon:d.photo_info.lat_long[1]} : void 0,
            _attachments: {
                "thumbnail/512.jpg": {content_type: "image/jpeg", data:lg.toString('base64')},
                "thumbnail/64.jpg": {content_type: "image/jpeg", data:sm.toString('base64')}
            }
        });
    });
}

function processPhoto(path, cb) {
    fetchPhoto(path, function (e,d) {
        if (e) cb(e);
        else db.post(d, cb);
    });
}

processPhoto('/10000/_DSC0012.jpg', logback);


//api.files(['/10000/_DSC0012.jpg']).get({Accept:"*/*"}, Buffer(0), function (e,d,m) {
//    console.log("META", e,d,m);
//    var md5 = require('crypto').createHash('md5'),
//        sha = require('crypto').createHash('sha1');
//    md5.update(d), sha.update(d);
//    md5 = md5.digest(), sha = sha.digest();
//    console.log("MD5", md5.toString('hex'), md5.toString('base64'));
//    console.log("SHA1", sha.toString('hex'), sha.toString('base64'));
//    // NOTE: neither of these seem to be exposed in Dropbox metadata
//});

/*
api.metadata.auto(['10000'], {include_media_info:true}).get(function (e,d) {
    if (e) throw e;
    
    var q = Q(1);
    d.contents.forEach(function (file) {
        q.defer(processPhoto, file, db);  
    })
    q.awaitAll(function (e,arr) {
        if (e) throw e;
        console.log("Processed results, count:", arr.length);
    });
    
    console.log("# files:", d.contents.length);
    console.log(d.contents[0]);
});
*/<commit_msg>bulk fetch, with stubborntastic retry and hackolicious local file listing<commit_after>var f = require('fermata'),
    Q = require('queue-async');


function makeSet(arr) {
    return arr.reduce(function (obj, k) {
        obj[k] = true;
        return obj;
    }, Object.create(null))
}

var CONTENT = makeSet(['files', 'thumbnails', 'previews', 'files_put', 'chunked_upload', 'commit_chunked_upload']),
    NO_AUTO = makeSet(['account', 'delta', 'longpoll_delta', 'chunked_upload', 'fileops']);

f.registerPlugin('dropbox', function (transport, key) {
    this.base = "/1/";       // (placeholder)
    this.path = [];
    transport = transport.using('statusCheck').using('autoConvert', "application/json");
    
    return function (req, callback) {
        // TODO: this check is not production-grade, and there are a couple subdomains missing
        if (req.path[0] in CONTENT) req.base = "https://api-content.dropbox.com/1/";
        else req.base = "https://api.dropbox.com/1/";
        
        // HACK: allow elision of common "auto" path component
        if (req.path[0] in NO_AUTO || req.path[1] === 'auto') ;
        else req.path.splice(1, 0, 'auto');
        
        // via https://www.dropbox.com/developers/blog/45/using-oauth-20-with-the-core-api
        req.headers['Authorization'] = "Bearer "+key;
        
        return transport(req, callback);
    };
});


var folder = process.env.FOLDER,
    api = f.dropbox(process.env.API_TOKEN),
    db = f.json(process.env.DB_URL);

function logback(e,d) {
    if (e) throw e;
    else console.log(d);
}

//api.account.info.get(logback);


// WORKAROUND: https://github.com/natevw/fermata/issues/28
// NOTE: breaks unless run with `node --harmony_proxies`
function _apply(fn) {
    var args = Array.prototype.slice.call(arguments, 1);
    return Function.prototype.apply.call(fn, this, args);
}

function fetchPhoto(path, cb) {
    var doc = {},
        q = Q();
    if (typeof path === 'object') {
        q.defer(function (d, cb) { cb(null, d); }, path);
        path = path.path;
    } else q.defer(_apply, api.metadata([path], {include_media_info:true}).get);
    console.log("Fetching", path);
    
    q.defer(_apply, api.thumbnails([path], {size:'s'}).get, {Accept:"image/jpeg"}, Buffer(0));
    q.defer(_apply, api.thumbnails([path], {size:'l'}).get, {Accept:"image/jpeg"}, Buffer(0));
    q.await(function (e,d,sm,lg) {
        if (e) cb(e);
        else if (typeof d.photo_info !== 'object') cb(new Error("Photo metadata not available: "+meta.photo_info));  // e.g. 'pending'
        else cb(null, {
            "com.shutterstem.image": true,
            identifiers: {
                path: {
                    source: {_id:"imgsrc-DROPBOX_DEMO","net.stemstorage.reference":true},
                    name: d.path.slice(1), dbg_size: d.bytes
                }
            },
            timestamp: new Date(d.photo_info.time_taken).toISOString(),
            geotag: (d.photo_info.lat_long) ? {lat:d.photo_info.lat_long[0], lon:d.photo_info.lat_long[1]} : void 0,
            _attachments: {
                "thumbnail/512.jpg": {content_type: "image/jpeg", data:lg.toString('base64')},
                "thumbnail/64.jpg": {content_type: "image/jpeg", data:sm.toString('base64')}
            }
        });
    });
}

function _processPhoto(path, cb) {
    fetchPhoto(path, function (e,d) {
        if (e) cb(e);
        else db.post(d, function (e,d) {
            if (e) cb(e);
            else console.log(" -> saved as", d.id), cb();
        });
    });
}

function processPhoto(path, cb) {
    _processPhoto(path, function (e) {
        if (e) {
            console.warn("Failed, retrying!");
            processPhoto(path, cb);
        } else cb.apply(this, arguments);
    });
}

console.log("Fetching contents of Dropbox folder:", folder);
//api.metadata([folder], {include_media_info:true}).get(function (e,d) {
//if (!e) require('fs').writeFileSync("dbg-folder.json", JSON.stringify(d, null, 4));
require('fs').readFile("dbg-folder.json", {encoding:'utf8'}, function (e,d) {
if (!e) d = JSON.parse(d);

    if (e) throw e;
    
    console.log("Folder contains", d.contents.length, "files");
    
    var q = Q(1);
    d.contents.forEach(function (file) {
        // TODO: handle non-photos, subfolders…
        q.defer(processPhoto, file);  
    });
    q.awaitAll(function (e,arr) {
        if (e) console.error(e.stack);
        else console.log("Success!");
    });
});
{'fraction_non_alphanumeric': 0.1842889908256881, 'fraction_numerical': 0.011926605504587157, 'mean_word_length': 2.7094853253934494, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6604349', 'n_tokens_mistral': 3016, 'n_tokens_neox': 2936, 'n_words': 581}