<commit_before>#!/usr/bin/env node

'use strict';

var Promise = require('bluebird'),
    request = require('request'),
    mongoose = require('mongoose'),
    path = require('path'),
    fs = require('fs-extra'),
    checksum = require('checksum'),
    config = require('../lib/config'),
    fetchedItems = 0,
    downloadImageAsync = Promise.promisify(downloadImage),
    fsExists = Promise.promisify(function fsExists(src, cb) {
        fs.exists(src, function (exists) {
            cb(null, exists);
        });
    });

Promise.promisifyAll(fs);
Promise.promisifyAll(checksum);

Promise.promisify(config.load)()
    .then(function () {
        if (!config.get) {
            throw new Error('Server has not been configured yet. Please run bin/setup.');
        }
        var dburl = 'mongodb://' +
            config.get.mongodb.host + ':' +
            config.get.mongodb.port + '/' +
            config.get.mongodb.dbname;
        mongoose.connect(dburl);
        mongoose.model('Photo', require('../models/photo').Photo);
        return fs.mkdirpAsync(path.join('..', 'assets', 'photos'));
    })
    .then(function () {
        return mongoose.model('Photo').find({});
    })<commit_msg>fetch fix<commit_after>#!/usr/bin/env node

'use strict';

var Promise = require('bluebird'),
    request = require('request'),
    mongoose = require('mongoose'),
    path = require('path'),
    fs = require('fs-extra'),
    checksum = require('checksum'),
    config = require('../lib/config'),
    fetchedItems = 0,
    downloadImageAsync = Promise.promisify(downloadImage),
    fsExists = Promise.promisify(function fsExists(src, cb) {
        fs.exists(src, function (exists) {
            cb(null, exists);
        });
    });

Promise.promisifyAll(fs);
Promise.promisifyAll(checksum);

Promise.resolve()
    .then(function () {
        return config.load();
    })
    .then(function () {
        if (!config.get) {
            throw new Error('Server has not been configured yet. Please run bin/setup.');
        }
        var dburl = 'mongodb://' +
            config.get.mongodb.host + ':' +
            config.get.mongodb.port + '/' +
            config.get.mongodb.dbname;
        mongoose.connect(dburl);
        mongoose.model('Photo', require('../models/photo').Photo);
        return fs.mkdirpAsync(path.join('..', 'assets', 'photos'));
    })
    .then(function () {
        return mongoose.model('Photo').find({});
    })
{'fraction_non_alphanumeric': 0.17159277504105092, 'fraction_numerical': 0.0008210180623973727, 'mean_word_length': 2.537010159651669, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3335866', 'n_tokens_mistral': 751, 'n_tokens_neox': 746, 'n_words': 134}