<commit_before>/**
 * Server-side Backbone implementation for TileMill. This module should be
 * required instead of directly requiring Backbone.
 */
var _ = require('underscore')._,
    Backbone = require('../modules/backbone/backbone.js'),
    settings = require('settings'),
    rmrf = require('rm-rf'),
    fs = require('fs'),
    Step = require('step'),
    path = require('path');

/**
 * Override Backbone.sync() for the server-side context. Uses TileMill-specific
 * file storage methods for model CRUD operations.
 */
Backbone.sync = function(method, model, success, error) {
    switch (method) {
    case 'read':
        if (model.id) {
            load(model, function(err, model) {
                return err ? error(err) : success(model);
            });
        } else {
            loadAll(model, function(err, model) {
                return err ? error(err) : success(model);
            });
        }
        break;
    case 'create':
        create(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    case 'update':
        update(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    case 'delete':
        destroy(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    }
};

/**
 * Load a single model. Requires that model.id be populated.
 */
function load(model, callback) {
    var modelPath = path.join(settings.files, model.type);
    modelPath = (model.type == 'project') ? path.join(modelPath, model.id) : modelPath;
    var extension = (model.type == 'project') ? 'mml' : 'json';
    fs.readFile(path.join(modelPath, model.id + '.' + extension), 'utf-8',
    function(err, data) {
        if (err || !data) {
            return callback(new Error('Error reading model file.'));
        }
        var object = JSON.parse(data);
        // Set the object ID explicitly for multiple-load scenarios where
        // model parse()/set() is bypassed.
        object.id = model.id;
        if (model.type === 'project' && object.Stylesheet && object.Stylesheet.length > 0) {
            Step(
                function() {
                    var group = this.group();
                    _.each(object.Stylesheet, function(filename, index) {
                        fs.readFile(
                            path.join(modelPath, filename),
                            'utf-8',
                            group()
                        );
                    });
                },
                function(err, files) {
                    object.Stylesheet = _.reduce(
                        object.Stylesheet,
                        function(memo, filename, index) {
                            if (typeof files[index] !== 'undefined') {
                                memo.push({
                                    id: filename,
                                    data: files[index]
                                });
                            }
                            return memo;
                        },
                        []
                    );
                    return callback(null, object);
                }
            );
        }
        else {
            return callback(null, object);
        }
    });
};

/**
 * Load an array of all models.
 */
function loadAll(model, callback) {
    var basepath = path.join(settings.files, model.type);
    Step(
        function() {
            path.exists(basepath, this);
        },
        function(exists) {
            if (!exists) {
                fs.mkdir(basepath, 0777, this);
            } else {
                this();
            }
        },
        function() {
            fs.readdir(basepath, this);
        },
        function(err, files) {
            if (err) {
                return this(new Error('Error reading model directory.'));
            }
            else if (files.length === 0) {
                return this();
            }
            var group = this.group();
            for (var i = 0; i < files.length; i++) {
                if (model.type === 'project') {
                    var id = files[i];
                } else {
                    var id = path.basename(files[i], '.json');
                }
                load({ id: id, type: model.type }, group());
            }
        },
        function(err, models) {
            return callback(err, models);
        }
    );
};

/**
 * Create a new model.
 * @TODO assign a model id if not present.
 */
function create(model, callback) {
    if (model.type === 'project') {
        saveProject(model, callback);
    } else {
        save(model, callback);
    }<commit_msg>Don't bail on loading a collection if individual model loads fail. Fixes #146.<commit_after>/**
 * Server-side Backbone implementation for TileMill. This module should be
 * required instead of directly requiring Backbone.
 */
var _ = require('underscore')._,
    Backbone = require('../modules/backbone/backbone.js'),
    settings = require('settings'),
    rmrf = require('rm-rf'),
    fs = require('fs'),
    Step = require('step'),
    path = require('path');

/**
 * Override Backbone.sync() for the server-side context. Uses TileMill-specific
 * file storage methods for model CRUD operations.
 */
Backbone.sync = function(method, model, success, error) {
    switch (method) {
    case 'read':
        if (model.id) {
            load(model, function(err, model) {
                return err ? error(err) : success(model);
            });
        } else {
            loadAll(model, function(err, model) {
                return err ? error(err) : success(model);
            });
        }
        break;
    case 'create':
        create(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    case 'update':
        update(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    case 'delete':
        destroy(model, function(err, model) {
            return err ? error(err) : success(model);
        });
        break;
    }
};

/**
 * Load a single model. Requires that model.id be populated.
 */
function load(model, callback) {
    var modelPath = path.join(settings.files, model.type);
    modelPath = (model.type == 'project') ? path.join(modelPath, model.id) : modelPath;
    var extension = (model.type == 'project') ? 'mml' : 'json';
    fs.readFile(path.join(modelPath, model.id + '.' + extension), 'utf-8',
    function(err, data) {
        if (err || !data) {
            return callback(new Error('Error reading model file.'));
        }
        var object = JSON.parse(data);
        // Set the object ID explicitly for multiple-load scenarios where
        // model parse()/set() is bypassed.
        object.id = model.id;
        if (model.type === 'project' && object.Stylesheet && object.Stylesheet.length > 0) {
            Step(
                function() {
                    var group = this.group();
                    _.each(object.Stylesheet, function(filename, index) {
                        fs.readFile(
                            path.join(modelPath, filename),
                            'utf-8',
                            group()
                        );
                    });
                },
                function(err, files) {
                    object.Stylesheet = _.reduce(
                        object.Stylesheet,
                        function(memo, filename, index) {
                            if (typeof files[index] !== 'undefined') {
                                memo.push({
                                    id: filename,
                                    data: files[index]
                                });
                            }
                            return memo;
                        },
                        []
                    );
                    return callback(null, object);
                }
            );
        }
        else {
            return callback(null, object);
        }
    });
};

/**
 * Load an array of all models.
 */
function loadAll(model, callback) {
    var basepath = path.join(settings.files, model.type);
    Step(
        function() {
            path.exists(basepath, this);
        },
        function(exists) {
            if (!exists) {
                fs.mkdir(basepath, 0777, this);
            } else {
                this();
            }
        },
        function() {
            fs.readdir(basepath, this);
        },
        function(err, files) {
            if (err) {
                return this(new Error('Error reading model directory.'));
            }
            else if (files.length === 0) {
                return this();
            }
            var group = this.group();
            for (var i = 0; i < files.length; i++) {
                if (model.type === 'project') {
                    var id = files[i];
                } else {
                    var id = path.basename(files[i], '.json');
                }
                load({ id: id, type: model.type }, group());
            }
        },
        function(err, models) {
            // Ignore errors from loading individual models (e.g.
            // don't let one bad apple spoil the collection).
            models = _.select(models, function(model) {
                return (typeof model === 'object');
            });
            return callback(null, models);
        }
    );
};

/**
 * Create a new model.
 * @TODO assign a model id if not present.
 */
function create(model, callback) {
    if (model.type === 'project') {
        saveProject(model, callback);
    } else {
        save(model, callback);
    }
{'fraction_non_alphanumeric': 0.12754271765663142, 'fraction_numerical': 0.0021358828315703824, 'mean_word_length': 1.332858837485172, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7032741', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2502, 'n_words': 648}