<commit_before>var child_process = require('child_process'); // http://nodejs.org/api/child_process.html
var path = require('path'); // File path processing. http://nodejs.org/api/path.html
var _ = require('underscore'); // Utilities. http://underscorejs.org/
var Backbone = require('backbone'); // Data model utilities. http://backbonejs.org/
var moment = require('moment'); // Date processing. http://momentjs.com/
var request = require('request'); // Simpler HTTP. https://github.com/mikeal/request
var progress = require('request-progress'); // Progress events on downloads. https://github.com/IndigoUnited/node-request-progress
var XRegExp = require('xregexp').XRegExp; // Fancy regular expressions. http://xregexp.com/
var rimraf = require('rimraf'); // Recursive directory delete. https://github.com/isaacs/rimraf
var fs = require('node-fs'); // Recursive directory creation. https://github.com/bpedro/node-fs
var ncp = require('ncp').ncp; // Recursive directory copy. https://npmjs.org/package/ncp
var winston = require('winston'); // Logging. https://github.com/flatiron/winston

// Update content for the application, as well as the application itself.
exports.ContentUpdater = Backbone.Model.extend({

    defaults: {
        // When the content was last updated.
        downloaded: null,
        updated: null,

        // The remote URL to the root XML.
        remote: null,

        // The final local path for content.
        local: 'content/',

        // The temp path for content.
        temp: 'content.tmp/',

        // A collection of file objects that are being processed.
        files: null,
        needsUpdate: false,
    },

    // A callback to call when updating is complete.
    _callback: null,

    // Download new content to the temp folder.
    download: function(callback) {
        this.set('needsUpdate', false);
        this._callback = callback;
        this._initDirectories(_.bind(function() {
            if (this.get('remote').indexOf('http') === 0) {
                // We're going to pull down an XML file from the web and parse it for other files.<commit_msg>don't try to update if you don't have a URL<commit_after>var child_process = require('child_process'); // http://nodejs.org/api/child_process.html
var path = require('path'); // File path processing. http://nodejs.org/api/path.html
var _ = require('underscore'); // Utilities. http://underscorejs.org/
var Backbone = require('backbone'); // Data model utilities. http://backbonejs.org/
var moment = require('moment'); // Date processing. http://momentjs.com/
var request = require('request'); // Simpler HTTP. https://github.com/mikeal/request
var progress = require('request-progress'); // Progress events on downloads. https://github.com/IndigoUnited/node-request-progress
var XRegExp = require('xregexp').XRegExp; // Fancy regular expressions. http://xregexp.com/
var rimraf = require('rimraf'); // Recursive directory delete. https://github.com/isaacs/rimraf
var fs = require('node-fs'); // Recursive directory creation. https://github.com/bpedro/node-fs
var ncp = require('ncp').ncp; // Recursive directory copy. https://npmjs.org/package/ncp
var winston = require('winston'); // Logging. https://github.com/flatiron/winston

// Update content for the application, as well as the application itself.
exports.ContentUpdater = Backbone.Model.extend({

    defaults: {
        // When the content was last updated.
        downloaded: null,
        updated: null,

        // The remote URL to the root XML.
        remote: null,

        // The final local path for content.
        local: 'content/',

        // The temp path for content.
        temp: 'content.tmp/',

        // A collection of file objects that are being processed.
        files: null,
        needsUpdate: false,
    },

    // A callback to call when updating is complete.
    _callback: null,

    // Download new content to the temp folder.
    download: function(callback) {
        if (!this.get('remote')) {
            return;
        }

        this.set('needsUpdate', false);
        this._callback = callback;
        this._initDirectories(_.bind(function() {
            if (this.get('remote').indexOf('http') === 0) {
                // We're going to pull down an XML file from the web and parse it for other files.
{'fraction_non_alphanumeric': 0.1356050069541029, 'fraction_numerical': 0.0004636068613815484, 'mean_word_length': 2.8355555555555556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '82210', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1176, 'n_words': 371}