<commit_before>/* jshint globalstrict: true */
/* global require, module, __dirname, Buffer */

"use strict";

let Promise = require("bluebird"),
    AssetGraph = require("assetgraph"),
    streamifer = require("streamifier"),
    mongoose = require("mongoose"),
    //Grid = require("gridfs-stream"),
    mongo = mongoose.mongo,
    Grid = mongo.Grid,
    path = require("path"),
    parseTorrent = require("parse-torrent"),
    createTorrent = Promise.promisify(require("create-torrent")),
    Torrent = require("./model/Torrent");

// Async magic!
//Promise.promisifyAll(Grid);
//Promise.promisifyAll(Grid.prototype);

// XXX: Should the saving of objects be the responsibility of the caller, or the model methods themselves?
let buildUrlTorrent = Promise.coroutine(function *(url) {
    let urlTorrent = new Torrent({
        type: Torrent.TYPE_URL,
        url: url,
        data: yield createTorrent(new Buffer(url), { name: "pageUrl" })
    });

    return urlTorrent.saveAsync().get(0);
});

let crawlUrl = Promise.coroutine(function *(url) {
    // TODO: Cleanup decls, define closest to first use<commit_msg>Change url torrent to contain json file rather than plain text<commit_after>/* jshint globalstrict: true */
/* global require, module, __dirname, Buffer */

"use strict";

let Promise = require("bluebird"),
    AssetGraph = require("assetgraph"),
    streamifer = require("streamifier"),
    mongoose = require("mongoose"),
    Grid = mongoose.mongo.Grid,
    path = require("path"),
    parseTorrent = require("parse-torrent"),
    createTorrent = Promise.promisify(require("create-torrent")),
    Torrent = require("./model/Torrent");

// Async magic!
// FIXME: Move this to async-grid module
Promise.promisifyAll(Grid);
Promise.promisifyAll(Grid.prototype);

// TODO: Some parts of this module really should be static or instance methods on Page/Torrent

// XXX: Should the saving of objects be the responsibility of the caller, or the model methods themselves?
let buildUrlTorrent = Promise.coroutine(function *(url) {
    let urlTorrent = new Torrent({
        type: Torrent.TYPE_URL,
        url: url,
        data: yield createTorrent(new Buffer(JSON.stringify({
            pageUrl: url
        })), { name: "url.json" })
    });

    return urlTorrent.saveAsync().get(0);
});

let crawlUrl = Promise.coroutine(function *(url) {
    // TODO: Cleanup decls, define closest to first use
{'fraction_non_alphanumeric': 0.13278008298755187, 'fraction_numerical': 0.0008298755186721991, 'mean_word_length': 3.755424063116371, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5767826', 'n_tokens_mistral': 704, 'n_tokens_neox': 707, 'n_words': 202}