<commit_before>#!/usr/bin/env ringo

// prepend the web app's directory to the module search path
require.paths.push(module.directory);

// Load additional RingoJS packages:
require("packages").loadPackages(getRepository( require("config").DIRS.packages));

// Add triggers for jsonfs:
var jsonfs = require("ctl/objectfs/json");
var SolrClient = require("PolicyFeed/Solr/Client");
var DocQueries = require("PolicyFeed/Crawler/DocQueries");

jsonfs.addTrigger("afterWrite", "/docs/", SolrClient.onItemChange);
jsonfs.addTrigger("afterRemove", "/docs/", SolrClient.onItemChange);
jsonfs.addTrigger("afterCreate", "/docs/", DocQueries.runQueries);

// Start Crawler:
var Crawler = require("PolicyFeed/Crawler");
Crawler.init(require("config")["PolicyFeed/Crawler"]);
Crawler.checkUpdates();

// Some vars and functions for easier error handling:
var CrawlerErrors = require("PolicyFeed/Crawler/Errors");
var CrawlerFailures = require("PolicyFeed/Crawler/Failures");

function retryUrl(url) {
    print(url);
    CrawlerErrors.removeUrl(url);
    Crawler.reindexUrl(url);
}

function failUrl(url, reason) {
    reason = reason || "Unknown (manual).".
    CrawlerErrors.removeUrl(url);
    CrawlerFailures.write(false, { url: url, reason: reason });
}

function retryAllUrls() {
    return CrawlerErrors.getUrlStats().map(function(item) {
            try {
                retryUrl(item.url);
                return true;
            } catch (e) {
                print(e);
                return false;
            }
    });<commit_msg>Fixed a typo in failUrl() crawler command.<commit_after>#!/usr/bin/env ringo

// prepend the web app's directory to the module search path
require.paths.push(module.directory);

// Load additional RingoJS packages:
require("packages").loadPackages(getRepository( require("config").DIRS.packages));

// Add triggers for jsonfs:
var jsonfs = require("ctl/objectfs/json");
var SolrClient = require("PolicyFeed/Solr/Client");
var DocQueries = require("PolicyFeed/Crawler/DocQueries");

jsonfs.addTrigger("afterWrite", "/docs/", SolrClient.onItemChange);
jsonfs.addTrigger("afterRemove", "/docs/", SolrClient.onItemChange);
jsonfs.addTrigger("afterCreate", "/docs/", DocQueries.runQueries);

// Start Crawler:
var Crawler = require("PolicyFeed/Crawler");
Crawler.init(require("config")["PolicyFeed/Crawler"]);
Crawler.checkUpdates();

// Some vars and functions for easier error handling:
var CrawlerErrors = require("PolicyFeed/Crawler/Errors");
var CrawlerFailures = require("PolicyFeed/Crawler/Failures");

function retryUrl(url) {
    print(url);
    CrawlerErrors.removeUrl(url);
    Crawler.reindexUrl(url);
}

function failUrl(url, reason) {
    reason = reason || "Unknown (manual).";
    CrawlerErrors.removeUrl(url);
    CrawlerFailures.write(false, { url: url, reason: reason });
}

function retryAllUrls() {
    return CrawlerErrors.getUrlStats().map(function(item) {
            try {
                retryUrl(item.url);
                return true;
            } catch (e) {
                print(e);
                return false;
            }
    });
{'fraction_non_alphanumeric': 0.14322664080181055, 'fraction_numerical': 0.0, 'mean_word_length': 3.88783570300158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4087009', 'n_tokens_mistral': 953, 'n_tokens_neox': 944, 'n_words': 192}