<commit_before>[![NPM version](https://badge.fury.io/js/spider.js.png)](https://www.npmjs.com/package/spider.js)


# spider.js
#### A node module and command line tool for crawling a website for dead links, permanent and or fatal redirects, resource load issues, and script errors. It is based on the casperJS navigation scripting and testing utility.

## Usage


#### Node Module
```js
var spider = require( "spider.js" );

spider( options );
```

#### Command line
```
spiderjs --url=http://example.com [, option1 ] [, option2 ]
```

## Examples


#### Node Module
```js
var spider = require( "spider.js" );

spider( {
	url: "http://example.com",
	ignore: "error.html",
	redirectError: false
} );
```

#### Command line
```
spiderjs --url=http://example.com -ignore=error.html --redirectError=false
```<commit_msg>Fix typo in command line example<commit_after>[![NPM version](https://badge.fury.io/js/spider.js.png)](https://www.npmjs.com/package/spider.js)


# spider.js
#### A node module and command line tool for crawling a website for dead links, permanent and or fatal redirects, resource load issues, and script errors. It is based on the casperJS navigation scripting and testing utility.

## Usage


#### Node Module
```js
var spider = require( "spider.js" );

spider( options );
```

#### Command line
```
spiderjs --url=http://example.com [, option1 ] [, option2 ]
```

## Examples


#### Node Module
```js
var spider = require( "spider.js" );

spider( {
	url: "http://example.com",
	ignore: "error.html",
	redirectError: false
} );
```

#### Command line
```
spiderjs --url=http://example.com --ignore=error.html --redirectError=false
```
{'fraction_non_alphanumeric': 0.17736077481840193, 'fraction_numerical': 0.002421307506053269, 'mean_word_length': 4.247619047619048, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7400181', 'n_tokens_mistral': 565, 'n_tokens_neox': 524, 'n_words': 157}