<commit_before>CREATE DATABASE crawl;

CREATE USER 'tesco'@'localhost' IDENTIFIED BY 'bank';
GRANT ALL PRIVILEGES ON crawl.tescobank TO 'tesco'@'localhost' IDENTIFIED BY PASSWORD '<PASSWORD>' WITH GRANT OPTION;
FLUSH PRIVILEGES;

DROP TABLE tescobank;

CREATE TABLE tescobank (
 url VARCHAR(512),
 status VARCHAR(16) DEFAULT 'DISCOVERED',
 nextfetchdate TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 metadata TEXT,
 bucket SMALLINT DEFAULT 0,
 PRIMARY KEY(url)
);

INSERT INTO tescobank (url, metadata) VALUES ("http://www.tescobank.com/sitemap.xml","isSitemap=true");

# configure CloudSearch domain

The script `createCSDomain.sh` can be used with the AWS CLI to configure a CloudSearch domain.

# run

`mvn clean install`

`mvn exec:java -Dexec.mainClass=com.digitalpebble.stormcrawler.CrawlTopology -Dexec.args="-conf crawler-conf.yaml -local`

or with Storm installed 

`storm jar target/stormcrawler-tescobank-0.1-SNAPSHOT.jar com.digitalpebble.stormcrawler.CrawlTopology -conf crawler-conf.yaml -local`

remove `-local` to run in deployed mode.
<commit_msg>README mentions tutorial<commit_after>Resources for a StormCrawler tutorial [INSERT URL HERE]


{'fraction_non_alphanumeric': 0.0947002606429192, 'fraction_numerical': 0.0069504778453518675, 'mean_word_length': 5.6976744186046515, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1282190', 'n_tokens_mistral': 417, 'n_tokens_neox': 391, 'n_words': 112}