<commit_before>#!/bin/sh

# make sure we exit immediately when there's an error, for CI:
set -e

# start a webserver
(python -u ./mock-webserver.py 2>&1 | grep -v '" 200 ') &
# give it a chance to bind
sleep 1

# if COVERAGE is set, use it, else python
if [ -z "$COVERAGE" ]; then COVERAGE=python; fi

NOCH=--no-confighome

echo
echo test-deep
echo
$COVERAGE ../scripts/crawl.py --configfile test-deep.yml $NOCH
# tests against the logfiles
grep -q "/denied/" robotslog.jsonl || (echo "FAIL: nothing about /denied/ in robotslog"; exit 1)
(grep "/denied/" crawllog.jsonl | grep -q -v '"robots"' ) && (echo "FAIL: should not have seen /denied/ in crawllog.jsonl"; exit 1)
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-wide
echo
$COVERAGE ../scripts/crawl.py --configfile test-wide.yml --config Testing.doesnotexist:1 $NOCH
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-wide with save and load first half
echo
rm -f test-wide-save
cat test-wide.yml test-wide-save.yml > test-wide-tmp.yml
$COVERAGE ../scripts/crawl.py --configfile test-wide-tmp.yml --no-test --config Crawl.MaxCrawledUrls:5 --config Crawl.MaxWorkers:3 $NOCH
rm -f test-wide-tmp.yml

ls -l test-wide-save

echo
echo test wide save and load second half: load
echo
$COVERAGE ../scripts/crawl.py --configfile test-wide.yml --load test-wide-save $NOCH
rm -f test-wide-save
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-failures
echo
$COVERAGE ../scripts/crawl.py --configfile test-failures.yml --config error --config error:1 --config error.error:1 $NOCH
rm -f robotslog.jsonl crawllog.jsonl

# tear down the webserver. fails in travis, so ignore
kill %1 || true

echo
echo run_burner
echo
<commit_msg>try a more reliable way to kill the mock webserver<commit_after>#!/bin/sh

# make sure we exit immediately when there's an error, for CI:
set -e

# start a webserver
(python -u ./mock-webserver.py 2>&1 | grep -v '" 200 ') &
# give it a chance to bind
sleep 1

# if COVERAGE is set, use it, else python
if [ -z "$COVERAGE" ]; then COVERAGE=python; fi

NOCH=--no-confighome

echo
echo test-deep
echo
$COVERAGE ../scripts/crawl.py --configfile test-deep.yml $NOCH
# tests against the logfiles
grep -q "/denied/" robotslog.jsonl || (echo "FAIL: nothing about /denied/ in robotslog"; exit 1)
(grep "/denied/" crawllog.jsonl | grep -q -v '"robots"' ) && (echo "FAIL: should not have seen /denied/ in crawllog.jsonl"; exit 1)
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-wide
echo
$COVERAGE ../scripts/crawl.py --configfile test-wide.yml --config Testing.doesnotexist:1 $NOCH
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-wide with save and load first half
echo
rm -f test-wide-save
cat test-wide.yml test-wide-save.yml > test-wide-tmp.yml
$COVERAGE ../scripts/crawl.py --configfile test-wide-tmp.yml --no-test --config Crawl.MaxCrawledUrls:5 --config Crawl.MaxWorkers:3 $NOCH
rm -f test-wide-tmp.yml

ls -l test-wide-save

echo
echo test wide save and load second half: load
echo
$COVERAGE ../scripts/crawl.py --configfile test-wide.yml --load test-wide-save $NOCH
rm -f test-wide-save
rm -f robotslog.jsonl crawllog.jsonl

echo
echo test-failures
echo
$COVERAGE ../scripts/crawl.py --configfile test-failures.yml --config error --config error:1 --config error.error:1 $NOCH
rm -f robotslog.jsonl crawllog.jsonl

# tear down the mock webserver a couple of ways
kill %1 || true
pkill -e -f mock-webserver.py || true

echo
echo run_burner
echo

{'fraction_non_alphanumeric': 0.1270026216137489, 'fraction_numerical': 0.00815613166326828, 'mean_word_length': 3.5604249667994687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2793902', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1230, 'n_words': 420}