<commit_before>#!/bin/bash

# Run from the project root as ./scripts/run-100.sh
rm -f results.csv
rm -f error.log
rm -f domains-with-ads-txt.txt
rm -f domains-without-ads-txt.txt

java -jar "./target/uberjar/ads-txt-crawler-standalone.jar" -t ./doc/top-100-programmatic-domains.txt >results.csv 2>error.log


# Build list of domains with Ads.txt files
cat results.csv | cut -d, -f1 | sort | uniq > domains-with-ads-txt.txt

# Build list of domains without Ads.txt files
cat error.log | sed "s/.*http:\/\///" | cut -d/ -f1 | uniq | sort > domains-without-ads-txt.txt



<commit_msg>comment when done<commit_after>#!/bin/bash

# Run from the project root as ./scripts/run-100.sh
rm -f results.csv
rm -f error.log
rm -f domains-with-ads-txt.txt
rm -f domains-without-ads-txt.txt

java -jar "./target/uberjar/ads-txt-crawler-standalone.jar" -t ./doc/top-100-programmatic-domains.txt >results.csv 2>error.log


# Build list of domains with Ads.txt files
cat results.csv | cut -d, -f1 | sort | uniq > domains-with-ads-txt.txt

# Build list of domains without Ads.txt files
cat error.log | sed "s/.*http:\/\///" | cut -d/ -f1 | uniq | sort > domains-without-ads-txt.txt


echo "See the files results.csv, error.log, domains-with-ads-txt.txt and domains-without-ads-txt.txt"
echo "done"
{'fraction_non_alphanumeric': 0.15492957746478872, 'fraction_numerical': 0.014084507042253521, 'mean_word_length': 3.1258064516129034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5787774', 'n_tokens_mistral': 488, 'n_tokens_neox': 466, 'n_words': 131}