<commit_before>#!/bin/sh

# fetch.sh
# <NAME>
# Script to download reference pages from Arduino website and change links
# to point to local copies of the pages.  A terrible hack.

mkdir reference
cd reference
curl http://www.arduino.cc/en/Main/Howto -o howto.html
curl http://www.arduino.cc/en/Reference/HomePage -o index.html
curl http://www.arduino.cc/en/pub/skins/arduino/arduino.css -o arduino.css
for i in `grep -o "http://www.arduino.cc/en/Reference/[^']*" index.html | sort -u | grep -v '?' | cut -d '/' -f 6`; do curl http://www.arduino.cc/en/Reference/$i -o $i.html; done
perl -i -pe "s|http://www.arduino.cc/en/Reference/[^?\"']*\?[^'\"]*|#|g" *.html
perl -i -pe "s|http://www.arduino.cc/en/Reference/([^']*)|\1.html|g" *.html
perl -i -pe "s|http://www.arduino.cc/en/pub/skins/arduino/arduino.css|arduino.css|g" *.html
perl -i -pe "s|HomePage.html|index.html|g" *.html
perl -i -pe "s|href=\"/\"|href=\"http://www.arduino.cc/\"|g" *.html
cd ..
zip -r shared/reference.zip reference
rm -rf reference<commit_msg>Now including the FAQ (but no images) in the reference.<commit_after>#!/bin/sh

# fetch.sh
# <NAME>
# Script to download reference pages from Arduino website and change links
# to point to local copies of the pages.  A terrible hack.

mkdir reference
cd reference
curl http://www.arduino.cc/en/Main/Howto -o howto.html
curl http://www.arduino.cc/en/Main/FAQ -o FAQ.html
curl http://www.arduino.cc/en/Reference/HomePage -o index.html
curl http://www.arduino.cc/en/pub/skins/arduino/arduino.css -o arduino.css
for i in `grep -o "http://www.arduino.cc/en/Reference/[^']*" index.html | sort -u | grep -v '?' | cut -d '/' -f 6`; do curl http://www.arduino.cc/en/Reference/$i -o $i.html; done
perl -i -pe "s|http://www.arduino.cc/en/Reference/[^?\"']*\?[^'\"]*|#|g" *.html
perl -i -pe "s|http://www.arduino.cc/en/Reference/([^']*)|\1.html|g" *.html
perl -i -pe "s|http://www.arduino.cc/en/pub/skins/arduino/arduino.css|arduino.css|g" *.html
perl -i -pe "s|HomePage.html|index.html|g" *.html
perl -i -pe "s|href=\"/\"|href=\"http://www.arduino.cc/\"|g" *.html
cd ..
zip -r shared/reference.zip reference
rm -rf reference
{'fraction_non_alphanumeric': 0.21237693389592124, 'fraction_numerical': 0.001875293014533521, 'mean_word_length': 3.4458333333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 19, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175025', 'n_tokens_mistral': 906, 'n_tokens_neox': 848, 'n_words': 202}