<commit_before><commit_msg>#26: Download data for netherlands<commit_after>#! /bin/sh

OSMOSIS=${HOME}/osmosis-0.43.1
STORAGE=${HOME}/Dropbox/osmdata
TMPDIR=${HOME}/tmp/osm-place-diyanet-nl
WEBDATA=/home/tomcat/osm-mosques/data

COUNTRY=netherlands

SOURCE=diyanet-nl

extract_data() {

    country=${COUNTRY}
    county=${SOURCE}

    FILE=${TMPDIR}/${country}-${source}.html

    mkdir -p ${STORAGE}/${country}-${source}/${MONTH}/${DAY}

    cp -f \
	${FILE} \
	${STORAGE}/${country}-${source}/${MONTH}/${DAY}/${country}-${source}.html

    cp -f \
	${FILE} \
	${WEBDATA}/${country}-${source}.html
}


country=${COUNTRY}
source=${SOURCE}


mkdir -p ${TMPDIR}
cd ${TMPDIR}

FILE=${TMPDIR}/${country}-${source}.html

wget "http://www.diyanet.nl/hdv-cami-hizmetleri/sube-cami-adresleri/" -O ${FILE} \
	> ${FILE}.out 2> ${FILE}.err

if [ -a ${FILE} ] 
then
    if [ -s ${FILE} ]
    then
	MONTH=$(date +%Y%m --reference ${FILE})
	DAY=$(date +%Y%m%d --reference ${FILE})

	extract_data
    fi
fi

# find ${STORAGE}/${country} -type f -a -mtime +14 | xargs --no-run-if-empty rm
# find ${STORAGE}/${country} -type d -a -empty | xargs --no-run-if-empty rmdir
{'fraction_non_alphanumeric': 0.234375, 'fraction_numerical': 0.0078125, 'mean_word_length': 3.706122448979592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2461181', 'n_tokens_mistral': 488, 'n_tokens_neox': 478, 'n_words': 80}