<commit_before>#!/bin/sh

# bot para scrape e push dos deputados

DATAPKG_REPO_URL="<EMAIL>:centraldedados/parlamento-deputados.git"
SCRAPER_REPO_URL="<EMAIL>:transparenciahackday/scraper-deputados.git"
SCRAPER_SCRIPT="depscrap -f csv -v -p 1 -o ../deputados.csv"

SCRAPER_DIR="scraper-deputados"
DATAPKG_DIR="datapkg-deputados"
OUTFILENAME="deputados.csv"
COMMIT_MSG="Auto update"

# site está online?
wget -q --tries=10 --timeout=20 --spider http://www.parlamento.pt
if [ $? -eq 0 ]; then
        echo "Online"
else
        echo "Offline"
fi

if [ ! -d "$SCRAPER_DIR" ]; then
    # clonar repos
    git clone $SCRAPER_REPO_URL $SCRAPER_DIR --quiet
    git clone $DATAPKG_REPO_URL $DATAPKG_DIR --quiet
else
    # pull
    cd $SCRAPER_DIR; git pull --quiet; cd ..
    cd $DATAPKG_DIR; git pull --quiet; cd ..
fi


# commit e push se os ficheiros forem diferentes
# src: http://stackoverflow.com/questions/12900538/unix-fastest-way-to-tell-if-two-files-are-the-same
cd $SCRAPER_DIR && ./$SCRAPER_SCRIPT && cd .. && \
cmp --silent $OUTFILENAME $DATAPKG_DIR/data/$OUTFILENAME || \
  cp $OUTFILENAME $DATAPKG_DIR/data -f && \
  cd $DATAPKG_DIR && git commit -am "$COMMIT_MSG" && git push && \
  cd ..<commit_msg>Improved autoscrape_deputados.sh.<commit_after>#!/bin/sh

# This script
# 1. automatically scraps parlamento's website data
# 2. commits and pushes the data to a github repository

DATAPKG_REPO_URL="git<EMAIL>:publicos_pt/parlamento-deputados.git"
SCRAPER_REPO_URL="<EMAIL>:publicos_pt/scraper-deputados.git"

SCRAPER_DIR="scraper-deputados"
DATAPKG_DIR="datapkg-deputados"
OUTFILENAME="deputados.csv"
COMMIT_MSG="Automatically updated the list of deputies via scraper."

git pull origin master --force
cd ..
# is the website online?
wget -q --tries=10 --timeout=20 --spider http://www.parlamento.pt
if [ $? -eq 0 ]; then
        echo "Online"
else
        echo "Offline"
fi

if [ ! -d "$DATAPKG_REPO_URL" ]; then
    # clonar repos
    git clone $DATAPKG_REPO_URL $DATAPKG_DIR --quiet
else
    # pull
    cd $DATAPKG_DIR; git pull --quiet; cd ..
fi

# run the script
cd $SCRAPER_DIR
python -m depscrape -f csv -v -p 1 --end 5000 -o ../$OUTFILENAME
cd ..

# if there is no difference, it will not commit anyway.
cd $DATAPKG_DIR && git commit -am "$COMMIT_MSG" && git push
{'fraction_non_alphanumeric': 0.1320754716981132, 'fraction_numerical': 0.011408512505484861, 'mean_word_length': 3.6817248459958933, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5795413', 'n_tokens_mistral': 958, 'n_tokens_neox': 886, 'n_words': 219}