<commit_before>#!/bin/bash

cd $(echo $0 | sed 's#/[^/]*$##')/..

echo "Downloading..."
echo "--------------"
source /usr/local/bin/virtualenvwrapper.sh
workon twitter-parls
#./download_twitter.py "AssembleeNat" "les-députés"
./download_twitter.py "TwitterPolFR" "d-put-s-2017-22"
./download_twitter.py "Senat" "senateurs"
curl -sL "http://www.nosdeputes.fr/deputes/json" > .cache/deputes.json
curl -sL "http://www.nossenateurs.fr/senateurs/json" > .cache/senateurs.json

echo
echo "Associating..."
echo "--------------"
#./associate_twitter.py deputes .cache/twitter-AssembleeNat.json
./associate_twitter.py deputes .cache/twitter-TwitterPolFR.json
./associate_twitter.py senateurs .cache/twitter-Senat_Info.json
echo
<commit_msg>change source list twitter députés<commit_after>#!/bin/bash

cd $(echo $0 | sed 's#/[^/]*$##')/..

echo "Downloading..."
echo "--------------"
source /usr/local/bin/virtualenvwrapper.sh
workon twitter-parls
#./download_twitter.py "AssembleeNat" "les-députés"
./download_twitter.py "lcp" "d-put-s-xve-l-gislature"
./download_twitter.py "Senat" "senateurs"
curl -sL "http://www.nosdeputes.fr/deputes/json" > .cache/deputes.json
curl -sL "http://www.nossenateurs.fr/senateurs/json" > .cache/senateurs.json

echo
echo "Associating..."
echo "--------------"
#./associate_twitter.py deputes .cache/twitter-AssembleeNat.json
./associate_twitter.py deputes .cache/twitter-lcp.json
./associate_twitter.py senateurs .cache/twitter-Senat.json
echo

{'fraction_non_alphanumeric': 0.20776566757493187, 'fraction_numerical': 0.005449591280653951, 'mean_word_length': 3.678343949044586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2037598', 'n_tokens_mistral': 569, 'n_tokens_neox': 535, 'n_words': 92}