<commit_before>#!/usr/bin/env bash

# Run wordcount locally
python big_data/wordcount.py ../data/wikipedia/en_perline001.txt > results_local_wordcount.txt

# Run wordcount on Amazon
# IMPORTANT: you must export your AWS_ACCESS_KEY and AWS_SECRET_KEY variables before running this!
# python big_data/wordcount.py -r emr ../data/wikipedia/en_perline001.txt --num-ec2-instances 2 --aws-region eu-west-1 > results_AWS_wordcount.txt

# Run language detection locally
python big_data/kmers.py ../data/wikipedia/en_perline001.txt > output.en.txt
python big_data/kmers.py ../data/wikipedia/pt_perline01.txt > output.pt.txt
python big_data/postprocess.py

# Run language detection on Amazon
# python big_data/kmers.py -r emr s3://lxmls-labs/en_perline01.txt --num-ec2-instances 10 --aws-region eu-west-1 > output.en.txt
# python big_data/kmers.py -r emr s3://lxmls-labs/pt_perline10.txt --num-ec2-instances 10 --aws-region eu-west-1 > output.pt.txt
# python big_data/postprocess.py<commit_msg>Updated test script for day5<commit_after>#!/usr/bin/env bash

echo "------------"
echo "Exercise 5.1"
echo "------------"

# Run wordcount locally 
python lxmls/big_data/wordcount.py data/wikipedia/en_perline001.txt > results.txt
grep '\<will\>' results.txt

echo "------------"
echo "Exercise 5.2"
echo "------------"

echo "# Run the amazon part (see in $0, Line $LINENO)"
# Run wordcount on Amazon ...
echo "# Remote login with"
echo ""
echo "ssh <username>@ec2-46-137-27-37.eu-west-1.compute.amazonaws.com"
echo ""
echo "# Use <username> and <password> given in the handout, then run"
echo ""
echo "cd wordcount"
echo "python wordcount.py -r emr en_perline001.txt > results2.txt"
echo "grep '\<will\>' results2.txt"
echo ""
echo "python wordcount.py -r emr s3://lxmls-labs/pt_perline10.txt"
echo ""

echo "------------"
echo "Exercise 5.3"
echo "------------"

# Run language detection locally
python lxmls/big_data/kmers.py ./data/wikipedia/en_perline001.txt > output.en.txt
python lxmls/big_data/kmers.py ./data/wikipedia/pt_perline01.txt > output.pt.txt
python lxmls/big_data/postprocess.py

# Note: I copied by scp the scripts there to do the test, the students do not know this (probably). How are they going to build the post processor from day1 code, al by hand? 
# scp lxmls/big_data/kmers.py <username>@ec2-46-137-27-37.eu-west-1.compute.amazonaws.com:./wordcount/
# scp lxmls/big_data/postprocess.py <username>@ec2-46-137-27-37.eu-west-1.compute.amazonaws.com:./wordcount/

echo "Run language detection on Amazon (see in $0, Line $LINENO)"
echo ""
echo "python kmers.py -r emr s3://lxmls-labs/en_perline01.txt --num-ec2-instances 10 --aws-region eu-west-1 > output.en.txt"
echo "python kmers.py -r emr s3://lxmls-labs/pt_perline10.txt --num-ec2-instances 10 --aws-region eu-west-1 > output.pt.txt"
echo "python postprocess.py"
{'fraction_non_alphanumeric': 0.15646017699115045, 'fraction_numerical': 0.03575221238938053, 'mean_word_length': 3.6176470588235294, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4695141', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1018, 'n_words': 254}