<commit_before>#!/bin/bash
# 
# EMR bootstrap task for our gdal stuff.

set -e

bucket=reddconfig
fwtools=FWTools-linux-x86_64-4.0.0.tar.gz
native=linuxnative.tar.gz
sources=/etc/apt/sources.list

# Install HFD4
sudo chown hadoop $sources
echo "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu maverick universe  multiverse" > $sources
sudo apt-get update
sudo apt-get install -y --force-yes libhdf4-dev

# FWTOOLS
wget -S -T 10 -t 5 http://$bucket.s3.amazonaws.com/$fwtools
sudo mkdir -p /usr/local/fwtools
sudo tar -C /usr/local/fwtools -xzf $fwtools

# Native Bindings
wget -S -T 10 -t 5 http://$bucket.s3.amazonaws.com/$native
sudo mkdir -p /home/hadoop/native
sudo tar -C /home/hadoop/native -xzf $native

echo "export LD_LIBRARY_PATH=/usr/local/fwtools/usr/lib" >> /home/hadoop/conf/hadoop-user-env.sh<commit_msg>more script updates.<commit_after>#!/bin/bash
# 
# EMR bootstrap task for our gdal stuff.

set -e

bucket=reddconfig
fwtools=FWTools-linux-x86_64-4.0.0.tar.gz
native=linuxnative.tar.gz
sources=/etc/apt/sources.list

# Install HFD4
sudo chown hadoop $sources
echo "deb http://us-east-1.ec2.archive.ubuntu.com/ubuntu maverick universe  multiverse" > $sources
sudo apt-get update
sudo apt-get install -y --force-yes libhdf4-dev

# FWTOOLS
wget -S -T 10 -t 5 http://$bucket.s3.amazonaws.com/$fwtools
sudo mkdir -p /usr/local/fwtools
sudo tar -C /usr/local/fwtools --strip-components=2 -xvzf $fwtools
sudo chown hadoop $fwtools

# Native Bindings
wget -S -T 10 -t 5 http://$bucket.s3.amazonaws.com/$native
sudo mkdir -p /home/hadoop/native
sudo tar -C /home/hadoop/native --strip-components=2 -xvzf $native
sudo chown hadoop $native

echo "export LD_LIBRARY_PATH=/usr/local/fwtools/usr/lib" >> /home/hadoop/conf/hadoop-user-env.sh
{'fraction_non_alphanumeric': 0.13688430698739978, 'fraction_numerical': 0.022909507445589918, 'mean_word_length': 3.7994505494505493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3651017', 'n_tokens_mistral': 724, 'n_tokens_neox': 663, 'n_words': 165}