<commit_before>#!/bin/bash

set -e
set -x

# put the log files in manta, because OBVIOUSLY
export MANTA_KEY_ID=55:5e:9a:bc:42:59:df:cb:ad:00:54:f6:59:53:20:83
export MANTA_USER=npm
export MANTA_URL=https://us-east.manta.joyent.com

id=$1
cmd="ls ~ubuntu/hosting/servers/isaacs/db/log/*.1"
d=$(date '+%Y-%m-%d')

for ip in $(dig +short isaacs.iriscouch.com | grep ^[0-9]); do
  for file in `ssh -i $id isaacs@$ip "$cmd"`; do
    gzfile=$(basename $file).gz
    ssh -i $id isaacs@$ip "cat $file | gzip" > $gzfile &&\
    mput -f $gzfile /npm/stor/logs/$d.$gzfile &&\
    rm -f $gzfile
  done
done
<commit_msg>use proper manta creds in cron.sh<commit_after>#!/bin/bash

set -e
set -x

# put the log files in manta, because OBVIOUSLY
export MANTA_KEY_ID=55:5e:9a:bc:42:59:df:cb:ad:00:54:f6:59:53:20:83
export MANTA_USER=npm
export MANTA_URL=https://us-east.manta.joyent.com

. ~node/manta.env

id=$1
cmd="ls ~ubuntu/hosting/servers/isaacs/db/log/*.1"
d=$(date '+%Y-%m-%d')

for ip in $(dig +short isaacs.iriscouch.com | grep ^[0-9]); do
  for file in `ssh -i $id isaacs@$ip "$cmd"`; do
    gzfile=$(basename $file).gz
    ssh -i $id isaacs@$ip "cat $file | gzip" > $gzfile &&\
    mput -f $gzfile /npm/stor/logs/$d.$gzfile &&\
    rm -f $gzfile
  done
done

{'fraction_non_alphanumeric': 0.19074221867517957, 'fraction_numerical': 0.03990422984836393, 'mean_word_length': 3.7680608365019013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2916428', 'n_tokens_mistral': 613, 'n_tokens_neox': 575, 'n_words': 128}