<commit_before>#!/bin/bash

set -eu

function usage_exit {
  cat <<EOS
Usage:
  $0 pull-dc
  $0 zip
  $0 pull-ec2 <instance-id>
EOS
  exit 1
}

if [ $# -eq 0 ]; then usage_exit; fi

function pull_dc {
  target=./log
  sudo docker ps --format '{{.Names}}' \
    | grep '^catcluster_cat' \
    | xargs -n 1 -I {} sudo docker cp {}:/opt/cat-cluster/log/profile.log $target/{}.profile.log
}

subcommand=$1
shift

case $subcommand in
  pull-dc) pull_dc "$@" ;;
  *) usage_exit ;;
esac<commit_msg>Add package subcommand<commit_after>#!/bin/bash

set -eu

function usage_exit {
  cat <<EOS
Usage:
  $0 pull-dc
  $0 package
  $0 pull-ec2 <instance-id>
EOS
  exit 1
}

if [ $# -eq 0 ]; then usage_exit; fi

function pull_dc {
  target=./log
  sudo docker ps --format '{{.Names}}' \
    | grep '^catcluster_cat' \
    | xargs -n 1 -I {} sudo docker cp {}:/opt/cat-cluster/log/profile.log $target/{}.profile.log
}

function package {
  base=./log
  target=$base/logs.gz
  tar zvcf $target --exclude "*.gz" --exclude ".gitkeep" --exclude "profile.log" $base
  echo Created $target
}

subcommand=$1
shift

case $subcommand in
  pull-dc) pull_dc "$@" ;;
  package) package "$@" ;;
  *) usage_exit ;;
esac
{'fraction_non_alphanumeric': 0.1689075630252101, 'fraction_numerical': 0.013445378151260505, 'mean_word_length': 2.97, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7256785', 'n_tokens_mistral': 494, 'n_tokens_neox': 458, 'n_words': 124}