<commit_before>#!/bin/bash

source ./munge/.env

read -p "DANGER! You are about to remove all images and videos. Are you sure? " -n 1 -r
echo    # (optional) move to a new line
if [[ $REPLY =~ ^[Yy]$ ]]
then
  cp /dev/null munge/munge.log
  cp /dev/null munge/ftp.log
  rm collection/{cams,log}.txt
  rm output/{images,videos,write,tmp}/*
  ssh <EMAIL> "rm ${GTP_BASE_DIR}{images,videos}/*; rm ${GTP_BASE_DIR}count.json"
fi<commit_msg>Updated clear<commit_after>#!/bin/bash

source ./munge/.env

read -p "DANGER! You are about to remove all images and videos. Are you sure? " -n 1 -r
echo    # (optional) move to a new line
if [[ $REPLY =~ ^[Yy]$ ]]
then
  cp /dev/null munge/munge.log
  cp /dev/null munge/ftp.log
  rm collection/{cams,log}.txt
  rm output/{images,videos}/*
  rm output/{write,tmp}/ -r
  rm output/count.json
  ssh <EMAIL> "rm ${GTP_BASE_DIR}{images,videos}/*; rm ${GTP_BASE_DIR}count.json"
fi
{'fraction_non_alphanumeric': 0.16245883644346873, 'fraction_numerical': 0.0021953896816684962, 'mean_word_length': 3.56, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '543408', 'n_tokens_mistral': 375, 'n_tokens_neox': 362, 'n_words': 103}