<commit_before>#!/bin/bash

# simple snapshotting mechanism

MIRROR_PATH=$1
DAYS_RETENTION=$2
SNAPSHOT_NAME=`date +snapshot-%Y-%m-%d`

cd ${MIRROR_PATH}

# taking a daily snapshot, if it already exists, exit
# this allows multiple mirror updates per day, but
# only one snapshot
if [ -d "${SNAPSHOT_NAME}" ]; then
	exit 0
fi

# packages are in a central place, so just clone the index
echo "Creating simple index snapshot ${SNAPSHOT_NAME}"
cp -r simple ${SNAPSHOT_NAME}

# remove everything but the last $DAYS_RETENTION snapshots
rm -rf `ls -1d snapshot*| head -n -	${DAYS_RETENTION}`

# snapshots are now available at http://your-mirror.network/snapshot-DATE/
<commit_msg>Fix silly tab issue<commit_after>#!/bin/bash

# simple snapshotting mechanism

MIRROR_PATH=$1
DAYS_RETENTION=$2
SNAPSHOT_NAME=`date +snapshot-%Y-%m-%d`

cd ${MIRROR_PATH}

# taking a daily snapshot, if it already exists, exit
# this allows multiple mirror updates per day, but
# only one snapshot
if [ -d "${SNAPSHOT_NAME}" ]; then
	exit 0
fi

# packages are in a central place, so just clone the index
echo "Creating simple index snapshot ${SNAPSHOT_NAME}"
cp -r simple ${SNAPSHOT_NAME}

# remove everything but the last $DAYS_RETENTION snapshots
rm -rf `ls -1d snapshot*| head -n -${DAYS_RETENTION}`

# snapshots are now available at http://your-mirror.network/snapshot-DATE/

{'fraction_non_alphanumeric': 0.1080680977054034, 'fraction_numerical': 0.005921539600296077, 'mean_word_length': 4.802575107296137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5752177', 'n_tokens_mistral': 467, 'n_tokens_neox': 470, 'n_words': 161}