<commit_before>#!/usr/bin/env bash

AUTH_URL="https://authserver.mojang.com/authenticate"
REFRESH_URL="https://authserver.mojang.com/refresh"
WORLD_URL="https://mcoapi.minecraft.net/worlds"

#get config values
source configuration.conf

echo "Getting Minecraft authentication credentials..."

#get auth server temp access token
RESPONSE1=$(curl -s -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d "{
    \"requestUser\": true,
    \"agent\":\"Agent{name='Minecraft', version=1}\",
    \"username\":\"$MINECRAFT_USERNAME\",
    \"clientToken\":\"$<PASSWORD>\",
    \"password\":\"<PASSWORD>\"
}" "$AUTH_URL")

#parse and store temp access token
TEMP_ACCESS_TOKEN=$(echo $RESPONSE1 | jq -r '.accessToken')

#get auth server long term access token
RESPONSE2=$(curl -s -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d "{
    \"requestUser\": true,
    \"accessToken\":\"$TEMP_ACCESS_TOKEN\",
    \"clientToken\":\"$<PASSWORD>\",
    \"selectedProfile\":{
        \"id\": \"$MINECRAFT_PROFILE_ID\"
    }
}" "$REFRESH_URL")

#parse and store long term access token
LONG_ACCESS_TOKEN=$(echo $RESPONSE2 | jq -r '.accessToken')
MINECRAFT_USER=$(echo $RESPONSE2 | jq -r '.selectedProfile.name')

echo "Getting temporary download link for world backup..."

#get Realms server ID
RESPONSE3=$(curl -s -X GET -H "Content-Type: application/json" -H "Cookie: sid=token:$LONG_ACCESS_TOKEN:$MINECRAFT_PROFILE_ID;user=$MINECRAFT_USER;version=$MINECRAFT_VERSION" -H "Cache-Control: no-cache" -H "Pragma: no-cache" "$WORLD_URL")

#parse and store server ID
SERVER_ID=$(echo $RESPONSE3 | jq -r '.servers[0].id')
DAYS_LEFT=$(echo $RESPONSE3 | jq -r '.servers[0].daysLeft')

#get world backup URL
RESPONSE4=$(curl -s -X GET -H "Content-Type: application/json" -H "Cookie: sid=token:$LONG_ACCESS_TOKEN:$MINECRAFT_PROFILE_ID;user=$MINECRAFT_USER;version=$MINECRAFT_VERSION" -H "Cache-Control: no-cache" -H "Pragma: no-cache" "$WORLD_URL/$SERVER_ID/slot/$MINECRAFT_WORLD_NUM/download")

DOWNLOAD_LINK=$(echo $RESPONSE4 | jq -r '.downloadLink')

DATE=`date +%Y-%m-%d`
FILE_PATH="backups/mcr_world_$DATE.tar.gz"

echo "Downloading world backup. This may take several minutes..."
wget $DOWNLOAD_LINK -O "$FILE_PATH"

echo "Extracting world backup..."
tar -xvf "$FILE_PATH" world

echo "Removing backups older than 7 days..."
find ./backups/*.tar.gz -mtime +7 -type f -delete

echo "Generating map..."

DATE=`date +%Y-%m-%d:%H:%M:%S`
echo "Started map generation at $DATE"

bin/mapcrafter/src/mapcrafter -c render.conf -j "$HW_THREADS"

DATE=`date +%Y-%m-%d:%H:%M:%S`
echo "Finished map generation at $DATE"

echo "Uploading map to web server..."

s3_sync
{
    echo "Running S3 sync..."
    s3cmd sync --delete-removed ./output/* $S3_URL #$S3_URL must include trailing slash!
    echo "S3 sync completed!"
}

#TODO: change to use configuration.conf to specify username and password info
ftp_sync
{
    echo "Running FTP sync..."
    ncftpput -R -u "$FTP_USERNAME" -p "$FTP_PASSWORD" "$FTP_SERVER" "$FTP_PATH" ./output
    echo "FTP sync completed!"
}<commit_msg>Fixed typo in sync methods<commit_after>#!/usr/bin/env bash

AUTH_URL="https://authserver.mojang.com/authenticate"
REFRESH_URL="https://authserver.mojang.com/refresh"
WORLD_URL="https://mcoapi.minecraft.net/worlds"

#get config values
source configuration.conf

echo "Getting Minecraft authentication credentials..."

#get auth server temp access token
RESPONSE1=$(curl -s -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d "{
    \"requestUser\": true,
    \"agent\":\"Agent{name='Minecraft', version=1}\",
    \"username\":\"$MINECRAFT_USERNAME\",
    \"clientToken\":\"$<PASSWORD>\",
    \"password\":\"<PASSWORD>\"
}" "$AUTH_URL")

#parse and store temp access token
TEMP_ACCESS_TOKEN=$(echo $RESPONSE1 | jq -r '.accessToken')

#get auth server long term access token
RESPONSE2=$(curl -s -X POST -H "Content-Type: application/json" -H "Cache-Control: no-cache" -d "{
    \"requestUser\": true,
    \"accessToken\":\"$TEMP_ACCESS_TOKEN\",
    \"clientToken\":\"$<PASSWORD>\",
    \"selectedProfile\":{
        \"id\": \"$MINECRAFT_PROFILE_ID\"
    }
}" "$REFRESH_URL")

#parse and store long term access token
LONG_ACCESS_TOKEN=$(echo $RESPONSE2 | jq -r '.accessToken')
MINECRAFT_USER=$(echo $RESPONSE2 | jq -r '.selectedProfile.name')

echo "Getting temporary download link for world backup..."

#get Realms server ID
RESPONSE3=$(curl -s -X GET -H "Content-Type: application/json" -H "Cookie: sid=token:$LONG_ACCESS_TOKEN:$MINECRAFT_PROFILE_ID;user=$MINECRAFT_USER;version=$MINECRAFT_VERSION" -H "Cache-Control: no-cache" -H "Pragma: no-cache" "$WORLD_URL")

#parse and store server ID
SERVER_ID=$(echo $RESPONSE3 | jq -r '.servers[0].id')
DAYS_LEFT=$(echo $RESPONSE3 | jq -r '.servers[0].daysLeft')

#get world backup URL
RESPONSE4=$(curl -s -X GET -H "Content-Type: application/json" -H "Cookie: sid=token:$LONG_ACCESS_TOKEN:$MINECRAFT_PROFILE_ID;user=$MINECRAFT_USER;version=$MINECRAFT_VERSION" -H "Cache-Control: no-cache" -H "Pragma: no-cache" "$WORLD_URL/$SERVER_ID/slot/$MINECRAFT_WORLD_NUM/download")

DOWNLOAD_LINK=$(echo $RESPONSE4 | jq -r '.downloadLink')

DATE=`date +%Y-%m-%d`
FILE_PATH="backups/mcr_world_$DATE.tar.gz"

echo "Downloading world backup. This may take several minutes..."
wget $DOWNLOAD_LINK -O "$FILE_PATH"

echo "Extracting world backup..."
tar -xvf "$FILE_PATH" world

echo "Removing backups older than 7 days..."
find ./backups/*.tar.gz -mtime +7 -type f -delete

echo "Generating map..."

DATE=`date +%Y-%m-%d:%H:%M:%S`
echo "Started map generation at $DATE"

bin/mapcrafter/src/mapcrafter -c render.conf -j "$HW_THREADS"

DATE=`date +%Y-%m-%d:%H:%M:%S`
echo "Finished map generation at $DATE"

echo "Uploading map to web server..."

function s3_sync {
    echo "Running S3 sync..."
    s3cmd sync --delete-removed ./output/* $S3_URL #$S3_URL must include trailing slash!
    echo "S3 sync completed!"
}

#TODO: change to use configuration.conf to specify username and password info
function ftp_sync {
    echo "Running FTP sync..."
    ncftpput -R -u "$FTP_USERNAME" -p "$FTP_PASSWORD" "$FTP_SERVER" "$FTP_PATH" ./output
    echo "FTP sync completed!"
}
{'fraction_non_alphanumeric': 0.1686280831855554, 'fraction_numerical': 0.006770917298081574, 'mean_word_length': 4.446883230904302, 'pattern_counts': {'":': 20, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3590198', 'n_tokens_mistral': 2286, 'n_tokens_neox': 2231, 'n_words': 566}