<commit_before>#!/bin/bash
# LinuxGSM core_dl.sh function
# Author: <NAME>
# Contributor: UltimateByte
# Website: https://gameservermanagers.com
# Description: Deals with all downloads for LinuxGSM.

# remote_fileurl: The URL of the file: http://example.com/dl/File.tar.bz2
# local_filedir: location the file is to be saved: /home/server/lgsm/tmp
# local_filename: name of file (this can be different from the url name): file.tar.bz2
# chmodx: Optional, set to "chmodx" to make file executable using chmod +x
# run: Optional, set run to execute the file after download
# forcedl: Optional, force re-download of file even if exists
# md5: Optional, set an md5 sum and will compare it against the file.
#
# Downloads can be defined in code like so:
# fn_fetch_file "${remote_fileurl}" "${local_filedir}" "${local_filename}" "${chmodx}" "${run}" "${forcedl}" "${md5}"
# fn_fetch_file "http://example.com/file.tar.bz2" "/some/dir" "file.tar.bz2" "chmodx" "run" "forcedl" "10cd7353aa9d758a075c600a6dd193fd"

local commandname="DOWNLOAD"
local commandaction="Download"
local function_selfname="$(basename $(readlink -f "${BASH_SOURCE[0]}"))"

fn_dl_md5(){
	# Runs MD5 Check if available
	if [ "${md5}" != "0" ]&&[ "${md5}" != "nomd5" ]; then
		echo -ne "verifying ${local_filename} with MD5..."
		sleep 1
		local md5sumcmd=$(md5sum "${local_filedir}/${local_filename}"|awk '{print $1;}')
		if [ "${md5sumcmd}" != "${md5}" ]; then
			fn_print_fail_eol_nl
			echo "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			echo "expected MD5 checksum: ${md5}"
			fn_script_log_fatal "Verifying ${local_filename} with MD5: FAIL"
			fn_script_log_info "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			fn_script_log_info "Expected MD5 checksum: ${md5}"
			core_exit.sh
		else
			fn_print_ok_eol_nl
			fn_script_log_pass "Verifying ${local_filename} with MD5: OK"
			fn_script_log_info "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			fn_script_log_info "Expected MD5 checksum: ${md5}"
		fi
	fi
}

# Extracts bzip2, gzip or zip files
# Extracts can be defined in code like so:
# fn_dl_extract "${local_filedir}" "${local_filename}" "${extractdir}"
# fn_dl_extract "/home/gameserver/lgsm/tmp" "file.tar.bz2" "/home/gamserver/serverfiles"
fn_dl_extract(){
	local_filedir="${1}"
	local_filename="${2}"
	extractdir="${3}"
	# extracts archives
	echo -ne "extracting ${local_filename}..."
	mime=$(file -b --mime-type "${local_filedir}/${local_filename}")
	if [ ! -d "${extractdir}" ]; then
		mkdir "${extractdir}"
	fi
	if [ "${mime}" == "application/gzip" ]||[ "${mime}" == "application/x-gzip" ]; then
		extractcmd=$(tar -zxf "${local_filedir}/${local_filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/x-bzip2" ]; then
		tarcmd=$(tar -jxf "${filedir}/${filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/x-xz" ]; then
		tarcmd=$(tar -xf "${filedir}/${filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/zip" ]; then
		extractcmd=$(unzip -d "${extractdir}" "${local_filedir}/${local_filename}")
	fi
	local exitcode=$?
	if [ ${exitcode} -ne 0 ]; then
		fn_print_fail_eol_nl
		fn_script_log_fatal "Extracting download: FAIL"
		if [ -f "${lgsmlog}" ]; then
			echo "${extractcmd}" >> "${lgsmlog}"
		fi
		echo "${extractcmd}"
		core_exit.sh
	else
		fn_print_ok_eol_nl
		fn_script_log_pass "Extracting download: OK"
	fi
}

# Trap to remove file download if canceled before completed
fn_fetch_trap(){
	echo ""
	echo -ne "downloading ${local_filename}..."
	fn_print_canceled_eol_nl
	fn_script_log_info "Downloading ${local_filename}...CANCELED"
	sleep 1
	rm -f "${local_filedir}/${local_filename}"
	echo -ne "downloading ${local_filename}..."
	fn_print_removed_eol_nl
	fn_script_log_info "Downloading ${local_filename}...REMOVED"
	core_exit.sh<commit_msg>update var names local_filedir local_filename<commit_after>#!/bin/bash
# LinuxGSM core_dl.sh function
# Author: <NAME>
# Contributor: UltimateByte
# Website: https://gameservermanagers.com
# Description: Deals with all downloads for LinuxGSM.

# remote_fileurl: The URL of the file: http://example.com/dl/File.tar.bz2
# local_filedir: location the file is to be saved: /home/server/lgsm/tmp
# local_filename: name of file (this can be different from the url name): file.tar.bz2
# chmodx: Optional, set to "chmodx" to make file executable using chmod +x
# run: Optional, set run to execute the file after download
# forcedl: Optional, force re-download of file even if exists
# md5: Optional, set an md5 sum and will compare it against the file.
#
# Downloads can be defined in code like so:
# fn_fetch_file "${remote_fileurl}" "${local_filedir}" "${local_filename}" "${chmodx}" "${run}" "${forcedl}" "${md5}"
# fn_fetch_file "http://example.com/file.tar.bz2" "/some/dir" "file.tar.bz2" "chmodx" "run" "forcedl" "10cd7353aa9d758a075c600a6dd193fd"

local commandname="DOWNLOAD"
local commandaction="Download"
local function_selfname="$(basename $(readlink -f "${BASH_SOURCE[0]}"))"

fn_dl_md5(){
	# Runs MD5 Check if available
	if [ "${md5}" != "0" ]&&[ "${md5}" != "nomd5" ]; then
		echo -ne "verifying ${local_filename} with MD5..."
		sleep 1
		local md5sumcmd=$(md5sum "${local_filedir}/${local_filename}"|awk '{print $1;}')
		if [ "${md5sumcmd}" != "${md5}" ]; then
			fn_print_fail_eol_nl
			echo "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			echo "expected MD5 checksum: ${md5}"
			fn_script_log_fatal "Verifying ${local_filename} with MD5: FAIL"
			fn_script_log_info "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			fn_script_log_info "Expected MD5 checksum: ${md5}"
			core_exit.sh
		else
			fn_print_ok_eol_nl
			fn_script_log_pass "Verifying ${local_filename} with MD5: OK"
			fn_script_log_info "${local_filename} returned MD5 checksum: ${md5sumcmd}"
			fn_script_log_info "Expected MD5 checksum: ${md5}"
		fi
	fi
}

# Extracts bzip2, gzip or zip files
# Extracts can be defined in code like so:
# fn_dl_extract "${local_filedir}" "${local_filename}" "${extractdir}"
# fn_dl_extract "/home/gameserver/lgsm/tmp" "file.tar.bz2" "/home/gamserver/serverfiles"
fn_dl_extract(){
	local_filedir="${1}"
	local_filename="${2}"
	extractdir="${3}"
	# extracts archives
	echo -ne "extracting ${local_filename}..."
	mime=$(file -b --mime-type "${local_filedir}/${local_filename}")
	if [ ! -d "${extractdir}" ]; then
		mkdir "${extractdir}"
	fi
	if [ "${mime}" == "application/gzip" ]||[ "${mime}" == "application/x-gzip" ]; then
		extractcmd=$(tar -zxf "${local_filedir}/${local_filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/x-bzip2" ]; then
		tarcmd=$(tar -jxf "${local_filedir}/${local_filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/x-xz" ]; then
		tarcmd=$(tar -xf "${local_filedir}/${local_filename}" -C "${extractdir}")
	elif [ "${mime}" == "application/zip" ]; then
		extractcmd=$(unzip -d "${extractdir}" "${local_filedir}/${local_filename}")
	fi
	local exitcode=$?
	if [ ${exitcode} -ne 0 ]; then
		fn_print_fail_eol_nl
		fn_script_log_fatal "Extracting download: FAIL"
		if [ -f "${lgsmlog}" ]; then
			echo "${extractcmd}" >> "${lgsmlog}"
		fi
		echo "${extractcmd}"
		core_exit.sh
	else
		fn_print_ok_eol_nl
		fn_script_log_pass "Extracting download: OK"
	fi
}

# Trap to remove file download if canceled before completed
fn_fetch_trap(){
	echo ""
	echo -ne "downloading ${local_filename}..."
	fn_print_canceled_eol_nl
	fn_script_log_info "Downloading ${local_filename}...CANCELED"
	sleep 1
	rm -f "${local_filedir}/${local_filename}"
	echo -ne "downloading ${local_filename}..."
	fn_print_removed_eol_nl
	fn_script_log_info "Downloading ${local_filename}...REMOVED"
	core_exit.sh
{'fraction_non_alphanumeric': 0.1643979057591623, 'fraction_numerical': 0.016492146596858638, 'mean_word_length': 5.257985257985258, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '325458', 'n_tokens_mistral': 2899, 'n_tokens_neox': 2713, 'n_words': 697}