<commit_before>#!/usr/bin/env bash

#save current directory
starting_dir=$(pwd)

#cd to the directory of this script
#from http://www.ostricher.com/2014/10/the-right-way-to-get-the-directory-of-a-bash-script/

get_script_dir  () {
	SOURCE="${BASH_SOURCE[0]}"
	# While $SOURCE is a symlink, resolve it
	while [ -h "$SOURCE" ]; do
		DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
		SOURCE="$( readlink "$SOURCE" )"
		# If $SOURCE was a relative symlink (so no "/" as prefix, need to resolve it relative to the symlink base directory
		[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
	done
	DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
	echo $DIR
}

cd_to_current_dir () {
	DIR="$(get_script_dir)"
	info "CD'ing to ${DIR}"
	cd "${DIR}"
}

add_line_to_file_if_not_present () {
	LINE=$1
	FILE=$2
	sudo grep -q "$LINE" "$FILE" || sudo echo "$LINE" >> "$FILE"
}


#see if we are supposed to install dependencies or just refresh code from Dendro repositories
# code from http://stackoverflow.com/questions/7069682/how-to-get-arguments-with-flags-in-bash-script

#When you want getopts to expect an argument for an option, just place a : (colon) after the proper option flag. If you want -A to expect an argument (i.e. to become -A SOMETHING) just do: getopts fA:x VARNAME
# http://wiki.bash-hackers.org/howto/getopts_tutorial

refresh_code_only="false"

while getopts 'r' flag; do
  case $flag in
    r)
		refresh_code_only="true"
		;;
    *)
		error "Unexpected option ${flag}"
		;;
  esac
done

cd_to_current_dir
source ./constants.sh

#apply pre-installation fixes such as DNS fixes (thank you bugged Ubuntu distros)
info "Applying pre-installation fixes..."
source ./Fixes/fix_dns.sh

#fix any unfinished installations
	info "Preparing setup..."
	sudo dpkg --configure -a
	sudo apt-get -qq update

#save current working directory
	setup_dir=$(pwd)
	info "Setup running at : ${setup_dir}"

#create temp downloads folder
	info "Creating temporary folder for downloads at : ${setup_dir}"
	sudo mkdir -p $temp_downloads_folder

#install dependencies
	if [[ "${refresh_code_only}" == "true" ]]
	then
		warning "Bypassing dependency installation"
	else
		warning "Installing dependencies"
		source ./Dependencies/misc.sh
		source ./Dependencies/node.sh<commit_msg>Apply locales fix<commit_after>#!/usr/bin/env bash

#save current directory
starting_dir=$(pwd)

#cd to the directory of this script
#from http://www.ostricher.com/2014/10/the-right-way-to-get-the-directory-of-a-bash-script/

get_script_dir  () {
	SOURCE="${BASH_SOURCE[0]}"
	# While $SOURCE is a symlink, resolve it
	while [ -h "$SOURCE" ]; do
		DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
		SOURCE="$( readlink "$SOURCE" )"
		# If $SOURCE was a relative symlink (so no "/" as prefix, need to resolve it relative to the symlink base directory
		[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
	done
	DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
	echo $DIR
}

cd_to_current_dir () {
	DIR="$(get_script_dir)"
	info "CD'ing to ${DIR}"
	cd "${DIR}"
}

add_line_to_file_if_not_present () {
	LINE=$1
	FILE=$2
	sudo grep -q "$LINE" "$FILE" || sudo echo "$LINE" >> "$FILE"
}


#see if we are supposed to install dependencies or just refresh code from Dendro repositories
# code from http://stackoverflow.com/questions/7069682/how-to-get-arguments-with-flags-in-bash-script

#When you want getopts to expect an argument for an option, just place a : (colon) after the proper option flag. If you want -A to expect an argument (i.e. to become -A SOMETHING) just do: getopts fA:x VARNAME
# http://wiki.bash-hackers.org/howto/getopts_tutorial

refresh_code_only="false"

while getopts 'r' flag; do
  case $flag in
    r)
		refresh_code_only="true"
		;;
    *)
		error "Unexpected option ${flag}"
		;;
  esac
done

cd_to_current_dir
source ./constants.sh

#apply pre-installation fixes such as DNS fixes (thank you bugged Ubuntu distros)
info "Applying pre-installation fixes..."
source ./Fixes/fix_dns.sh
source ./Fixes/fix_locales.sh

#fix any unfinished installations
	info "Preparing setup..."
	sudo dpkg --configure -a
	sudo apt-get -qq update

#save current working directory
	setup_dir=$(pwd)
	info "Setup running at : ${setup_dir}"

#create temp downloads folder
	info "Creating temporary folder for downloads at : ${setup_dir}"
	sudo mkdir -p $temp_downloads_folder

#install dependencies
	if [[ "${refresh_code_only}" == "true" ]]
	then
		warning "Bypassing dependency installation"
	else
		warning "Installing dependencies"
		source ./Dependencies/misc.sh
		source ./Dependencies/node.sh
{'fraction_non_alphanumeric': 0.13544415127528583, 'fraction_numerical': 0.007036059806508356, 'mean_word_length': 4.116985376827897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7204622', 'n_tokens_mistral': 1640, 'n_tokens_neox': 1515, 'n_words': 487}