<commit_before>#!/bin/bash

# ============================================
# Setup environemnt
# ============================================

# Halt on first error
set -e 

# Fail on unset variable
set -u


# ============================================
# Configuration variables
# ============================================


# INPUT_FILE="~/Data/enwiki-20130204-pages-articles.xml.bz2"
INPUT_FILE="/Volumes/LocalScratchHD/LocalHome/Data/random.xml"


OUTPUT_DIR="out"

BYBLO_PATH="byblo"

WAG_PATH="wag"

CHARSET="UTF-8"


function main {
	
	# Convert paths to 
	INPUT_FILE=`canonicalize ${INPUT_FILE}`
	OUTPUT_DIR=`canonicalize ${OUTPUT_DIR}`
	BYBLO_PATH=`canonicalize ${BYBLO_PATH}`
	WAG_PATH=`canonicalize ${WAG_PATH}`
	WORKING_DIR=`canonicalize "$(pwd - P)"`
	INPUT_FILE_NAME="$(basename -- "$INPUT_FILE")"
	
	# ============================================
	# Run Wikipedia Alias Generator
	# ============================================
	
	ALIASES_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.aliases"
	
	if [ ! -f ${ALIASES_FILE} ]; then
		echo "Extracting aliases from ${INPUT_FILE} to ${ALIASES_FILE}"
		cd $WAG_PATH
		./wag.sh \
			--charset "${CHARSET}" \
			--identityAliases \
			--output "${ALIASES_FILE}" \
			--outputColumns SOURCE,TARGET \
			--outputFormat TSV \
			"${INPUT_FILE}"
		cd $WORKING_DIR
	else
		echo "Aliases file ${ALIASES_FILE} already exists; skipping."
	fi
	
	
	# ============================================
	# Split the source and target columns
	# ============================================

	SOURCE_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.source"
	TARGET_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.target"
<commit_msg>Changed default output format to TSV_SIMPLIFIED<commit_after>#!/bin/bash

# ============================================
# Setup environemnt
# ============================================

# Halt on first error
set -e 

# Fail on unset variable
set -u


# ============================================
# Configuration variables
# ============================================


# INPUT_FILE="~/Data/enwiki-20130204-pages-articles.xml.bz2"
INPUT_FILE="/Volumes/LocalScratchHD/LocalHome/Data/random.xml"


OUTPUT_DIR="out"

BYBLO_PATH="byblo"

WAG_PATH="wag"

CHARSET="UTF-8"


function main {
	
	# Convert paths to 
	INPUT_FILE=`canonicalize ${INPUT_FILE}`
	OUTPUT_DIR=`canonicalize ${OUTPUT_DIR}`
	BYBLO_PATH=`canonicalize ${BYBLO_PATH}`
	WAG_PATH=`canonicalize ${WAG_PATH}`
	WORKING_DIR=`canonicalize "$(pwd - P)"`
	INPUT_FILE_NAME="$(basename -- "$INPUT_FILE")"
	
	# ============================================
	# Run Wikipedia Alias Generator
	# ============================================
	
	ALIASES_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.aliases"
	
	if [ ! -f ${ALIASES_FILE} ]; then
		echo "Extracting aliases from ${INPUT_FILE} to ${ALIASES_FILE}"
		cd $WAG_PATH
		./wag.sh \
			--charset "${CHARSET}" \
			--identityAliases \
			--output "${ALIASES_FILE}" \
			--outputColumns SOURCE,TARGET \
			--outputFormat TSV_SIMPLIFIED \
			"${INPUT_FILE}"
		cd $WORKING_DIR
	else
		echo "Aliases file ${ALIASES_FILE} already exists; skipping."
	fi
	
	
	# ============================================
	# Split the source and target columns
	# ============================================

	SOURCE_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.source"
	TARGET_FILE="${OUTPUT_DIR%%/}/${INPUT_FILE_NAME}.target"

{'fraction_non_alphanumeric': 0.3308583308583309, 'fraction_numerical': 0.00594000594000594, 'mean_word_length': 5.603921568627451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3489640', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1058, 'n_words': 183}