<commit_before>#!/bin/bash

# Kerning Input File compiler for TrueType AAT
# Copyright 2013 <NAME>


# Prints a message to stderr and exits.
err () {
	echo >&2 $0: $1
	exit ${2-1}
}

# Make sure both arguments are given.
test $# != 2 &&
	err "usage: $(basename $0) <post.xml> <kern.kif>" 2

# Parse the 'post' table dump for an array of glyphs names.
glnames=($(sed -n 's/<PostScriptName ..* NameString=\"\(..*\)\".*>/\1/p' <$1))

# Remove comments from the KIF file.
kif="$(sed -e '/^\/\/.*/d' -e 's/[ 	]*\/\/.*//' $2)"

# Fills $index with index of a token within the list following the token.
indexof () {
	index=0
	token=$1
	shift
	for item
	do
		test $token = $item && return
		let index++
	done
	index=-1
}


printf "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\" ?>\n"
printf "<genericSFNTTable tag=\"kern\">\n"

# Print the table header before reading actual subtables.
ntables=$(grep -c '^Type ' <<<"$kif")
off=0 # current offset into the table

printf "\t<dataline offset=\"%08X\" hex=\"%04X%04X\"/> <!-- %s -->\n" \
	$off 1 0 "Table version" && let off+=4

printf "\t<dataline offset=\"%08X\" hex=\"%08X\"/> <!-- %s -->\n" \
	$off $ntables "No. of subtables" && let off+=4

# Read the KIF subtable by subtable until the end of file.
until test $eof
do

	vertical='no'
	crossstream='no'
	variation='no'
	classes=() # GID-indexed array of classes
	clnames=(EOT OOB DEL EOL) # Class names
	unset glstart # First glyph assigned to a class
	unset glend # Last glyph assigned to a class
	states=() # State records
	stnames=() # State names
	gotos=() # Next states
	gtnames=() # Names of the next states
	flpush=() # Flags for the push action
	fladvance=() # Flags for the advance action
	actions=() # Kern value lists to apply
	actnames=() # Names of kern value lists to apply
	values=() # Kerning values
	vlnames=() # Names of the kern value lists
	vlindices=() # Indexes of values beginning the lists

	until test "$REPLY"
	do read
	done
	line=($REPLY)
	test $line != "Type" &&
		err "fatal: kerning type expected"
	case ${line[@]:1} in
		Contextual) tabfmt=1;;
		*) err "fatal: unknown kerning type: ${line[@]:1}";;
	esac

	unset REPLY
	until test "$REPLY"
	do read
	done
	line=($REPLY)
	test $line != "Orientation" &&
		err "fatal: kerning orientation expected"
	case ${line[@]:1} in
		V) vertical='yes';;
		H) ;;
		*) err "fatal: bad orientation flag: ${line[@]:1}";;
	esac

	unset REPLY
	until test "$REPLY"
	do read
	done
	line=($REPLY)
	if test $line = "Cross-stream"
	then
		case ${line[@]:1} in
			yes) crossstream='yes';;
			no) ;;
			*) err "fatal: bad cross-stream flag: ${line[@]:1}";;
		esac

		unset REPLY
		until test "$REPLY"
		do read
		done
		line=($REPLY)
	fi

	if test $line = "Variation"
	then
		case ${line[@]:1} in
			yes) variation='yes';;
			no) ;;
			*) err "fatal: bad variation flag: ${line[@]:1}";;
		esac

		unset REPLY
		until test "$REPLY"
		do read
		done
	fi

	unset line
	let nclasses=4 # four built-in classes
	while true
	do
		# Skip blanks, but break on a state table header (an indent).
		test "$REPLY" || { read; continue; }
		grep -q '^[a-zA-Z+]' <<<"$REPLY" || break

		line=(${line[@]} ${linecont[@]-$REPLY})
		read
		if grep -q '^\+' <<<"$REPLY"
		then
			linetmp=($REPLY)
			test ${#linetmp[@]} -gt 1 && linecont=(${linetmp[@]:1})
			continue
		fi

		clnames[${#clnames[@]}]=$line

		for glyph in ${line[@]:1}
		do
			indexof $glyph ${glnames[@]}
			test $index -eq -1 &&
				err "fatal: glyph not found: $glyph"
			classes[$index]=$nclasses
			test $index -lt ${glstart=$index} && glstart=$index
			test $index -gt ${glend=$index} && glend=$index
		done

		let nclasses++
		unset line linecont
	done

	# Set an Out-of-Bounds class on glyphs inbetween the specified ones.
	for i in $(seq $glstart $glend)
	do test "${classes[$i]}" || classes[$i]=1
	done

	# Check if the class list and state table header match.
	header=($REPLY)
	test "${clnames[*]}" != "${header[*]}" &&
		err "fatal: classes and state header don't match"

	# Skip blanks directly beneath the header.
	unset REPLY
	until test "$REPLY"
	do read
	done
	while true
	do
		# Break on a blank line or an entry table header.
		test "$REPLY" || break
		grep -q '^[a-zA-Z]' <<<"$REPLY" || break

		line=($REPLY)<commit_msg>Do not use bash array for the glyph index lookup<commit_after>#!/bin/bash

# Kerning Input File compiler for TrueType AAT
# Copyright 2013 <NAME>


# Prints a message to stderr and exits.
err () {
	echo >&2 $0: $1
	exit ${2-1}
}

# Make sure both arguments are given.
test $# != 2 &&
	err "usage: $(basename $0) <post.xml> <kern.kif>" 2

post=$1 # Path to the 'post' table dump

# Find the line offset to the first glyph name record in the 'post' dump.
postoff=$(grep -n '\.notdef' $post | cut -d : -f 1)
test $postoff ||
	err "fatal: required glyph .notdef missing"

# Parse the 'post' table dump for an array of glyphs names.
glnames=($(sed -n 's/<PostScriptName ..* NameString=\"\(..*\)\".*>/\1/p' <$post))

# Remove comments from the KIF file.
kif="$(sed -e '/^\/\/.*/d' -e 's/[ 	]*\/\/.*//' $2)"

# Fills $index with index of a token within the list following the token.
indexof () {
	index=0
	token=$1
	shift
	for item
	do
		test $token = $item && return
		let index++
	done
	index=-1
}


printf "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\" ?>\n"
printf "<genericSFNTTable tag=\"kern\">\n"

# Print the table header before reading actual subtables.
ntables=$(grep -c '^Type ' <<<"$kif")
off=0 # current offset into the table

printf "\t<dataline offset=\"%08X\" hex=\"%04X%04X\"/> <!-- %s -->\n" \
	$off 1 0 "Table version" && let off+=4

printf "\t<dataline offset=\"%08X\" hex=\"%08X\"/> <!-- %s -->\n" \
	$off $ntables "No. of subtables" && let off+=4

# Read the KIF subtable by subtable until the end of file.
until test $eof
do

	vertical='no'
	crossstream='no'
	variation='no'
	classes=() # GID-indexed array of classes
	clnames=(EOT OOB DEL EOL) # Class names
	unset glstart # First glyph assigned to a class
	unset glend # Last glyph assigned to a class
	states=() # State records
	stnames=() # State names
	gotos=() # Next states
	gtnames=() # Names of the next states
	flpush=() # Flags for the push action
	fladvance=() # Flags for the advance action
	actions=() # Kern value lists to apply
	actnames=() # Names of kern value lists to apply
	values=() # Kerning values
	vlnames=() # Names of the kern value lists
	vlindices=() # Indexes of values beginning the lists

	until test "$REPLY"
	do read
	done
	line=($REPLY)
	test $line != "Type" &&
		err "fatal: kerning type expected"
	case ${line[@]:1} in
		Contextual) tabfmt=1;;
		*) err "fatal: unknown kerning type: ${line[@]:1}";;
	esac

	unset REPLY
	until test "$REPLY"
	do read
	done
	line=($REPLY)
	test $line != "Orientation" &&
		err "fatal: kerning orientation expected"
	case ${line[@]:1} in
		V) vertical='yes';;
		H) ;;
		*) err "fatal: bad orientation flag: ${line[@]:1}";;
	esac

	unset REPLY
	until test "$REPLY"
	do read
	done
	line=($REPLY)
	if test $line = "Cross-stream"
	then
		case ${line[@]:1} in
			yes) crossstream='yes';;
			no) ;;
			*) err "fatal: bad cross-stream flag: ${line[@]:1}";;
		esac

		unset REPLY
		until test "$REPLY"
		do read
		done
		line=($REPLY)
	fi

	if test $line = "Variation"
	then
		case ${line[@]:1} in
			yes) variation='yes';;
			no) ;;
			*) err "fatal: bad variation flag: ${line[@]:1}";;
		esac

		unset REPLY
		until test "$REPLY"
		do read
		done
	fi

	unset line
	let nclasses=4 # four built-in classes
	while true
	do
		# Skip blanks, but break on a state table header (an indent).
		test "$REPLY" || { read; continue; }
		grep -q '^[a-zA-Z+]' <<<"$REPLY" || break

		line=(${line[@]} ${linecont[@]-$REPLY})
		read
		if grep -q '^\+' <<<"$REPLY"
		then
			linetmp=($REPLY)
			test ${#linetmp[@]} -gt 1 && linecont=(${linetmp[@]:1})
			continue
		fi

		clnames[${#clnames[@]}]=$line

		for glyph in ${line[@]:1}
		do
			index=$(grep -n "\"$glyph\"" $post | cut -d : -f 1)
			test $index ||
				err "fatal: glyph not found: $glyph"
			let index-=$postoff
			classes[$index]=$nclasses
			test $index -lt ${glstart=$index} && glstart=$index
			test $index -gt ${glend=$index} && glend=$index
		done

		let nclasses++
		unset line linecont
	done

	# Set an Out-of-Bounds class on glyphs inbetween the specified ones.
	for i in $(seq $glstart $glend)
	do test "${classes[$i]}" || classes[$i]=1
	done

	# Check if the class list and state table header match.
	header=($REPLY)
	test "${clnames[*]}" != "${header[*]}" &&
		err "fatal: classes and state header don't match"

	# Skip blanks directly beneath the header.
	unset REPLY
	until test "$REPLY"
	do read
	done
	while true
	do
		# Break on a blank line or an entry table header.
		test "$REPLY" || break
		grep -q '^[a-zA-Z]' <<<"$REPLY" || break

		line=($REPLY)
{'fraction_non_alphanumeric': 0.17463924554027951, 'fraction_numerical': 0.011703215543688217, 'mean_word_length': 3.4454545454545453, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 2, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1572939', 'n_tokens_mistral': 3540, 'n_tokens_neox': 3295, 'n_words': 1120}