<commit_before>
declare -A _FORM_FILES

s::process-body () {
    local contenttype=$1 ; shift
    local contentlength=$1 ; shift

    RESULT=$(dd bs=$contentlength count=1 of=$_TEMPDIR/body 2>&1) || panic "failed to read body $RESULT"

    case $contenttype in
	multipart/form-data*)
	    (
		cat <<EOF
Content-Type: $contenttype

EOF
		cat $_TEMPDIR/body
	    ) > $_TEMPDIR/mimebits
	    while IFS=$'\t' read name fname ; do
		log::debug "pulling field from form: $fname -> $name"
		_FORM_FILES[$name]=$_TEMPDIR/$fname
	    done < <(cd $_TEMPDIR && rm -rf form && $SCHOCKDIR/mimeout < mimebits)<commit_msg>Read one byte at a time.  Use normal command logging.<commit_after>
declare -A _FORM_FILES

s::process-body () {
    local contenttype=$1 ; shift
    local contentlength=$1 ; shift

    log::cmd dd bs=1 count=$contentlength of=$_TEMPDIR/body || panic "failed to read body $RESULT"
    case $contenttype in
	multipart/form-data*)
	    (
		cat <<EOF
Content-Type: $contenttype

EOF
		cat $_TEMPDIR/body
	    ) > $_TEMPDIR/mimebits
	    while IFS=$'\t' read name fname ; do
		log::debug "pulling field from form: $fname -> $name"
		_FORM_FILES[$name]=$_TEMPDIR/$fname
	    done < <(cd $_TEMPDIR && rm -rf form && $SCHOCKDIR/mimeout < mimebits)
{'fraction_non_alphanumeric': 0.1347233360064154, 'fraction_numerical': 0.006415396952686447, 'mean_word_length': 3.5547445255474455, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4057908', 'n_tokens_mistral': 461, 'n_tokens_neox': 429, 'n_words': 120}