<commit_before>mydig() {
  dig +time=1 +short "$@" || { echo "DNS lookup error!" >&2; cleanup; false; }
}

# findns <domain>
# Find an authoritative NS server for domain
findns() {
  dd="$1"; ns=""; dig=${2:-mydig}
  while test -z "$ns"
  do
    ns=$($dig -t NS $dd) && echo $dd | grep -q '\.' && dd="${dd#*.}" || {
      unset ns
      break 1
    }
  done
  echo "$ns" | grep .
}

# printip <<EOF
# 1.2.3.4
# fc00:db20:35b:7399::5
# EOF
# ip4:1.2.3.4
# ip6:fc00:db20:35b:7399::5
printip() {
  while read line
  do
    if echo $line | grep -q ':'; then ver=6
    elif echo $line | grep -q '\.'; then ver=4
    else EXIT=1; break 1
    fi
    echo "ip${ver}:$line"
  done
  return $EXIT
}

# dea <hostname>
# dea both.spf-tools.ml
# 192.168.127.12
# fc00:db20:35b:7399::5
dea() {
  for TYPE in A AAAA; do mydig -t $TYPE $1 | printip; done
  true
}

# demx <domain>
# Get MX record for a domain
demx() {
  mymx=$(mydig -t MX $1 | awk '{print $2}' | grep -m1 .)
  for name in $mymx; do dea $name; done
}

# parsepf <host>
parsepf() {
  host=$1
  myns=$(findns $host)
  mydig -t TXT $host @$myns | sed 's/^"//;s/"$//;s/" "//' \
    | grep -E '^v=spf1\s+' | grep .
}

# getem <includes>
# e.g. includes="include:gnu.org include:google.com"
getem() {
  myloop=$1
  shift
  echo $* | tr " " "\n" | sed '/^$/d' | cut -b 9- | while read included
  do echo Getting $included 1>&2; despf $included $myloop
  done
}

# despf <domain>
despf() {
  host=$1
  myloop=$2

  # Detect loop
  echo $host | grep -qxFf $myloop && {
    #echo "Loop detected with $host!" 1>&2
    return
  }

  echo "$host" >> "${myloop}"
  myspf=$(parsepf $host)
  getem $myloop $(echo $myspf | grep -Eo 'include:\S+')
  echo $myspf | grep -qw a && dea $host
  echo $myspf | grep -qw mx && demx $host
  echo $myspf | grep -Eo 'ip[46]:\S+' || true
}

cleanup() {
  myloop=$1
  test -n "$myloop" && rm ${myloop}*
}
<commit_msg>Proper handling of a and mx modifiers to cover cases like this :a a:gnu.org a:google.com/24 printip print prefixes<commit_after>mydig() {
  dig +time=1 +short "$@" || { echo "DNS lookup error!" >&2; cleanup; false; }
}

# findns <domain>
# Find an authoritative NS server for domain
findns() {
  dd="$1"; ns=""; dig=${2:-mydig}
  while test -z "$ns"
  do
    ns=$($dig -t NS $dd) && echo $dd | grep -q '\.' && dd="${dd#*.}" || {
      unset ns
      break 1
    }
  done
  echo "$ns" | grep .
}

# printip <<EOF
# 192.168.127.12
# fc00:db20:35b:7399::5
# EOF
# ip4:1.2.3.4
# ip6:fc00:db20:35b:7399::5
printip() {
  if [ "x" != "x$1" ] ; then 
	  prefix="/$1";
  fi
  while read line
  do
    if echo $line | grep -q ':'; then ver=6
    elif echo $line | grep -q '\.'; then ver=4
    else EXIT=1; break 1
    fi
    echo "ip${ver}:${line}${prefix}"
  done
  return $EXIT
}

# dea <hostname>
# dea both.spf-tools.ml
# 1.2.3.4
# fc00:db20:35b:7399::5
dea() {
  for TYPE in A AAAA; do mydig -t $TYPE $1 | printip; done
  true
}

# demx <domain>
# Get MX record for a domain
demx() {
  mymx=$(mydig -t MX $1 | awk '{print $2}' | grep -m1 .)
  for name in $mymx; do dea $name $2; done
}

# parsepf <host>
parsepf() {
  host=$1
  myns=$(findns $host)
  mydig -t TXT $host @$myns | sed 's/^"//;s/"$//;s/" "//' \
    | grep -E '^v=spf1\s+' | grep .
}

# getem <includes>
# e.g. includes="include:gnu.org include:google.com"
getem() {
  myloop=$1
  shift
  echo $* | tr " " "\n" | sed '/^$/d' | cut -b 9- | while read included
  do echo Getting $included 1>&2; despf $included $myloop
  done
}

# getamx <a:modifiers>
# e.g. a="a a:gnu.org a:google.com/24"
getamx() {
	host=$1
	local myloop=$2
	shift
	shift
	for record in $* ; do 
		local ahost=$(echo $record | cut -s -d: -f2-)
		if [ "x" = "x$ahost" ] ; then
			lookuphost="$host";
			cidr="32";
			mech="$record"
		else
			mech=$(echo $record | cut -s -d: -f1)
			cidr=$(echo $ahost | cut -s -d\/ -f2-)
			if [ "x" = "x$cidr" ] ; then
				lookuphost=$ahost
				cidr="32";
			else
				lookuphost=$(echo $ahost | cut -d\/ -f1)
			fi
		fi
		if [ "$mech" = "a" ]; then
			dea $lookuphost $cidr
		elif [ "$mech" = "mx" ]; then
			demx $lookuphost $cidr
		fi
	done
}
# despf <domain>
despf() {
  host=$1
  myloop=$2

  # Detect loop
  echo $host | grep -qxFf $myloop && {
    #echo "Loop detected with $host!" 1>&2
    return
  }

  echo "$host" >> "${myloop}"
  myspf=$(parsepf $host)
  getem $myloop $(echo $myspf | grep -Eo 'include:\S+')
  getamx $host $myloop $(echo $myspf | grep -Eo -w '(mx|a)(:\S+)?')
  echo $myspf | grep -Eo 'ip[46]:\S+' || true
}

cleanup() {
  myloop=$1
  test -n "$myloop" && rm ${myloop}*
}

{'fraction_non_alphanumeric': 0.20628986144710798, 'fraction_numerical': 0.038926764899934026, 'mean_word_length': 2.7096247960848285, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5859892', 'n_tokens_mistral': 2121, 'n_tokens_neox': 2007, 'n_words': 554}