<commit_before>#!/bin/bash
set -e

repo="$1"
if [ -z "$repo" ]; then
	echo >&2 "usage: $0 repo"
	echo >&2 "   ie: $0 hylang"
	exit 1
fi

dir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
url='https://raw.githubusercontent.com/docker-library/official-images/master/library/'"$repo" 

IFS=$'\n'
tags=( $(curl -sSL $url | grep -vE '^$|^#' | cut -d':' -f1 | sort -u) )
unset IFS

text=
for tag in "${tags[@]}"; do
	if [ -f "$dir/variant-${tag}.md" ]; then
		text+=$'\n' # give a little space
		# because parameter expansion eats the trailing newline
		text+="$(<"$dir/variant-${tag}.md")"$'\n'
	fi
done
if [ "$text" ]; then
  latest=($(curl -sSL $url | sed -e 's/git:\/\/github.com\///' -e 's/@/ /' - | grep "latest"))
  dockerfile='https://raw.githubusercontent.com/'"${latest[1]}"'/'"${latest[2]}"'/'"${latest[3]}"'/Dockerfile'
  base_image=$(curl -sSL $dockerfile | sed 's/:/\t/' | nawk '/^FROM .*$/ { print $2 }')
  # give a little space
  echo
  echo
  if [ "$base_image" = "buildpack-deps" ]; then
    cat "$dir/variant-buildpacks.md"
  else
    cat "$dir/variant.md"
  fi
  echo "$text"
fi<commit_msg>Style changes and only process dockerfiles from github.com<commit_after>#!/bin/bash
set -e

repo="$1"
if [ -z "$repo" ]; then
	echo >&2 "usage: $0 repo"
	echo >&2 "   ie: $0 hylang"
	exit 1
fi

dir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
url='https://raw.githubusercontent.com/docker-library/official-images/master/library/'"$repo"

IFS=$'\n'
tags=( $(curl -sSL "$url" | grep -vE '^$|^#' | cut -d':' -f1 | sort -u) )
unset IFS

text=
for tag in "${tags[@]}"; do
	if [ -f "$dir/variant-${tag}.md" ]; then
		text+=$'\n' # give a little space
		# because parameter expansion eats the trailing newline
		text+="$(<"$dir/variant-${tag}.md")"$'\n'
	fi
done
if [ "$text" ]; then
	latest=($(curl -sSL "$url" | grep "latest.*github.com" | sed -e 's!git://github.com/!!' -e 's/@/ /' -))
	if [ -z "latest" ]; then
		exit 0 # If not github or no latest tag, we are done here
	fi
	dockerfile='https://raw.githubusercontent.com/'"${latest[1]}"'/'"${latest[2]}"'/'"${latest[3]}"'/Dockerfile'
	baseImage=$(curl -sSL $dockerfile | sed 's/:/\t/' | awk '$1 == "FROM" { print $2 }')
	# give a little space
	echo
	echo
	if [ "$baseImage" = "buildpack-deps" ]; then
		cat "$dir/variant-buildpacks.md"
	else
		cat "$dir/variant.md"
	fi
	echo "$text"
fi
{'fraction_non_alphanumeric': 0.24133504492939667, 'fraction_numerical': 0.010269576379974325, 'mean_word_length': 3.56640625, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3716211', 'n_tokens_mistral': 993, 'n_tokens_neox': 935, 'n_words': 239}