<commit_before>#!/bin/sh

echo "/* This file is automatically generated from"
echo " * $1"
echo " * Do not edit it directly, but edit the source file "
echo " * and rerun autogen.sh"
echo " */"
echo
echo "#define REP_CACHE_DB_SQL \\"

comment=false

while read line; do
  if [ "$comment" = "false" ] && echo "$line" | fgrep '/*' >/dev/null; then
    comment=true
  fi

  if [ "$comment" = "true" ] && echo "$line" | fgrep '*/' >/dev/null; then
    comment=false
    continue
  fi

  if [ "$comment" = "true" ]; then
    continue
  fi

  line="`echo "$line" | sed -e 's/"/\\"/g'`"
  if [ -n "$line" ]; then
    echo "    \"$line\" \\"
  fi
done

echo '    ""'
<commit_msg>Follow-up to r35676:<commit_after>#!/bin/sh

echo "/* This file is automatically generated from"
echo " * $1"
echo " * Do not edit it directly, but edit the source file "
echo " * and rerun autogen.sh"
echo " */"
echo
echo "#define REP_CACHE_DB_SQL \\"

comment=false

IFS="
"
while read line; do
  if [ "$comment" = "false" ] && echo "$line" | fgrep '/*' >/dev/null; then
    comment=true
  fi

  if [ "$comment" = "true" ] && echo "$line" | fgrep '*/' >/dev/null; then
    comment=false
    continue
  fi

  if [ "$comment" = "true" ]; then
    continue
  fi

  line="`echo "$line" | sed -e 's/"/\\"/g'`"
  if [ -n "$line" ]; then
    echo "  \"$line\"\\"
  fi
done

echo '  ""'

{'fraction_non_alphanumeric': 0.1930473372781065, 'fraction_numerical': 0.005177514792899409, 'mean_word_length': 2.6666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2060668', 'n_tokens_mistral': 531, 'n_tokens_neox': 515, 'n_words': 155}