<commit_before>#!/usr/bin/env bash

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage:
# normalize.sh log_file_name_1.txt log_file_name_2.txt ...

# Deal w/ BSD sed vs GNU sed
case "$(uname -s)" in
  Darwin|*BSD)
    SED='sed -i "" '
    ;;

  Linux)
    SED='sed -i '
    ;;

  *)
    echo "untested platform"
    SED='sed -i '
    ;;
esac

# Normalizes log files to make collation of warnings easier
# - Remove timestamp stuff, ie "09:54:12   INFO - "
# - Remove PIDs, ie "[Child 12345]". Note: it might be useful to preserve the
#   parent/child distinction.
# - Remove full path in warnings, ie "/builds/slave/buildbot1242/.../build/src"
# - Possibly: Remove hex addresses/values, ie: "0x7fccc5bbd2f0"
normalize_file()
{
  file=$1
  $SED -E -e 's/^[0-9:]+[[:space:]]+INFO[[:space:]]+-[[:space:]]+//g'         \
          -e 's/\[(Child|Parent|GMP|NPAPI)?[[:space:]]?[0-9]+\]/\[NNNNN\]/g'  \
          -e 's/file (\/builds\/.*\/build\/src\/)([^[:space:]]+)/file \2/g'   \
          -e 's/file (\/builds\/.*\/build\/gecko\/)([^[:space:]]+)/file \2/g' \
	  $file
}

for file in "$@"; do
  echo "normalizing file: $file"
  normalize_file "$file"
done
<commit_msg>Handle web platform tests format<commit_after>#!/usr/bin/env bash

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Usage:
# normalize.sh log_file_name_1.txt log_file_name_2.txt ...

# Deal w/ BSD sed vs GNU sed
case "$(uname -s)" in
  Darwin|*BSD)
    SED='sed -i "" '
    ;;

  Linux)
    SED='sed -i '
    ;;

  *)
    echo "untested platform"
    SED='sed -i '
    ;;
esac

# Normalizes log files to make collation of warnings easier
# - Remove timestamp stuff, ie "09:54:12   INFO - "
# - Remove PIDs, ie "[Child 12345]". Note: it might be useful to preserve the
#   parent/child distinction.
# - Remove full path in warnings, ie "/builds/slave/buildbot1242/.../build/src"
# - Possibly: Remove hex addresses/values, ie: "0x7fccc5bbd2f0"
normalize_file()
{
  file=$1
  $SED -E -e 's/^[0-9:]+[[:space:]]+INFO[[:space:]]+-[[:space:]]+//g'         \
          -e 's/^PROCESS \| [0-9]+ \| //g'                                    \
          -e 's/\[(Child|Parent|GMP|NPAPI)?[[:space:]]?[0-9]+\]/\[NNNNN\]/g'  \
          -e 's/file (\/builds\/.*\/build\/src\/)([^[:space:]]+)/file \2/g'   \
          -e 's/file (\/builds\/.*\/build\/gecko\/)([^[:space:]]+)/file \2/g' \
	  $file
}

for file in "$@"; do
  echo "normalizing file: $file"
  normalize_file "$file"
done

{'fraction_non_alphanumeric': 0.19919737322145203, 'fraction_numerical': 0.024808464064210143, 'mean_word_length': 2.4929936305732485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4458403', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1043, 'n_words': 276}