<commit_before>#!/bin/bash
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

ALL_DIRS="
ash/resources
ui/resources
chrome/app/theme
chrome/browser/resources
chrome/renderer/resources
webkit/glue/resources
remoting/resources
remoting/webapp
"

function sanitize_file {
  tput el
  echo -ne "$1\r"
  local file=$1
  local name=$(basename $file)
  pngcrush -d $TMP_DIR -brute -reduce -rem text -rem mkBT \
      -rem mkTS $file > /dev/null
  mv "$TMP_DIR/$name" "$file"
}
<commit_msg>Remove all ancillary chunks except for tRNS<commit_after>#!/bin/bash
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

ALL_DIRS="
ash/resources
ui/resources
chrome/app/theme
chrome/browser/resources
chrome/renderer/resources
webkit/glue/resources
remoting/resources
remoting/webapp
"

function sanitize_file {
  tput el
  echo -ne "$1\r"
  local file=$1
  local name=$(basename $file)
  # -rem alla removes all ancillary chunks except for tRNS
  pngcrush -d $TMP_DIR -brute -reduce -rem alla $file > /dev/null

  mv "$TMP_DIR/$name" "$file"
}

{'fraction_non_alphanumeric': 0.10254372019077901, 'fraction_numerical': 0.009538950715421303, 'mean_word_length': 3.937254901960784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4793881', 'n_tokens_mistral': 439, 'n_tokens_neox': 420, 'n_words': 143}