<commit_before>#!/bin/bash

# Loops through all text files tracked by Git.
git grep -zIl '' |
while IFS= read -rd '' f; do
    # Exclude some types of files.
    if [[ $f == *"proj" ]]; then
        continue
    elif [[ $f == *"filters" ]]; then
        continue
    elif [[ $f == *"sln" ]]; then
        continue
    elif [[ $f == *"json" ]]; then
        continue
    elif [[ $f == *"min.js" ]]; then
        continue
    fi
    # Ensures that files are UTF-8 formatted.
    recode UTF-8 $f 2> /dev/null
    # Ensures that files have LF line endings.<commit_msg>Trying to disable format testing on basis_transcoder.js<commit_after>#!/bin/bash

# Loops through all text files tracked by Git.
git grep -zIl '' |
while IFS= read -rd '' f; do
    # Exclude some types of files.
    if [[ $f == *"proj" ]]; then
        continue
    elif [[ $f == *"filters" ]]; then
        continue
    elif [[ $f == *"sln" ]]; then
        continue
    elif [[ $f == *"json" ]]; then
        continue
    elif [[ $f == *"min.js" ]]; then
        continue
    elif [[ $f == *"coder.js" ]]; then
        continue
    fi
    # Ensures that files are UTF-8 formatted.
    recode UTF-8 $f 2> /dev/null
    # Ensures that files have LF line endings.
{'fraction_non_alphanumeric': 0.15089722675367048, 'fraction_numerical': 0.004893964110929853, 'mean_word_length': 2.122137404580153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4381951', 'n_tokens_mistral': 422, 'n_tokens_neox': 395, 'n_words': 144}