<commit_before><?php

include 'config.php';

function validate_id($id)
{
    if (!(is_string($id) && preg_match('/^[A-Za-z0-9]+$/', $id) && strlen($id) <= 9
          && is_id_registered($id))) {
        die("This text could not be found.");
    }
}

function validate_year($year)
{
    global $start_year, $end_year;
    if (!(is_int($year) && $year >= $start_year && $year <= $end_year)) {
        die("Year out of range.");
    }
}

function validate_corpus($corpus)
{
    global $corpora;
    if (!(is_string($corpus) && in_array($corpus, $corpora, true))) {
        die("Invalid corpus!");
    }
}

function validate_title($title)
{
    if (!(is_string($title) && mb_check_encoding($title, 'UTF-8'))) {
        die("Title is not a valid UTF-8 string.");
    }
}

function validate_text($text)
{
    if (!(is_string($text) && mb_check_encoding($text, 'UTF-8'))
        || preg_match('/[\x00-\x08\x0B\x0C\x0E-\x1F\x80-\x9F]/u', $text)) {
        die("Text is not a valid UTF-8 string.");
    }
}

function validate_word($word)
{
    if (!(is_string($word) && mb_check_encoding($word, 'UTF-8') && strlen($word) <= 63)) {<commit_msg>Made the validation code a little more lenient with unicode control characters, since Project Gutenberg sometimes uses them.<commit_after><?php

include 'config.php';

function validate_id($id)
{
    if (!(is_string($id) && preg_match('/^[A-Za-z0-9]+$/', $id) && strlen($id) <= 9
          && is_id_registered($id))) {
        die("This text could not be found.");
    }
}

function validate_year($year)
{
    global $start_year, $end_year;
    if (!(is_int($year) && $year >= $start_year && $year <= $end_year)) {
        die("Year out of range.");
    }
}

function validate_corpus($corpus)
{
    global $corpora;
    if (!(is_string($corpus) && in_array($corpus, $corpora, true))) {
        die("Invalid corpus!");
    }
}

function validate_title($title)
{
    if (!(is_string($title) && mb_check_encoding($title, 'UTF-8'))) {
        die("Title is not a valid UTF-8 string.");
    }
}

function validate_text($text)
{
    if (!(is_string($text) && mb_check_encoding($text, 'UTF-8'))
        || preg_match('/[\x00-\x08\x0B\x0C\x0E-\x1F]/u', $text)) {
        die("Text is not a valid UTF-8 string.");
    }
}

function validate_word($word)
{
    if (!(is_string($word) && mb_check_encoding($word, 'UTF-8') && strlen($word) <= 63)) {
{'fraction_non_alphanumeric': 0.20379746835443038, 'fraction_numerical': 0.016455696202531647, 'mean_word_length': 3.407063197026022, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5080455', 'n_tokens_mistral': 874, 'n_tokens_neox': 844, 'n_words': 177}