<commit_before><?php

require __DIR__ . '/vendor/autoload.php';

$wikipedia = (new \GuzzleHttp\Client())->get('https://en.wikipedia.org/wiki/List_of_Unicode_characters')->getBody()->getContents();

$crawler = new \Symfony\Component\DomCrawler\Crawler($wikipedia);

$crawler->filter('tr')->each(function (\Symfony\Component\DomCrawler\Crawler $node) {
//    dump($node->html());
//    $code = $node->eq(0);
//    $glyph = $node->eq(1);

//    if ($node->children()) {
//
//        $tds = $node->filter('td');
//        if ($tds && $tds->count() >= 4) {
//            $decimal = $node->filter('td')->eq(2);
////            dump($decimal->html());
//            if ($match = preg_replace("/^&amp;#(\\d{3});&$/", "$1", $decimal->text())) {
//
//                dump($match);
//
////                if (strlen($match) === 3 && $match >= 0 && $match <= 255) {
////                    $tinyint = (int) $match;
////
////                    dump($match);
////                }
//
//
//            }
//        }

//    }

//    if ($decimal)
//        dump($decimal->html());

    // tr has children
    if ($node->children()) {
        $tds = $node->filter('td');

        // check if all required columns are present
        if ($tds->count() === 4 || $tds->count() === 5 || $tds->count() === 6) {

//            if (startsWith($tds->first()->text(), 'U+')) {
            if (strpos($node->html(), 'U+')) {
//                dump($node->html());
//                $html = $tds->eq($tds->count() -3)->text();

                preg_match_all("/&amp;#(\\d{3});/", $node->html(), $matches);

                if (isset($matches[1][0])) {

                    $decimal = (int)$matches[1][0];

//                dump($decimal);
//                dump($node->html());

                    if ($decimal >= 0 && $decimal <= 255) {
                        dump([$decimal => $tds->eq($tds->count() -2)->text()]);
                    }
                }
            }

//            switch($tds->count()) {
//                case 4:
//                    list($code, $decimal, $description, $abbreviation) = $tds;
//                    break;
//                case 5:
//                    list($code, $glyph, $decimal, $description) = $tds;
//                    break;
//            }

        }
    }
});

function startsWith($haystack, $needle)
{
    $length = strlen($needle);
    return (substr($haystack, 0, $length) === $needle);
}<commit_msg>Cleanup code<commit_after><?php

require __DIR__ . '/vendor/autoload.php';

$wikipedia = (new \GuzzleHttp\Client())->get('https://en.wikipedia.org/wiki/List_of_Unicode_characters')->getBody()->getContents();

$crawler = new \Symfony\Component\DomCrawler\Crawler($wikipedia);

$crawler->filter('tr')->each(function (\Symfony\Component\DomCrawler\Crawler $node) {

    // tr has children
    if ($node->children()) {
        $tds = $node->filter('td');

        // check if all required columns are present
        if ($tds->count() === 4 || $tds->count() === 5 || $tds->count() === 6) {

            // check if row contains the unicode code
            if (strpos($node->html(), 'U+')) {

                // check if row contains a html entity number of length 3
                preg_match_all("/&amp;#(\\d{3});/", $node->html(), $matches);
                if (isset($matches[1][0])) {
                    $decimal = (int)$matches[1][0];

                    // check if number is in the range we intend to use
                    if ($decimal >= 0 && $decimal <= 255) {
                        dump([$decimal => $tds->eq($tds->count() -2)->text()]);
                    }
                }
            }
        }
    }
});
{'fraction_non_alphanumeric': 0.2243502051983584, 'fraction_numerical': 0.01149110807113543, 'mean_word_length': 1.6226685796269726, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 58, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4770139', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1132, 'n_words': 206}