<commit_before><?php

// TODO(<EMAIL>): Check if reachable from given entry points

$pagesDirectory = realpath(__DIR__ . '/../../pages');
$agiDirectory = realpath($pagesDirectory.'/agi');

$startPages = [
    $agiDirectory => true,
];

$pattern = '/\.md$/';
$directoryIterator = new RecursiveDirectoryIterator($agiDirectory);
$iterator = new RecursiveIteratorIterator($directoryIterator);
$files = new RegexIterator($iterator, $pattern, RegexIterator::MATCH);

$redBackground = "\033[41m";
$greenBackground = "\033[42m";
$endColor = "\033[0m";

$countBroken = 0;
$references = [];
$articles = [];

// Currently ignores assets, might be nice to test them at some point
$articleRegex = '/\[[^\]]+\]\((?!.*\/\/)(?<path>[^)]+)\)/';
foreach ($files as $file) {
    echo 'Scanning ' . $file . PHP_EOL;
    $fileDirectory = dirname($file);
    $articles[$fileDirectory] = true;
    $content = file_get_contents($file);
    preg_match_all($articleRegex, $content, $matches);
    foreach ($matches['path'] as $path) {<commit_msg>Slightly improve the regex used to find links to other articles so that it doesn't ignore lines with both a link to an article and a link to an external web page.<commit_after><?php

// TODO(<EMAIL>): Check if reachable from given entry points

$pagesDirectory = realpath(__DIR__ . '/../../pages');
$agiDirectory = realpath($pagesDirectory.'/agi');

$startPages = [
    $agiDirectory => true,
];

$pattern = '/\.md$/';
$directoryIterator = new RecursiveDirectoryIterator($agiDirectory);
$iterator = new RecursiveIteratorIterator($directoryIterator);
$files = new RegexIterator($iterator, $pattern, RegexIterator::MATCH);

$redBackground = "\033[41m";
$greenBackground = "\033[42m";
$endColor = "\033[0m";

$countBroken = 0;
$references = [];
$articles = [];

// Currently ignores assets, might be nice to test them at some point
$articleRegex = '/\[[^\]]+\]\((?!.{0,5}\:\/\/)(?<path>[^)]+)\)/';
foreach ($files as $file) {
    echo 'Scanning ' . $file . PHP_EOL;
    $fileDirectory = dirname($file);
    $articles[$fileDirectory] = true;
    $content = file_get_contents($file);
    preg_match_all($articleRegex, $content, $matches);
    foreach ($matches['path'] as $path) {
{'fraction_non_alphanumeric': 0.1876993166287016, 'fraction_numerical': 0.01457858769931663, 'mean_word_length': 4.7486910994764395, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5382089', 'n_tokens_mistral': 717, 'n_tokens_neox': 686, 'n_words': 181}