<commit_before><?php

class Wiki
{
    protected $_renderers = array(
        'md' => 'Markdown',
        'htm' => 'HTML', 'html' => 'HTML'
    );
    protected $_ignore = array(".", "..", ".svn", ".git", ".hg", "CVS", ".sass-cache", ".bundle", ".gitignore", ".gitkeep", ".sass-cache", ".DS_Store");

    protected $_action;

    protected $_default_page_data = array(
        'title'       => false, // will use APP_NAME by default
        'description' => 'Wikitten is a small, fast, PHP wiki.',
        'tags'        => array('wikitten', 'wiki')
    );

    protected function _getRenderer($extension)
    {
        if (!isset($this->_renderers[$extension])) {
            return false;
        }

        $renderer = $this->_renderers[$extension];

        require_once __DIR__ . DIRECTORY_SEPARATOR . 'renderers' . DIRECTORY_SEPARATOR . "$renderer.php";

        return $renderer;
    }

    protected function _render($page)
    {
        $not_found = function () use ($page) {
            $page = htmlspecialchars($page, ENT_QUOTES);
            throw new Exception("Page '$page' was not found");
        };

        $path = realpath(LIBRARY . DIRECTORY_SEPARATOR . $page);

        if (!$path) {
            return $not_found();
        }

        if (strpos($path, LIBRARY) !== 0) {
            return $not_found();
        }

        if (!file_exists($path)) {
            return $not_found();
        }

        $finfo = finfo_open(FILEINFO_MIME);
        $mime_type = finfo_file($finfo, $path);

        if (substr($mime_type, 0, 4) != 'text') {
            // not an ASCII file, send it directly to the browser

            $file = fopen($path, 'rb');

            header("Content-Type: $mime_type");
            header("Content-Length: " . filesize($path));

            fpassthru($file);
            exit();
        }

        $source    = file_get_contents($path);
        $extension = pathinfo($path, PATHINFO_EXTENSION);
        $renderer  = $this->_getRenderer($extension);
        $page_data = $this->_default_page_data;

        // Extract the JSON header, if the feature is enabled:
        if(USE_PAGE_METADATA) {
            list($source, $meta_data) = $this->_extractJsonFrontMatter($source);
            $page_data = array_merge($page_data, $meta_data);
        }

        $html = false;
        if ($renderer) {
            $html = $renderer($source);
        }

        $parts = explode('/', $page);

        return $this->_view('render', array(
            'html'      => $html,
            'source'    => $source,
            'extension' => $extension,
            'parts'     => $parts,
            'page'      => $page_data
        ));
    }

    /**
     * Given a string with a page's source, attempts to locate a
     * section of JSON Front Matter in the heading, and returns
     * the remaining source, and an array of extracted meta data.
     *
     * JSON Front Matter will only be considered when present
     * within two lines consisting of three dashes:
     *
     * ---
     * { "title": "hello world" }
     * ---
     *
     * Additionally, the opening and closing brackets may be dropped,
     * and this method will still interpret the content as a hash:
     *
     * ---
     * "title": "hello, world",
     * "tags":  ["hello", "world"]
     * ---
     *
     * @param  string $source
     * @return array  array($remaining_source, $meta_data)
     */
    protected function _extractJsonFrontMatter($source)
    {
        static $front_matter_regex = "/^---\n(.*)\n---\n(.*)/ms";

        $source    = ltrim($source);
        $meta_data = array();

        if(preg_match($front_matter_regex, $source, $matches)) {
            $json   = trim($matches[1]);
            $source = trim($matches[2]);
<commit_msg>Fix so that metadata is only read in header<commit_after><?php

class Wiki
{
    protected $_renderers = array(
        'md' => 'Markdown',
        'htm' => 'HTML', 'html' => 'HTML'
    );
    protected $_ignore = array(".", "..", ".svn", ".git", ".hg", "CVS", ".sass-cache", ".bundle", ".gitignore", ".gitkeep", ".sass-cache", ".DS_Store");

    protected $_action;

    protected $_default_page_data = array(
        'title'       => false, // will use APP_NAME by default
        'description' => 'Wikitten is a small, fast, PHP wiki.',
        'tags'        => array('wikitten', 'wiki')
    );

    protected function _getRenderer($extension)
    {
        if (!isset($this->_renderers[$extension])) {
            return false;
        }

        $renderer = $this->_renderers[$extension];

        require_once __DIR__ . DIRECTORY_SEPARATOR . 'renderers' . DIRECTORY_SEPARATOR . "$renderer.php";

        return $renderer;
    }

    protected function _render($page)
    {
        $not_found = function () use ($page) {
            $page = htmlspecialchars($page, ENT_QUOTES);
            throw new Exception("Page '$page' was not found");
        };

        $path = realpath(LIBRARY . DIRECTORY_SEPARATOR . $page);

        if (!$path) {
            return $not_found();
        }

        if (strpos($path, LIBRARY) !== 0) {
            return $not_found();
        }

        if (!file_exists($path)) {
            return $not_found();
        }

        $finfo = finfo_open(FILEINFO_MIME);
        $mime_type = finfo_file($finfo, $path);

        if (substr($mime_type, 0, 4) != 'text') {
            // not an ASCII file, send it directly to the browser

            $file = fopen($path, 'rb');

            header("Content-Type: $mime_type");
            header("Content-Length: " . filesize($path));

            fpassthru($file);
            exit();
        }

        $source    = file_get_contents($path);
        $extension = pathinfo($path, PATHINFO_EXTENSION);
        $renderer  = $this->_getRenderer($extension);
        $page_data = $this->_default_page_data;

        // Extract the JSON header, if the feature is enabled:
        if(USE_PAGE_METADATA) {
            list($source, $meta_data) = $this->_extractJsonFrontMatter($source);
            $page_data = array_merge($page_data, $meta_data);
        }

        $html = false;
        if ($renderer) {
            $html = $renderer($source);
        }

        $parts = explode('/', $page);

        return $this->_view('render', array(
            'html'      => $html,
            'source'    => $source,
            'extension' => $extension,
            'parts'     => $parts,
            'page'      => $page_data
        ));
    }

    /**
     * Given a string with a page's source, attempts to locate a
     * section of JSON Front Matter in the heading, and returns
     * the remaining source, and an array of extracted meta data.
     *
     * JSON Front Matter will only be considered when present
     * within two lines consisting of three dashes:
     *
     * ---
     * { "title": "hello world" }
     * ---
     *
     * Additionally, the opening and closing brackets may be dropped,
     * and this method will still interpret the content as a hash:
     *
     * ---
     * "title": "hello, world",
     * "tags":  ["hello", "world"]
     * ---
     *
     * @param  string $source
     * @return array  array($remaining_source, $meta_data)
     */
    protected function _extractJsonFrontMatter($source)
    {
        static $front_matter_regex = "/^---\n(.*)\n---\n(.*)/s";

        $source    = ltrim($source);
        $meta_data = array();

        if(preg_match($front_matter_regex, $source, $matches)) {
            $json   = trim($matches[1]);
            $source = trim($matches[2]);

{'fraction_non_alphanumeric': 0.15013901760889714, 'fraction_numerical': 0.0013239772275916854, 'mean_word_length': 1.968172888015717, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3238037', 'n_tokens_mistral': 2303, 'n_tokens_neox': 2210, 'n_words': 560}