<commit_before><?php

/**
 * @copyright 2017 <NAME>
 * @license   https://github.com/allejo/stakx/blob/master/LICENSE.md MIT
 */

namespace allejo\stakx\Engines\Markdown;

use allejo\stakx\Configuration;
use allejo\stakx\Engines\ParsingEngine;
use allejo\stakx\Manager\HighlighterManager;
use allejo\stakx\Service;
use Highlight\Highlighter;

class MarkdownEngine extends \ParsedownExtra implements ParsingEngine
{
    protected $highlighter;

    public function __construct()
    {
        parent::__construct();

        $this->highlighter = new Highlighter();
    }

    protected function blockHeader($line)
    {
        $Block = parent::blockHeader($line);

        // Create our unique ids by sanitizing the header content
        $id = strtolower($Block['element']['text']);
        $id = str_replace(' ', '-', $id);
        $id = preg_replace('/[^0-9a-zA-Z-_]/', '', $id);
        $id = preg_replace('/-+/', '-', $id);

        $Block['element']['attributes']['id'] = $id;

        return $Block;
    }

    public function blockFencedCodeComplete($block)
    {
        // The class has a `language-` prefix, remove this to get the language
        if (isset($block['element']['text']['attributes']) && Service::getParameter(Configuration::HIGHLIGHTER_ENABLED))
        {
            $cssClass = &$block['element']['text']['attributes']['class'];
            $language = substr($cssClass, 9);
            $cssClass = 'hljs ' . $cssClass;

            try
            {
                $highlighted = $this->highlighter->highlight($language, $block['element']['text']['text']);
                $block['element']['text']['text'] = $highlighted->value;

                // Only return the block if Highlighter knew the language and how to handle it.
                return $block;
            }
            // Exception thrown when language not supported
            catch (\DomainException $exception)
            {
                trigger_error("An unsupported language ($language) was detected in a code block", E_USER_WARNING);
            }
        }

        return parent::blockFencedCodeComplete($block);
    }
}
<commit_msg>Fix MD anchors on certain MD headings<commit_after><?php

/**
 * @copyright 2017 <NAME>
 * @license   https://github.com/allejo/stakx/blob/master/LICENSE.md MIT
 */

namespace allejo\stakx\Engines\Markdown;

use allejo\stakx\Configuration;
use allejo\stakx\Engines\ParsingEngine;
use allejo\stakx\Manager\HighlighterManager;
use allejo\stakx\Service;
use Highlight\Highlighter;

class MarkdownEngine extends \ParsedownExtra implements ParsingEngine
{
    protected $highlighter;

    public function __construct()
    {
        parent::__construct();

        $this->highlighter = new Highlighter();
    }

    protected function blockHeader($Line)
    {
        $Block = parent::blockHeader($Line);

        if (isset($Block['element']['text']))
        {
            $Block['element']['attributes']['id'] = $this->slugifyHeader($Block);
        }

        return $Block;
    }

    protected function blockSetextHeader($Line, array $Block = null)
    {
        $Block = parent::blockSetextHeader($Line, $Block);

        if (isset($Block['element']['name']))
        {
            $element = $Block['element']['name'];

            if ($element == 'h1' || $element == 'h2')
            {
                $Block['element']['attributes']['id'] = $this->slugifyHeader($Block);
            }
        }

        return $Block;
    }

    private function slugifyHeader($Block)
    {
        $id = strtolower($Block['element']['text']);
        $id = str_replace(' ', '-', $id);
        $id = preg_replace('/[^0-9a-zA-Z-_]/', '', $id);

        return preg_replace('/-+/', '-', $id);
    }

    protected function blockFencedCodeComplete($block)
    {
        // The class has a `language-` prefix, remove this to get the language
        if (isset($block['element']['text']['attributes']) && Service::getParameter(Configuration::HIGHLIGHTER_ENABLED))
        {
            $cssClass = &$block['element']['text']['attributes']['class'];
            $language = substr($cssClass, 9);
            $cssClass = 'hljs ' . $cssClass;

            try
            {
                $highlighted = $this->highlighter->highlight($language, $block['element']['text']['text']);
                $block['element']['text']['text'] = $highlighted->value;

                // Only return the block if Highlighter knew the language and how to handle it.
                return $block;
            }
            // Exception thrown when language not supported
            catch (\DomainException $exception)
            {
                trigger_error("An unsupported language ($language) was detected in a code block", E_USER_WARNING);
            }
        }

        return parent::blockFencedCodeComplete($block);
    }
}

{'fraction_non_alphanumeric': 0.15008257638315442, 'fraction_numerical': 0.0033030553261767133, 'mean_word_length': 2.4483985765124556, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1883994', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1325, 'n_words': 290}