<commit_before><?php

namespace Ciconia\Extension\Gfm;

use Ciconia\Common\Collection;
use Ciconia\Common\Tag;
use Ciconia\Common\Text;
use Ciconia\Extension\ExtensionInterface;
use Ciconia\Markdown;
use Ciconia\Renderer\HtmlRenderer;
use Ciconia\Renderer\RendererAwareInterface;
use Ciconia\Renderer\RendererAwareTrait;

/**
 * Gfm tables
 *
 * @author <NAME> <<EMAIL>>
 */
class TableExtension implements ExtensionInterface, RendererAwareInterface
{

    use RendererAwareTrait;

    /**
     * @var Markdown
     */
    private $markdown;

    /**
     * @var string
     */
    private $hash;

    /**
     * {@inheritdoc}
     */
    public function register(Markdown $markdown)
    {
        $this->markdown = $markdown;
        $this->hash = '{gfm:table:escape(' . md5('|') . ')}';

        if ($this->getRenderer() instanceof HtmlRenderer) {
            // Output format depends on HTML for now
            $markdown->on('block', array($this, 'processTable'));
        }
    }

    /**
     * Gfm tables
     *
     * @param Text  $text
     * @param array $options
     */
    public function processTable(Text $text, array $options = array())
    {
        $text->replace('/^
            (?:[ ]{0,3}      #  table header
                (?:\|?)      #  optional outer pipe
                (.*?)        #1 table header
                (?:\|?)      #  optional outer pipe
            )\n
            (?:[ ]{0,3}      #  second line
                (?:\|?)      #  optional outer pipe
                ([-:| ]+?)   #2 dashes and pipes
                (?:\|?)      #  optional outer pipe
            )\n
            [ ]{0,3}(        #3 table body
                (?:.*\n?)*
            )\n\n
        /mx', function (Text $w, Text $header, Text $rule, Text $body) use ($options) {
            // Escape pipe to hash, so you can include pipe in cells by escaping it like this: `\\|`
            $this->escapePipes($header);
            $this->escapePipes($rule);
            $this->escapePipes($body);

            $baseTags = $this->createBaseTags($rule->split('/\|/'));

            $headerCells = new Collection();
            $bodyRows = new Collection();

            $header->split('/\|/')->each(function (Text $cell, $index) use ($baseTags, &$headerCells) {
                /* @var Tag $tag */
                $tag = clone $baseTags->get($index);
                $tag->setName('th');
                $this->markdown->emit('inline', array($cell));
                $tag->setText($cell->trim());

                $headerCells->add($tag);
            });

            $body->split('/\n/')->each(function (Text $row) use ($baseTags, &$bodyRows) {
                $row->trim()->trim('|');
                $cells = new Collection();
                $row->split('/\|/')->each(function (Text $cell, $index) use (&$baseTags, &$cells) {
                    /* @var Tag $tag */
                    $tag = clone $baseTags->get($index);
                    $this->markdown->emit('inline', array($cell));
                    $tag->setText($cell->trim());

                    $cells->add($tag);
                });

                $bodyRows->add($cells);
            });

            $tHeadRow = new Tag('tr');
            $tHeadRow->setText("\n" . $headerCells->join("\n") . "\n");

            $tHead = new Tag('thead');
            $tHead->setText("\n" . $tHeadRow . "\n");

            $tBody = new Tag('tbody');

            $bodyRows->apply(function (Collection $row) use (&$options) {
                $tr = new Tag('tr');
                $tr->setText("\n" . $row->join("\n") . "\n");

                return $tr;
            });

            $tBody->setText("\n" .$bodyRows->join("\n") . "\n");

            $table = new Tag('table');
            $table->setText("\n" . $tHead . "\n" . $tBody . "\n");

            $html = new Text($table->render());
            $this->unescapePipes($html);

            return $html . "\n\n";
        });
    }

    /**
     * @param Collection $rules
     *
     * @return Collection|\Ciconia\Common\Tag[]
     */
    protected function createBaseTags(Collection $rules)
    {
        /* @var Collection|Tag[] $baseTags */
        $baseTags = new Collection();

        $rules->each(function (Text $cell) use (&$baseTags) {
            $cell->trim();
            $tag = new Tag('td');

            if ($cell->match('/^-.*:$/')) {
                $tag->setAttribute('align', 'right');
            } elseif ($cell->match('/^:.*:$/')) {
                $tag->setAttribute('align', 'center');
            }

            $baseTags->add($tag);
        });

        return $baseTags;
    }

    /**
     * @param Text $text
     */
    protected function escapePipes(Text $text)
    {
        $text->replaceString('\\|', $this->hash);
    }

    /**
     * @param Text $text
     */
    protected function unescapePipes(Text $text)
    {
        $text->replaceString($this->hash, '|');
    }

    /**
     * {@inheritdoc}
     */
    public function getName()
    {
        return 'table'; // Not gfmTable
    }

}<commit_msg>number of leading spaces on regexp should be optional<commit_after><?php

namespace Ciconia\Extension\Gfm;

use Ciconia\Common\Collection;
use Ciconia\Common\Tag;
use Ciconia\Common\Text;
use Ciconia\Extension\ExtensionInterface;
use Ciconia\Markdown;
use Ciconia\Renderer\HtmlRenderer;
use Ciconia\Renderer\RendererAwareInterface;
use Ciconia\Renderer\RendererAwareTrait;

/**
 * Gfm tables
 *
 * @author <NAME> <<EMAIL>>
 */
class TableExtension implements ExtensionInterface, RendererAwareInterface
{

    use RendererAwareTrait;

    /**
     * @var Markdown
     */
    private $markdown;

    /**
     * @var string
     */
    private $hash;

    /**
     * {@inheritdoc}
     */
    public function register(Markdown $markdown)
    {
        $this->markdown = $markdown;
        $this->hash = '{gfm:table:escape(' . md5('|') . ')}';

        if ($this->getRenderer() instanceof HtmlRenderer) {
            // Output format depends on HTML for now
            $markdown->on('block', array($this, 'processTable'));
        }
    }

    /**
     * {@inheritdoc}
     */
    public function getName()
    {
        return 'table'; // Not gfmTable
    }

    /**
     * Gfm tables
     *
     * @param Text  $text
     * @param array $options
     */
    public function processTable(Text $text, array $options = array())
    {
        $lessThanTab = $options['tabWidth'] - 1;

        $text->replace('/^
            (?:[ ]{0,' . $lessThanTab . '}      #  table header
                (?:\|?)                         #  optional outer pipe
                (.*?)                           #1 table header
                (?:\|?)                         #  optional outer pipe
            )\n
            (?:[ ]{0,' . $lessThanTab . '}      #  second line
                (?:\|?)                         #  optional outer pipe
                ([-:| ]+?)                      #2 dashes and pipes
                (?:\|?)                         #  optional outer pipe
            )\n
            [ ]{0,' . $lessThanTab . '}(        #3 table body
                (?:.*\n?)*
            )\n\n
        /mx', function (Text $w, Text $header, Text $rule, Text $body) use ($options) {
            // Escape pipe to hash, so you can include pipe in cells by escaping it like this: `\\|`
            $this->escapePipes($header);
            $this->escapePipes($rule);
            $this->escapePipes($body);

            $baseTags = $this->createBaseTags($rule->split('/\|/'));

            $headerCells = new Collection();
            $bodyRows = new Collection();

            $header->split('/\|/')->each(function (Text $cell, $index) use ($baseTags, &$headerCells) {
                /* @var Tag $tag */
                $tag = clone $baseTags->get($index);
                $tag->setName('th');
                $this->markdown->emit('inline', array($cell));
                $tag->setText($cell->trim());

                $headerCells->add($tag);
            });

            $body->split('/\n/')->each(function (Text $row) use ($baseTags, &$bodyRows) {
                $row->trim()->trim('|');
                $cells = new Collection();
                $row->split('/\|/')->each(function (Text $cell, $index) use (&$baseTags, &$cells) {
                    /* @var Tag $tag */
                    $tag = clone $baseTags->get($index);
                    $this->markdown->emit('inline', array($cell));
                    $tag->setText($cell->trim());

                    $cells->add($tag);
                });

                $bodyRows->add($cells);
            });

            $html = $this->createView($headerCells, $bodyRows);

            $this->unescapePipes($html);

            return $html . "\n\n";
        });
    }

    /**
     * @param Collection $headerCells
     * @param Collection $bodyRows
     *
     * @return Text
     */
    protected function createView(Collection $headerCells, Collection $bodyRows)
    {
        $tHeadRow = new Tag('tr');
        $tHeadRow->setText("\n" . $headerCells->join("\n") . "\n");

        $tHead = new Tag('thead');
        $tHead->setText("\n" . $tHeadRow . "\n");

        $tBody = new Tag('tbody');

        $bodyRows->apply(function (Collection $row) use (&$options) {
            $tr = new Tag('tr');
            $tr->setText("\n" . $row->join("\n") . "\n");

            return $tr;
        });

        $tBody->setText("\n" .$bodyRows->join("\n") . "\n");

        $table = new Tag('table');
        $table->setText("\n" . $tHead . "\n" . $tBody . "\n");

        return new Text($table->render());
    }

    /**
     * @param Collection $rules
     *
     * @return Collection|\Ciconia\Common\Tag[]
     */
    protected function createBaseTags(Collection $rules)
    {
        /* @var Collection|Tag[] $baseTags */
        $baseTags = new Collection();

        $rules->each(function (Text $cell) use (&$baseTags) {
            $cell->trim();
            $tag = new Tag('td');

            if ($cell->match('/^-.*:$/')) {
                $tag->setAttribute('align', 'right');
            } elseif ($cell->match('/^:.*:$/')) {
                $tag->setAttribute('align', 'center');
            }

            $baseTags->add($tag);
        });

        return $baseTags;
    }

    /**
     * @param Text $text
     */
    protected function escapePipes(Text $text)
    {
        $text->replaceString('\\|', $this->hash);
    }

    /**
     * @param Text $text
     */
    protected function unescapePipes(Text $text)
    {
        $text->replaceString($this->hash, '|');
    }

}
{'fraction_non_alphanumeric': 0.1802737140160453, 'fraction_numerical': 0.0016989145823501652, 'mean_word_length': 1.5888101636941119, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 118, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3860725', 'n_tokens_mistral': 3194, 'n_tokens_neox': 3005, 'n_words': 637}