<commit_before><?php namespace OParl\Spec;

use Illuminate\Contracts\Filesystem\Filesystem;
use Pandoc\Pandoc;

class Chapter
{
  protected $filename = '';
  protected $raw = '';
  protected $html = '';
  protected $headlines = [];

  public function __construct(Filesystem $fs, $filename)
  {
    $this->filename = $filename;
    $this->raw = $fs->get($filename);

    $this->parse();
  }

  public function __toString()
  {
    return $this->html;
  }

  protected function parse()
  {
    $this->parseMarkdown();
    $this->parseHeadlines();
  }

  /**
   * @return string
   */
  public function getFilename()
  {
    return basename($this->filename);
  }

  /**
   * @return string
   */
  public function getRaw()
  {
    return $this->raw;
  }

  /**
   * @return string
   */
  public function getHtml()
  {
    return $this->html;
  }

  /**
   * @return array
   */
  public function getHeadlines()
  {
    return $this->headlines;
  }

  protected function parseHeadlines()
  {
    // normalize line breaks
    $text = str_replace(array("\r\n", "\r"), "\n", $this->raw);

    // convert underline-style headlines to hash-style headlines
    $text = preg_replace('/(.+)\n(=){2,}/', '# $1', $text);
    $text = preg_replace('/(.+)\n(-){2,}/', '## $1', $text);

    // check for headlines per line
    $this->headlines = collect(explode("\n", $text))->filter(function ($line) {
      return starts_with(trim($line), '#');
    })->map(function ($headline) {
      $level = 1;

      while (strlen($headline) >= $level && $headline[$level] === '#')
        $level++;

      return [
        'level' => $level,
        'text' => trim($headline, '# ')
      ];
    });
  }

  protected function parseMarkdown()
  {
    // transform with pandoc
    $pandoc = new Pandoc();

    $this->html = $pandoc->runWith($this->raw, [
      'from' => 'markdown',
      'to' => 'html5',
    ]);

    // fix image urls
    $this->html = preg_replace('/"(.?)(images\/.+\.png)"/', '"$1/specification/$2"', $this->html);

    // fix image tags
    $this->html = str_replace('<img ', '<img class="img-responsive"', $this->html);
  }<commit_msg>better pandoc config<commit_after><?php namespace OParl\Spec;

use Illuminate\Contracts\Filesystem\Filesystem;
use Pandoc\Pandoc;

class Chapter
{
  protected $filename = '';
  protected $raw = '';
  protected $html = '';
  protected $headlines = [];

  protected static $chapterCount = -1;
  protected static $sectionCount = -1;

  public function __construct(Filesystem $fs, $filename)
  {
    $this->filename = $filename;
    $this->raw = $fs->get($filename);

    $this->parse();
  }

  public function __toString()
  {
    return $this->html;
  }

  protected function parse()
  {
    $this->parseMarkdown();
    $this->parseHeadlines();
  }

  /**
   * @return string
   */
  public function getFilename()
  {
    return basename($this->filename);
  }

  /**
   * @return string
   */
  public function getRaw()
  {
    return $this->raw;
  }

  /**
   * @return string
   */
  public function getHtml()
  {
    return $this->html;
  }

  /**
   * @return array
   */
  public function getHeadlines()
  {
    return $this->headlines;
  }

  protected function parseHeadlines()
  {
    // normalize line breaks
    $text = str_replace(array("\r\n", "\r"), "\n", $this->raw);

    // convert underline-style headlines to hash-style headlines
    $text = preg_replace('/(.+)\n(=){2,}/', '# $1', $text);
    $text = preg_replace('/(.+)\n(-){2,}/', '## $1', $text);

    // check for headlines per line
    $this->headlines = collect(explode("\n", $text))->filter(function ($line) {
      return starts_with(trim($line), '#');
    })->map(function ($headline) {
      $level = 1;

      while (strlen($headline) >= $level && $headline[$level] === '#')
        $level++;

      if ($level == 1)
      {
        static::$chapterCount++;
        static::$sectionCount = 0;
      }

      //if ($level == 2)

      return [
        'level' => $level,
        'text' => trim($headline, '# ')
      ];
    });
  }

  protected function parseMarkdown()
  {
    // transform with pandoc
    $pandoc = new Pandoc();

    $this->html = $pandoc->runWith($this->raw, [
      'to' => 'html5',
      'number-sections' => null,
      'section-divs' => null,
      'from' => 'markdown',
      'smart' => null,
      'number-offset' => static::$chapterCount
    ]);

    // fix image urls
    $this->html = preg_replace('/"(.?)(images\/.+\.png)"/', '"$1/specification/$2"', $this->html);

    // fix image tags
    $this->html = str_replace('<img ', '<img class="img-responsive"', $this->html);
  }
{'fraction_non_alphanumeric': 0.20082411624376492, 'fraction_numerical': 0.004554326610279766, 'mean_word_length': 2.600312256049961, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7618630', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1525, 'n_words': 317}