<commit_before><?php

/*
 * This file is a part of Sculpin.
 *
 * (c) Dragonfly Development Inc.
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Sculpin\Core;

use Dflydev\DotAccessConfiguration\Configuration;
use Sculpin\Core\Converter\ConverterManager;
use Sculpin\Core\Event\SourceSetEvent;
use Sculpin\Core\Formatter\FormatterManager;
use Sculpin\Core\Generator\GeneratorManager;
use Sculpin\Core\Output\SourceOutput;
use Sculpin\Core\Output\WriterInterface;
use Sculpin\Core\Permalink\SourcePermalinkFactory;
use Sculpin\Core\Source\DataSourceInterface;
use Sculpin\Core\Source\SourceSet;
use Symfony\Component\EventDispatcher\EventDispatcher;

/**
 * Sculpin.
 *
 * @author <NAME> <<EMAIL>>
 */
class Sculpin
{
    const GIT_VERSION = '@git_version@';

    const EVENT_BEFORE_RUN = 'sculpin.core.before_run';
    const EVENT_AFTER_RUN = 'sculpin.core.after_run';

    const EVENT_BEFORE_CONVERT = 'sculpin.core.before_convert';
    const EVENT_AFTER_CONVERT = 'sculpin.core.after_convert';

    const EVENT_BEFORE_FORMAT = 'sculpin.core.before_format';
    const EVENT_AFTER_FORMAT = 'sculpin.core.after_format';

    /**
     * Site Configuration
     *
     * @var Configuration
     */
    protected $siteConfiguration;

    /**
     * Event Dispatcher
     *
     * @var EventDispatcher
     */
    protected $eventDispatcher;

    /**
     * Permalink factory
     *
     * @var SourcePermalinkFactory
     */
    protected $permalinkFactory;

    /**
     * Writer
     *
     * @var WriterInterface
     */
    protected $writer;

    /**
     * Generator Manager
     *
     * @var GeneratorManager
     */
    protected $generatorManager;

    /**
     * Formatter Manager
     *
     * @var FormatterManager
     */
    protected $formatterManager;

    /**
     * Converter Manager
     *
     * @var ConverterManager
     */
    protected $converterManager;

    /**
     * Constructor.
     *
     * @param Configuration          $siteConfiguration Site Configuration
     * @param EventDispatcher        $eventDispatcher   Event dispatcher
     * @param SourcePermalinkFactory $permalinkFactory  Permalink factory
     * @param WriterInterface        $writer            Writer
     * @param GeneratorManager       $generatorManager  Generator Manager
     * @param FormatterManager       $formatterManager  Formatter Manager
     * @param ConverterManager       $converterManager  Converter Manager
     */
    public function __construct(
        Configuration $siteConfiguration,
        EventDispatcher $eventDispatcher,
        SourcePermalinkFactory $permalinkFactory,
        WriterInterface $writer,
        GeneratorManager $generatorManager,
        FormatterManager $formatterManager,
        ConverterManager $converterManager
    ) {
        $this->siteConfiguration = $siteConfiguration;
        $this->eventDispatcher = $eventDispatcher;
        $this->permalinkFactory = $permalinkFactory;
        $this->writer = $writer;
        $this->generatorManager = $generatorManager;
        $this->formatterManager = $formatterManager;
        $this->converterManager = $converterManager;
    }

    /**
     * Run.
     *
     * @param DataSourceInterface $dataSource Data source
     * @param SourceSet           $sourceSet  Source set
     */
    public function run(DataSourceInterface $dataSource, SourceSet $sourceSet)
    {
        $dataSource->refresh($sourceSet);

        $this->eventDispatcher->dispatch(self::EVENT_BEFORE_RUN, new SourceSetEvent($sourceSet));

        foreach ($sourceSet->updatedSources() as $source) {
            if (!$source->isGenerated()) {
                $this->generatorManager->generate($source, $sourceSet);
            }
        }

        foreach ($sourceSet->updatedSources() as $source) {
            $permalink = $this->permalinkFactory->create($source);
            $source->setPermalink($permalink);
            $source->data()->set('url', $permalink->relativeUrlPath());
        }

        foreach ($sourceSet->updatedSources() as $source) {
            $this->converterManager->convertSource($source);
        }

        foreach ($sourceSet->updatedSources() as $source) {
            if ($source->canBeFormatted()) {
                $source->setFormattedContent($this->formatterManager->formatSourcePage($source));
            } else {
                $source->setFormattedContent($source->content());
            }
        }

        $found = false;

        foreach ($sourceSet->updatedSources() as $source) {
            if ($source->isGenerator() || $source->shouldBeSkipped()) {
                continue;
            }

            if (!$found) {
                print "Detected new or updated files\n";
                $found = true;
            }

            $this->writer->write(new SourceOutput($source));

            print " + {$source->sourceId()}\n";
        }

        $this->eventDispatcher->dispatch(self::EVENT_AFTER_RUN, new SourceSetEvent($sourceSet));
    }
}<commit_msg>Give all sources the blocks metadata.<commit_after><?php

/*
 * This file is a part of Sculpin.
 *
 * (c) Dragonfly Development Inc.
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Sculpin\Core;

use Dflydev\DotAccessConfiguration\Configuration;
use Sculpin\Core\Converter\ConverterManager;
use Sculpin\Core\Event\SourceSetEvent;
use Sculpin\Core\Formatter\FormatterManager;
use Sculpin\Core\Generator\GeneratorManager;
use Sculpin\Core\Output\SourceOutput;
use Sculpin\Core\Output\WriterInterface;
use Sculpin\Core\Permalink\SourcePermalinkFactory;
use Sculpin\Core\Source\DataSourceInterface;
use Sculpin\Core\Source\SourceSet;
use Symfony\Component\EventDispatcher\EventDispatcher;

/**
 * Sculpin.
 *
 * @author <NAME> <<EMAIL>>
 */
class Sculpin
{
    const GIT_VERSION = '@git_version@';

    const EVENT_BEFORE_RUN = 'sculpin.core.before_run';
    const EVENT_AFTER_RUN = 'sculpin.core.after_run';

    const EVENT_BEFORE_CONVERT = 'sculpin.core.before_convert';
    const EVENT_AFTER_CONVERT = 'sculpin.core.after_convert';

    const EVENT_BEFORE_FORMAT = 'sculpin.core.before_format';
    const EVENT_AFTER_FORMAT = 'sculpin.core.after_format';

    /**
     * Site Configuration
     *
     * @var Configuration
     */
    protected $siteConfiguration;

    /**
     * Event Dispatcher
     *
     * @var EventDispatcher
     */
    protected $eventDispatcher;

    /**
     * Permalink factory
     *
     * @var SourcePermalinkFactory
     */
    protected $permalinkFactory;

    /**
     * Writer
     *
     * @var WriterInterface
     */
    protected $writer;

    /**
     * Generator Manager
     *
     * @var GeneratorManager
     */
    protected $generatorManager;

    /**
     * Formatter Manager
     *
     * @var FormatterManager
     */
    protected $formatterManager;

    /**
     * Converter Manager
     *
     * @var ConverterManager
     */
    protected $converterManager;

    /**
     * Constructor.
     *
     * @param Configuration          $siteConfiguration Site Configuration
     * @param EventDispatcher        $eventDispatcher   Event dispatcher
     * @param SourcePermalinkFactory $permalinkFactory  Permalink factory
     * @param WriterInterface        $writer            Writer
     * @param GeneratorManager       $generatorManager  Generator Manager
     * @param FormatterManager       $formatterManager  Formatter Manager
     * @param ConverterManager       $converterManager  Converter Manager
     */
    public function __construct(
        Configuration $siteConfiguration,
        EventDispatcher $eventDispatcher,
        SourcePermalinkFactory $permalinkFactory,
        WriterInterface $writer,
        GeneratorManager $generatorManager,
        FormatterManager $formatterManager,
        ConverterManager $converterManager
    ) {
        $this->siteConfiguration = $siteConfiguration;
        $this->eventDispatcher = $eventDispatcher;
        $this->permalinkFactory = $permalinkFactory;
        $this->writer = $writer;
        $this->generatorManager = $generatorManager;
        $this->formatterManager = $formatterManager;
        $this->converterManager = $converterManager;
    }

    /**
     * Run.
     *
     * @param DataSourceInterface $dataSource Data source
     * @param SourceSet           $sourceSet  Source set
     */
    public function run(DataSourceInterface $dataSource, SourceSet $sourceSet)
    {
        $dataSource->refresh($sourceSet);

        $this->eventDispatcher->dispatch(self::EVENT_BEFORE_RUN, new SourceSetEvent($sourceSet));

        foreach ($sourceSet->updatedSources() as $source) {
            if (!$source->isGenerated()) {
                $this->generatorManager->generate($source, $sourceSet);
            }
        }

        foreach ($sourceSet->updatedSources() as $source) {
            $permalink = $this->permalinkFactory->create($source);
            $source->setPermalink($permalink);
            $source->data()->set('url', $permalink->relativeUrlPath());
        }

        foreach ($sourceSet->updatedSources() as $source) {
            $this->converterManager->convertSource($source);

            if ($source->canBeFormatted()) {
                $source->data()->set('blocks', $this->formatterManager->formatSourceBlocks($source));
            }
        }

        foreach ($sourceSet->updatedSources() as $source) {
            if ($source->canBeFormatted()) {
                $source->setFormattedContent($this->formatterManager->formatSourcePage($source));
            } else {
                $source->setFormattedContent($source->content());
            }
        }

        $found = false;

        foreach ($sourceSet->updatedSources() as $source) {
            if ($source->isGenerator() || $source->shouldBeSkipped()) {
                continue;
            }

            if (!$found) {
                print "Detected new or updated files\n";
                $found = true;
            }

            $this->writer->write(new SourceOutput($source));

            print " + {$source->sourceId()}\n";
        }

        $this->eventDispatcher->dispatch(self::EVENT_AFTER_RUN, new SourceSetEvent($sourceSet));
    }
}
{'fraction_non_alphanumeric': 0.11056201550387597, 'fraction_numerical': 0.0, 'mean_word_length': 2.5333789798014377, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6381134', 'n_tokens_mistral': 2858, 'n_tokens_neox': 2724, 'n_words': 545}