<commit_before><?php

namespace Marquine\Etl;

use IteratorAggregate;

class Pipeline
{
    /**
     * The pipeline data flow.
     *
     * @var \IteratorAggregate
     */
    protected $flow;

    /**
     * The array of tasks.
     *
     * @var array
     */
    protected $tasks = [];

    /**
     * Current row.
     *
     * @var int
     */
    protected $current = 0;

    /**
     * Total rows.
     *
     * @var int
     */
    protected $total = 0;

    /**
     * Maximum number of rows.
     *
     * @var int
     */
    protected $limit;

    /**
     * Number of rows to skip.
     *
     * @var int
     */
    protected $skip;

    /**
     * Pre execution callbacks.
     *
     * @var array
     */
    protected $preExecutionCallbacks = [];

    /**
     * Post execution callbacks.
     *
     * @var array
     */
    protected $postExecutionCallbacks = [];

    /**
     * Set the pipeline flow.
     *
     * @param  \IteratorAggregate  $flow
     * @return void
     */
    public function flow(IteratorAggregate $flow)
    {
        $this->flow = $flow;
    }

    /**
     * Pipe a task.
     *
     * @param  callable  $task
     * @return $this
     */
    public function pipe(callable $task)
    {
        $this->tasks[] = $task;

        return $this;
    }

    /**
     * Get the pipeline data generator.
     *
     * @return \Generator
     */
    public function get()
    {
        $this->total = $this->getRowsCount();

        foreach ($this->preExecutionCallbacks as $callback) {
            $callback();
        }

        foreach ($this->flow as $index => $row) {
            if ($this->skip && $index < $this->skip) {
                continue;
            }

            if ($this->limit && $this->current == $this->limit) {
                break;
            }

            $this->current++;

            $row = $this->runTasks($row);

            if (!empty($row)) {
                yield $row;
            }
        }

        foreach ($this->postExecutionCallbacks as $callback) {
            $callback();
        }
    }

    /**
     * Get a sample row of the flow.
     *
     * @return array
     */
    public function sample()
    {
        return $this->flow->getIterator()->current();
    }

    /**
     * Run tasks for the given row.
     *
     * @param  array  $row
     * @return array
     */
    protected function runTasks($row)
    {
        foreach ($this->tasks as $task) {
            $row = $task($row);
        }

        return $row;
    }

    /**
     * Get the total rows count.
     *
     * @return int
     */
    protected function getRowsCount()
    {
        $count = iterator_count($this->flow);

        if ($this->skip) {
            $count -= $this->skip;
        }

        if ($this->limit && $count > $this->limit) {
            $count = $this->limit;
        }
<commit_msg>Prevent running steps for empty rows<commit_after><?php

namespace Marquine\Etl;

use IteratorAggregate;

class Pipeline
{
    /**
     * The pipeline data flow.
     *
     * @var \IteratorAggregate
     */
    protected $flow;

    /**
     * The array of tasks.
     *
     * @var array
     */
    protected $tasks = [];

    /**
     * Current row.
     *
     * @var int
     */
    protected $current = 0;

    /**
     * Total rows.
     *
     * @var int
     */
    protected $total = 0;

    /**
     * Maximum number of rows.
     *
     * @var int
     */
    protected $limit;

    /**
     * Number of rows to skip.
     *
     * @var int
     */
    protected $skip;

    /**
     * Pre execution callbacks.
     *
     * @var array
     */
    protected $preExecutionCallbacks = [];

    /**
     * Post execution callbacks.
     *
     * @var array
     */
    protected $postExecutionCallbacks = [];

    /**
     * Set the pipeline flow.
     *
     * @param  \IteratorAggregate  $flow
     * @return void
     */
    public function flow(IteratorAggregate $flow)
    {
        $this->flow = $flow;
    }

    /**
     * Pipe a task.
     *
     * @param  callable  $task
     * @return $this
     */
    public function pipe(callable $task)
    {
        $this->tasks[] = $task;

        return $this;
    }

    /**
     * Get the pipeline data generator.
     *
     * @return \Generator
     */
    public function get()
    {
        $this->total = $this->getRowsCount();

        foreach ($this->preExecutionCallbacks as $callback) {
            $callback();
        }

        foreach ($this->flow as $index => $row) {
            if ($this->skip && $index < $this->skip) {
                continue;
            }

            if ($this->limit && $this->current == $this->limit) {
                break;
            }

            $this->current++;

            $row = $this->runTasks($row);

            if (!empty($row)) {
                yield $row;
            }
        }

        foreach ($this->postExecutionCallbacks as $callback) {
            $callback();
        }
    }

    /**
     * Get a sample row of the flow.
     *
     * @return array
     */
    public function sample()
    {
        return $this->flow->getIterator()->current();
    }

    /**
     * Run tasks for the given row.
     *
     * @param  array  $row
     * @return array
     */
    protected function runTasks($row)
    {
        foreach ($this->tasks as $task) {
            $row = $task($row);

            if (empty($row)) {
                break;
            }
        }

        return $row;
    }

    /**
     * Get the total rows count.
     *
     * @return int
     */
    protected function getRowsCount()
    {
        $count = iterator_count($this->flow);

        if ($this->skip) {
            $count -= $this->skip;
        }

        if ($this->limit && $count > $this->limit) {
            $count = $this->limit;
        }

{'fraction_non_alphanumeric': 0.14050154586052901, 'fraction_numerical': 0.0006870491240123669, 'mean_word_length': 1.3034018987341771, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6767193', 'n_tokens_mistral': 1781, 'n_tokens_neox': 1728, 'n_words': 410}