<commit_before><?php

/*
 * This file is part of the Endec package.
 *
 * Copyright © 2014 <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Eloquent\Endec\Transform;

use php_user_filter;

/**
 * An abstract base class for implementing native stream filters based on
 * transforms.
 */
abstract class AbstractNativeStreamFilter extends php_user_filter
{
    /**
     * Called upon filter creation.
     */
    public function onCreate()
    {
        $this->transform = $this->createTransform();
        $this->buffer = '';

        return true;
    }

    /**
     * Filter the input data through the transform.
     *
     * @param resource $input          The input bucket brigade.
     * @param resource $output         The output bucket brigade.
     * @param integer  &$consumedBytes The number of bytes consumed.
     * @param boolean  $isEnd          True if the stream is closing.
     *
     * @return integer The result code.
     */
    public function filter($input, $output, &$consumedBytes, $isEnd)
    {
        if ($isEnd) {
            $bucket = stream_bucket_new($this->stream, '');
        } else {
            $bucket = stream_bucket_make_writeable($input);
        }

        $hasOutput = false;
        while ($bucket) {
            $this->buffer .= $bucket->data;
            $bufferLength = strlen($this->buffer);

            try {
                list($outputBuffer, $thisConsumedBytes) = $this->transform
                    ->transform($this->buffer, $isEnd);
            } catch (Exception\TransformExceptionInterface $e) {
                return PSFS_ERR_FATAL;
            }

            $consumedBytes += $thisConsumedBytes;
            if ($bufferLength === $thisConsumedBytes) {
                $this->buffer = '';
            } else {
                $this->buffer = substr($this->buffer, $thisConsumedBytes);
            }

            if ('' !== $outputBuffer) {
                $bucket->data = $outputBuffer;
                stream_bucket_append($output, $bucket);
                $hasOutput = true;
            }

            $bucket = stream_bucket_make_writeable($input);
        }

        if ($hasOutput || $isEnd) {
            return PSFS_PASS_ON;
        }

        return PSFS_FEED_ME;
    }

    /**
     * Create the transform.
     *
     * @return DataTransformInterface The data transform.
     */
    abstract protected function createTransform();

    private $transform;
    private $buffer;
}
<commit_msg>Fix native stream filters for shutdown case.<commit_after><?php

/*
 * This file is part of the Endec package.
 *
 * Copyright © 2014 <NAME>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Eloquent\Endec\Transform;

use php_user_filter;

/**
 * An abstract base class for implementing native stream filters based on
 * transforms.
 */
abstract class AbstractNativeStreamFilter extends php_user_filter
{
    /**
     * Called upon filter creation.
     */
    public function onCreate()
    {
        $this->transform = $this->createTransform();
        $this->buffer = '';

        return true;
    }

    /**
     * Filter the input data through the transform.
     *
     * @param resource $input          The input bucket brigade.
     * @param resource $output         The output bucket brigade.
     * @param integer  &$consumedBytes The number of bytes consumed.
     * @param boolean  $isEnd          True if the stream is closing.
     *
     * @return integer The result code.
     */
    public function filter($input, $output, &$consumedBytes, $isEnd)
    {
        if ($isEnd && is_resource($this->stream)) {
            $bucket = stream_bucket_new($this->stream, '');
        } else {
            $bucket = stream_bucket_make_writeable($input);
        }

        $hasOutput = false;
        while ($bucket) {
            $this->buffer .= $bucket->data;
            $bufferLength = strlen($this->buffer);

            try {
                list($outputBuffer, $thisConsumedBytes) = $this->transform
                    ->transform($this->buffer, $isEnd);
            } catch (Exception\TransformExceptionInterface $e) {
                return PSFS_ERR_FATAL;
            }

            $consumedBytes += $thisConsumedBytes;
            if ($bufferLength === $thisConsumedBytes) {
                $this->buffer = '';
            } else {
                $this->buffer = substr($this->buffer, $thisConsumedBytes);
            }

            if ('' !== $outputBuffer) {
                $bucket->data = $outputBuffer;
                stream_bucket_append($output, $bucket);
                $hasOutput = true;
            }

            $bucket = stream_bucket_make_writeable($input);
        }

        if ($hasOutput || $isEnd) {
            return PSFS_PASS_ON;
        }

        return PSFS_FEED_ME;
    }

    /**
     * Create the transform.
     *
     * @return DataTransformInterface The data transform.
     */
    abstract protected function createTransform();

    private $transform;
    private $buffer;
}

{'fraction_non_alphanumeric': 0.10787902138316317, 'fraction_numerical': 0.001541128876902331, 'mean_word_length': 1.9250704225352113, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3216086', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1338, 'n_words': 373}