<commit_before><?php
/*
 * Copyright (c) 2015 <NAME> <<EMAIL>>,
 * All rights reserved.
 *
 * This file is part of Quartet/Haydn.
 *
 * This program and the accompanying materials are made available under
 * the terms of the BSD 2-Clause License which accompanies this
 * distribution, and is available at http://opensource.org/licenses/BSD-2-Clause
 */

namespace Quartet\Haydn;

use Quartet\Haydn\IO\SourceInterface;
use Quartet\Haydn\Matcher\MatcherInterface;
use Quartet\Haydn\Set\EmptySet;
use Quartet\Haydn\Set\FilterSet;
use Quartet\Haydn\Set\IdenticalSet;
use Quartet\Haydn\Set\ProductSet;
use Quartet\Haydn\Set\SelectSet;
use Quartet\Haydn\Set\UnionSet;

class Set implements SetInterface
{
    /**
     * @var SourceInterface
     */
    private $source;

    /**
     * @var \Traversable
     */
    protected $it;

    /**
     * @param SourceInterface $source
     */
    public function __construct(SourceInterface $source = null)
    {
        $this->source = $source;
        $this->rewind();
    }

    /**
     * {@inheritdoc}
     */
    public function getIterator()
    {
        return $this->it;
    }

    /**
     * {@inheritdoc}
     */
    public function rewind()
    {
        $this->source->rewind();
        $this->it = $this->source->getIterator();
    }

    /**
     * {@inheritdoc}
     */
    public function product(SetInterface $that)
    {
        if ($this->isIdentical() || $that->isIdentical()) {
            return $this->firstOfNotIdentical([$this, $that]);
        }
        if ($this->isEmpty() || $that->isEmpty()) {
            return new EmptySet();
        }

        $product = new ProductSet($this, $that);
        $product->setPrefixing(true);

        return $product;
    }

    /**
     * {@inheritdoc}
     */
    public function union(SetInterface $that)
    {
        if ($this->isIdentical() || $that->isIdentical()) {
            return $this->firstOfNotIdentical([$this, $that]);
        }
        if ($this->isEmpty() || $that->isEmpty()) {
            return $this->firstOfNotEmpty([$this, $that]);
        }

        return new UnionSet($this, $that);
    }

    /**
     * {@inheritdoc}
     */
    public function select($selects)
    {
        if ($this->isIdentical()) {
            return $this;
        }
        if ($this->isEmpty()) {
            return $this;
        }
        return new SelectSet($this, $selects);
    }

    /**
     * {@inheritdoc}
     */
    public function filter(MatcherInterface $matcher)
    {
        if ($this->isIdentical()) {
            return $this;
        }
        if ($this->isEmpty()) {
            return $this;
        }
        return new FilterSet($this, $matcher);
    }

    /**
     * {@inheritdoc}
     */
    public function devide($matchers)
    {
        if ($this->isIdentical()) {
            return [$this];
        }
        if ($this->isEmpty()) {
            return [$this];
        }
        $sets = [];
        foreach ($matchers as $key => $matcher)
        {
            $sets[] = $this->filter($matcher);
        }

        return $sets;
    }

    /**
     * {@inheritdoc}
     */
    public function toArray()
    {
        $buf = [];
        $this->rewind();
        foreach ($this as $line) {
            $buf[] = $line;
        }

        return $buf;
    }

    /**
     * {@inheritdoc}
     */
    public function setPrefixing($prefixing)
    {
        $this->source->setPrefixing($prefixing);
    }<commit_msg>remove unused key on foreach<commit_after><?php
/*
 * Copyright (c) 2015 GOTO Hidenori <<EMAIL>>,
 * All rights reserved.
 *
 * This file is part of Quartet/Haydn.
 *
 * This program and the accompanying materials are made available under
 * the terms of the BSD 2-Clause License which accompanies this
 * distribution, and is available at http://opensource.org/licenses/BSD-2-Clause
 */

namespace Quartet\Haydn;

use Quartet\Haydn\IO\SourceInterface;
use Quartet\Haydn\Matcher\MatcherInterface;
use Quartet\Haydn\Set\EmptySet;
use Quartet\Haydn\Set\FilterSet;
use Quartet\Haydn\Set\IdenticalSet;
use Quartet\Haydn\Set\ProductSet;
use Quartet\Haydn\Set\SelectSet;
use Quartet\Haydn\Set\UnionSet;

class Set implements SetInterface
{
    /**
     * @var SourceInterface
     */
    private $source;

    /**
     * @var \Traversable
     */
    protected $it;

    /**
     * @param SourceInterface $source
     */
    public function __construct(SourceInterface $source = null)
    {
        $this->source = $source;
        $this->rewind();
    }

    /**
     * {@inheritdoc}
     */
    public function getIterator()
    {
        return $this->it;
    }

    /**
     * {@inheritdoc}
     */
    public function rewind()
    {
        $this->source->rewind();
        $this->it = $this->source->getIterator();
    }

    /**
     * {@inheritdoc}
     */
    public function product(SetInterface $that)
    {
        if ($this->isIdentical() || $that->isIdentical()) {
            return $this->firstOfNotIdentical([$this, $that]);
        }
        if ($this->isEmpty() || $that->isEmpty()) {
            return new EmptySet();
        }

        $product = new ProductSet($this, $that);
        $product->setPrefixing(true);

        return $product;
    }

    /**
     * {@inheritdoc}
     */
    public function union(SetInterface $that)
    {
        if ($this->isIdentical() || $that->isIdentical()) {
            return $this->firstOfNotIdentical([$this, $that]);
        }
        if ($this->isEmpty() || $that->isEmpty()) {
            return $this->firstOfNotEmpty([$this, $that]);
        }

        return new UnionSet($this, $that);
    }

    /**
     * {@inheritdoc}
     */
    public function select($selects)
    {
        if ($this->isIdentical()) {
            return $this;
        }
        if ($this->isEmpty()) {
            return $this;
        }
        return new SelectSet($this, $selects);
    }

    /**
     * {@inheritdoc}
     */
    public function filter(MatcherInterface $matcher)
    {
        if ($this->isIdentical()) {
            return $this;
        }
        if ($this->isEmpty()) {
            return $this;
        }
        return new FilterSet($this, $matcher);
    }

    /**
     * {@inheritdoc}
     */
    public function devide($matchers)
    {
        if ($this->isIdentical()) {
            return [$this];
        }
        if ($this->isEmpty()) {
            return [$this];
        }
        $sets = [];
        foreach ($matchers as $matcher)
        {
            $sets[] = $this->filter($matcher);
        }

        return $sets;
    }

    /**
     * {@inheritdoc}
     */
    public function toArray()
    {
        $buf = [];
        $this->rewind();
        foreach ($this as $line) {
            $buf[] = $line;
        }

        return $buf;
    }

    /**
     * {@inheritdoc}
     */
    public function setPrefixing($prefixing)
    {
        $this->source->setPrefixing($prefixing);
    }
{'fraction_non_alphanumeric': 0.1527898236484533, 'fraction_numerical': 0.0017346053772766695, 'mean_word_length': 1.840311986863711, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3974396', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2068, 'n_words': 432}