<commit_before><?php
/*
 * This file is part of the Marmoset project.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace EAMann\Marmoset\Command;

use EAMann\Marmoset;
use EAMann\Marmoset\Console\Status;
use Symfony\Component\Console\Command\Command as SCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

/**
 * The actual wiring for the Symfony command.
 *
 * @package EAMann\Marmoset
 */
class Command extends SCommand
{
    /**
     * @var Status
     */
    protected $status;

    /**
     * @var array
     */
    protected $population;

    /**
     * Configure the CLI command we're going to run
     */
    protected function configure()
    {
        $this->setName('run')
             ->setDescription('Make the monkeys type')
             ->addOption(
                 'mode',
                 null,
                 InputOption::VALUE_OPTIONAL,
                 'Run-mode',
                 'synchronous'
             );
    }

    /**
     * Update the status display.
     *
     * @param int $generation
     * @param string $best
     */
    protected function updateStatus(int $generation, string $best)
    {
        $this->status->setGeneration($generation)->setBest($best);
    }
    
    /**
     * Create a new generation given the existing entities and using their relative fitnesses to
     * determine which monkeys are allowed to breed.
     *
     * @return array
     */
    protected function create_next_generation()
    {
        // Max fitness
        $maxFitness = array_reduce($this->population, function(float $max, string $current) {
            return max($max, Marmoset\fitness($current));
        }, 0) + 1.0;

        // Sum of max minus fitness
        $sumOfMaxMinusFitness = array_reduce($this->population, function(float $carry, string $current) use ($maxFitness) {
            return $carry + ($maxFitness - Marmoset\fitness($current));
        }, 0);

        if (getenv('ASYNC')) {
            $pool = new Marmoset\Troop(3, Marmoset\Worker::class, [$this->population, $sumOfMaxMinusFitness, $maxFitness]);

            foreach (range(1, count($this->population) / 2) as $counter) {
                $pool->submit(new Marmoset\Job());
            }

            $children = $pool->process();

            while( count($children) < count($this->population)) {
                $children[] = $this->population[rand(0, count($this->population) - 1)];
            }

            return (array) $children;
        } else {
            $newPop = [];

            foreach (range(1, count($this->population) / 2) as $counter) {
                $parent1 = Marmoset\random_high_quality_parent($this->population, $sumOfMaxMinusFitness, $maxFitness);
                $parent2 = Marmoset\random_high_quality_parent($this->population, $sumOfMaxMinusFitness, $maxFitness);
                $newPop = array_merge($newPop, Marmoset\create_children($parent1, $parent2));
            }

            return $newPop;
        }
    }

    /**
     * Get the best member of the generation
     *
     * @return string
     */
    protected function best()
    {
        return array_reduce($this->population, function (string $best, string $current) {
            if (Marmoset\fitness($current) < Marmoset\fitness($best)) {
                return $current;
            }

            return $best;
        }, "");
    }

    /**
     * Actually execute the command
     *
     * @param InputInterface $input
     * @param OutputInterface $output
     *
     * @return null|int null or 0 if everything went fine, or an error code
     */
    public function execute(InputInterface $input, OutputInterface $output)
    {
        $this->status = new Status($output);

        // Set initial state
        $generation = 0;
        $bestGenome = null;

        $this->population = Marmoset\random_population(30, strlen(Marmoset\TARGET));

        $running = true;
        do {
            // Move to the next generation
            $generation += 1;
            $this->population = $this->create_next_generation();

            // If we've found the best iteration so far, update the UI
            $bestSoFar = $this->best();
            $this->status->setGeneration($generation)->setBest($bestSoFar);
            if ( null === $bestGenome || Marmoset\fitness($bestSoFar) < Marmoset\fitness($bestGenome)) {
                $bestGenome = $bestSoFar;

                if (0 === Marmoset\fitness($bestGenome)) {
                    $running = false;
                    $this->status->display();
                }
            }

        } while ($running);
    }
}<commit_msg>Don't backfill with unfit parents<commit_after><?php
/*
 * This file is part of the Marmoset project.
 *
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace EAMann\Marmoset\Command;

use EAMann\Marmoset;
use EAMann\Marmoset\Console\Status;
use Symfony\Component\Console\Command\Command as SCommand;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

/**
 * The actual wiring for the Symfony command.
 *
 * @package EAMann\Marmoset
 */
class Command extends SCommand
{
    /**
     * @var Status
     */
    protected $status;

    /**
     * @var array
     */
    protected $population;

    /**
     * Configure the CLI command we're going to run
     */
    protected function configure()
    {
        $this->setName('run')
             ->setDescription('Make the monkeys type')
             ->addOption(
                 'mode',
                 null,
                 InputOption::VALUE_OPTIONAL,
                 'Run-mode',
                 'synchronous'
             );
    }

    /**
     * Update the status display.
     *
     * @param int $generation
     * @param string $best
     */
    protected function updateStatus(int $generation, string $best)
    {
        $this->status->setGeneration($generation)->setBest($best);
    }
    
    /**
     * Create a new generation given the existing entities and using their relative fitnesses to
     * determine which monkeys are allowed to breed.
     *
     * @return array
     */
    protected function create_next_generation()
    {
        // Max fitness
        $maxFitness = array_reduce($this->population, function(float $max, string $current) {
            return max($max, Marmoset\fitness($current));
        }, 0) + 1.0;

        // Sum of max minus fitness
        $sumOfMaxMinusFitness = array_reduce($this->population, function(float $carry, string $current) use ($maxFitness) {
            return $carry + ($maxFitness - Marmoset\fitness($current));
        }, 0);

        if (getenv('ASYNC')) {
            $pool = new Marmoset\Troop(2, Marmoset\Worker::class, [$this->population, $sumOfMaxMinusFitness, $maxFitness]);

            foreach (range(1, count($this->population) / 2) as $counter) {
                $pool->submit(new Marmoset\Job());
            }

            $children = $pool->process();

            // Because sometimes the thread pool exits early and evicts a child ... backfill
            while( count($children) < count($this->population)) {
                $children[] = Marmoset\random_high_quality_parent($this->population, $sumOfMaxMinusFitness, $maxFitness);
            }

            return (array) $children;
        } else {
            $newPop = [];

            foreach (range(1, count($this->population) / 2) as $counter) {
                $parent1 = Marmoset\random_high_quality_parent($this->population, $sumOfMaxMinusFitness, $maxFitness);
                $parent2 = Marmoset\random_high_quality_parent($this->population, $sumOfMaxMinusFitness, $maxFitness);
                $newPop = array_merge($newPop, Marmoset\create_children($parent1, $parent2));
            }

            return $newPop;
        }
    }

    /**
     * Get the best member of the generation
     *
     * @return string
     */
    protected function best()
    {
        return array_reduce($this->population, function (string $best, string $current) {
            if (Marmoset\fitness($current) < Marmoset\fitness($best)) {
                return $current;
            }

            return $best;
        }, "");
    }

    /**
     * Actually execute the command
     *
     * @param InputInterface $input
     * @param OutputInterface $output
     *
     * @return null|int null or 0 if everything went fine, or an error code
     */
    public function execute(InputInterface $input, OutputInterface $output)
    {
        $this->status = new Status($output);

        // Set initial state
        $generation = 0;
        $bestGenome = null;

        $this->population = Marmoset\random_population(30, strlen(Marmoset\TARGET));

        $running = true;
        do {
            // Move to the next generation
            $generation += 1;
            $this->population = $this->create_next_generation();

            // If we've found the best iteration so far, update the UI
            $bestSoFar = $this->best();
            $this->status->setGeneration($generation)->setBest($bestSoFar);
            if ( null === $bestGenome || Marmoset\fitness($bestSoFar) < Marmoset\fitness($bestGenome)) {
                $bestGenome = $bestSoFar;

                if (0 === Marmoset\fitness($bestGenome)) {
                    $running = false;
                    $this->status->display();
                }
            }

        } while ($running);
    }
}
{'fraction_non_alphanumeric': 0.12146575482290498, 'fraction_numerical': 0.00408288251505563, 'mean_word_length': 2.0561447286338117, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3808924', 'n_tokens_mistral': 2841, 'n_tokens_neox': 2720, 'n_words': 664}