<commit_before><?php

namespace App\Console\Commands;

use App\Libraries\Elasticsearch\Es;
use App\Libraries\Elasticsearch\Indexing;
use App\Libraries\Elasticsearch\Search;
use App\Libraries\Elasticsearch\Sort;
use App\Libraries\OsuWiki;
use App\Libraries\Search\BasicSearch;
use App\Models\Wiki\Page;
use Illuminate\Console\Command;

class EsIndexWiki extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'es:index-wiki {--inplace} {--cleanup} {--yes}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Re-indexes wiki pages';

    private $cleanup;
    private $indexName;
    private $indicesToRemove;
    private $inplace;
    private $yes;

    public function handle()
    {
        $this->readOptions();

        $alias = Page::esIndexName();
        $oldIndices = Indexing::getOldIndices($alias);

        if (!$this->inplace || empty($oldIndices)) {
            $this->indexName = Page::esTimestampedIndexName();
        } else {
            $this->indexName = $oldIndices[0];
        }

        $this->indicesToRemove = array_filter($oldIndices, function ($index) {
            // because removing the index we just wrote to would be silly.
            return $this->indexName !== $index;
        });

        $continue = $this->starterMessage();
        if (!$continue) {
            return $this->error('User aborted!');
        }

        if (!Es::getClient()->indices()->exists(['index' => [$this->indexName]])) {
            $this->info("Creating '{$this->indexName}'...");
            Page::esCreateIndex($this->indexName);
        }

        $this->reindex();

        Indexing::updateAlias($alias, [$this->indexName]);

        $this->finish();
    }

    private function finish()
    {
        if (!$this->cleanup) {
            return;
        }

        foreach ($this->indicesToRemove as $index) {
            $this->warn("Removing '{$index}'...");
            Indexing::deleteIndex($index);
        }
    }

    private function newBaseSearch(): Search
    {
        return (new BasicSearch($this->indexName))
            ->query(['match_all' => new \stdClass])
            ->sort(new Sort('_id', 'asc'))
            ->source(false);
    }

    private function readOptions()
    {
        $this->inplace = $this->option('inplace');
        $this->cleanup = $this->option('cleanup');
        $this->yes = $this->option('yes');
    }

    private function reindex()
    {
        // for storing the paths as keys; the values don't matter in practise.
        $paths = [];

        $this->line('Fetching page list from Github...');
        OsuWiki::getPageList()->each(function ($path) use (&$paths) {
            $path = str_replace_first('wiki/', '', $path);
            $paths[$path] = false;
        });

        $this->line(count($paths).' pages found');

        if ($this->inplace) {
            $this->line('Fetching existing list...');
            $cursor = ['']; // works with Sort(_id, asc) to start at the beginning.
            while ($cursor !== null) {
                $search = $this->newBaseSearch()->searchAfter(array_values($cursor));
                $response = $search->response();

                foreach ($response as $hit) {
                    $paths[$hit['_id']] = true;
                }

                $cursor = $search->getSortCursor();
            }
        }

        $total = count($paths);
        $this->line("Total: {$total} documents");
        $bar = $this->output->createProgressBar($total);

        foreach ($paths as $path => $_inEs) {
            $pagePath = Page::parsePagePath($path);
            $page = new Page($pagePath['path'], $pagePath['locale']);
            $page->sync(true, $this->indexName);

            if (!$page->isVisible()) {
                $this->warn("delete {$pagePath['locale']}: {$pagePath['path']}");
                $page->esDeleteDocument(['index' => $this->indexName]);
            }

            $bar->advance();
        }

        $bar->finish();<commit_msg>new line after bar finishes<commit_after><?php

namespace App\Console\Commands;

use App\Libraries\Elasticsearch\Es;
use App\Libraries\Elasticsearch\Indexing;
use App\Libraries\Elasticsearch\Search;
use App\Libraries\Elasticsearch\Sort;
use App\Libraries\OsuWiki;
use App\Libraries\Search\BasicSearch;
use App\Models\Wiki\Page;
use Illuminate\Console\Command;

class EsIndexWiki extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'es:index-wiki {--inplace} {--cleanup} {--yes}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Re-indexes wiki pages';

    private $cleanup;
    private $indexName;
    private $indicesToRemove;
    private $inplace;
    private $yes;

    public function handle()
    {
        $this->readOptions();

        $alias = Page::esIndexName();
        $oldIndices = Indexing::getOldIndices($alias);

        if (!$this->inplace || empty($oldIndices)) {
            $this->indexName = Page::esTimestampedIndexName();
        } else {
            $this->indexName = $oldIndices[0];
        }

        $this->indicesToRemove = array_filter($oldIndices, function ($index) {
            // because removing the index we just wrote to would be silly.
            return $this->indexName !== $index;
        });

        $continue = $this->starterMessage();
        if (!$continue) {
            return $this->error('User aborted!');
        }

        if (!Es::getClient()->indices()->exists(['index' => [$this->indexName]])) {
            $this->info("Creating '{$this->indexName}'...");
            Page::esCreateIndex($this->indexName);
        }

        $this->reindex();

        Indexing::updateAlias($alias, [$this->indexName]);

        $this->finish();
    }

    private function finish()
    {
        if (!$this->cleanup) {
            return;
        }

        foreach ($this->indicesToRemove as $index) {
            $this->warn("Removing '{$index}'...");
            Indexing::deleteIndex($index);
        }
    }

    private function newBaseSearch(): Search
    {
        return (new BasicSearch($this->indexName))
            ->query(['match_all' => new \stdClass])
            ->sort(new Sort('_id', 'asc'))
            ->source(false);
    }

    private function readOptions()
    {
        $this->inplace = $this->option('inplace');
        $this->cleanup = $this->option('cleanup');
        $this->yes = $this->option('yes');
    }

    private function reindex()
    {
        // for storing the paths as keys; the values don't matter in practise.
        $paths = [];

        $this->line('Fetching page list from Github...');
        OsuWiki::getPageList()->each(function ($path) use (&$paths) {
            $path = str_replace_first('wiki/', '', $path);
            $paths[$path] = false;
        });

        $this->line(count($paths).' pages found');

        if ($this->inplace) {
            $this->line('Fetching existing list...');
            $cursor = ['']; // works with Sort(_id, asc) to start at the beginning.
            while ($cursor !== null) {
                $search = $this->newBaseSearch()->searchAfter(array_values($cursor));
                $response = $search->response();

                foreach ($response as $hit) {
                    $paths[$hit['_id']] = true;
                }

                $cursor = $search->getSortCursor();
            }
        }

        $total = count($paths);
        $this->line("Total: {$total} documents");
        $bar = $this->output->createProgressBar($total);

        foreach ($paths as $path => $_inEs) {
            $pagePath = Page::parsePagePath($path);
            $page = new Page($pagePath['path'], $pagePath['locale']);
            $page->sync(true, $this->indexName);

            if (!$page->isVisible()) {
                $this->warn("delete {$pagePath['locale']}: {$pagePath['path']}");
                $page->esDeleteDocument(['index' => $this->indexName]);
            }

            $bar->advance();
        }

        $bar->finish();
        $this->line('');
{'fraction_non_alphanumeric': 0.16809979411408502, 'fraction_numerical': 0.00024221872350732712, 'mean_word_length': 2.0894126449682004, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 112, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4994179', 'n_tokens_mistral': 2360, 'n_tokens_neox': 2316, 'n_words': 482}