<commit_before><?php

/**
 *    Copyright (c) ppy Pty Ltd <<EMAIL>>.
 *
 *    This file is part of osu!web. osu!web is distributed with the hope of
 *    attracting more community contributions to the core ecosystem of osu!.
 *
 *    osu!web is free software: you can redistribute it and/or modify
 *    it under the terms of the Affero GNU General Public License version 3
 *    as published by the Free Software Foundation.
 *
 *    osu!web is distributed WITHOUT ANY WARRANTY; without even the implied
 *    warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *    See the GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with osu!web.  If not, see <http://www.gnu.org/licenses/>.
 */

namespace App\Models\Wiki;

use App;
use App\Exceptions\GitHubNotFoundException;
use App\Jobs\EsDeleteDocument;
use App\Jobs\EsIndexDocument;
use App\Libraries\Elasticsearch\BoolQuery;
use App\Libraries\Elasticsearch\Es;
use App\Libraries\Markdown\OsuMarkdown;
use App\Libraries\OsuWiki;
use App\Libraries\Search\BasicSearch;
use Carbon\Carbon;
use Exception;

class Page
{
    // in minutes
    const REINDEX_AFTER = 300;
    const VERSION = 1;

    public $locale;
    public $requestedLocale;

    private $cache = [];
    private $defaultTitle;
    private $defaultSubtitle;

    private $source; // source document from elasticsearch;

    public static function cleanupPath($path)
    {
        return strtolower(str_replace(['-', '/', '_'], ' ', $path));
    }

    public static function searchIndexConfig($params = [])
    {
        return array_merge([
            'index' => config('osu.elasticsearch.index.wiki_pages'),
            'type' => 'wiki_page',
        ], $params);
    }

    public static function searchPath($path, $locale)
    {
        $searchPath = static::cleanupPath($path);

        $localeQuery = [
            ['constant_score' => [
                'boost' => 1000,
                'filter' => [
                    'match' => [
                        'locale' => $locale ?? App::getLocale(),
                    ],
                ],
            ]],
            ['constant_score' => [
                'filter' => [
                    'match' => [
                        'locale' => config('app.fallback_locale'),
                    ],
                ],
            ]],
        ];

        $query = (new BoolQuery())
            ->must(['match' => ['path_clean' => es_query_and_words($searchPath)]])
            ->should($localeQuery)
            ->shouldMatch(1);

        $search = (new BasicSearch(config('osu.elasticsearch.index.wiki_pages'), 'wiki_searchpath'))
            ->source('path')
            ->query($query);

        $response = $search->response();

        if ($response->total() === 0) {
            return;
        }

        foreach ($response as $hit) {
            $resultPath = static::cleanupPath($hit->source('path'));

            if ($resultPath === $searchPath) {
                return $hit->source('path');
            }
        }
    }

    public function __construct($path, $locale, $esCache = null)
    {
        $this->source = $esCache;
        if ($esCache !== null) {
            $path = $esCache['path'];
            $locale = $esCache['locale'];
            $this->cache['page'] = json_decode($esCache['page'], true);
        }

        $this->path = OsuWiki::cleanPath($path);
        $this->requestedLocale = $locale;
        $this->locale = $locale;

        $defaultTitles = explode('/', str_replace('_', ' ', $this->path));
        $this->defaultTitle = array_pop($defaultTitles);
        $this->defaultSubtitle = array_pop($defaultTitles);
    }

    public function editUrl()
    {
        return 'https://github.com/'.OsuWiki::USER.'/'.OsuWiki::REPOSITORY.'/tree/master/wiki/'.$this->pagePath();
    }

    public function esIndexDocument()
    {
        $params = static::searchIndexConfig();

        if ($this->page() === null) {
            $params['body'] = [
                'locale' => null,
                'page' => null,
                'page_text' => null,
                'path' => null,
                'path_clean' => null,
                'title' => null,
                'tags' => [],
            ];
        } else {
            $content = $this->getContent();
            $indexContent = (new OsuMarkdown('wiki', [
                'relative_url_root' => wiki_url($this->path),
            ]))->load($content)->toIndexable();

            $params['body'] = [
                'locale' => $this->locale,
                'page' => json_encode($this->page()),
                'page_text' => $indexContent,
                'path' => $this->path,
                'path_clean' => static::cleanupPath($this->path),
                'title' => $this->title(),
                'tags' => $this->tags(),
            ];
        }

        $params['id'] = $this->pagePath();
        $params['body']['indexed_at'] = json_time(Carbon::now());
        $params['body']['version'] = static::VERSION;

        return Es::getClient()->index($params);
    }

    public function esDeleteDocument()
    {
        return Es::getClient()->delete(static::searchIndexConfig([
            'id' => $this->pagePath(),
            'client' => ['ignore' => 404],
        ]));
    }

    /**
     * Gets the markdown content for the page from Github.
     *
     * @param bool $force Force any cached value to refresh.
     * @return string|null
     */
    public function getContent(bool $force = false)
    {
        if (!array_key_exists('content', $this->cache) || $force) {
            try {
                $this->cache['content'] = OsuWiki::fetchContent('wiki/'.$this->pagePath());
            } catch (GitHubNotFoundException $e) {
                $this->cache['content'] = null;
            }
        }

        return $this->cache['content'];
    }

    public function getSource()
    {
        return $this->source;
    }

    public function isOutdated()
    {
        return $this->page()['header']['outdated'] ?? false;
    }

    public function isLegalTranslation()
    {
        return $this->locale !== config('app.fallback_locale')
            && ($this->page()['header']['legal'] ?? false);
    }

    public function page()
    {
        if (!array_key_exists('page', $this->cache)) {
            foreach (array_unique([$this->requestedLocale, config('app.fallback_locale')]) as $locale) {
                $this->locale = $locale;

                $response = (new BasicSearch(config('osu.elasticsearch.index.wiki_pages'), 'wiki_page_lookup'))
                    ->source(['page', 'indexed_at', 'version'])<commit_msg>log to sentry when indexing wiki pages<commit_after><?php

/**
 *    Copyright (c) ppy Pty Ltd <<EMAIL>>.
 *
 *    This file is part of osu!web. osu!web is distributed with the hope of
 *    attracting more community contributions to the core ecosystem of osu!.
 *
 *    osu!web is free software: you can redistribute it and/or modify
 *    it under the terms of the Affero GNU General Public License version 3
 *    as published by the Free Software Foundation.
 *
 *    osu!web is distributed WITHOUT ANY WARRANTY; without even the implied
 *    warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *    See the GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with osu!web.  If not, see <http://www.gnu.org/licenses/>.
 */

namespace App\Models\Wiki;

use App;
use App\Exceptions\GitHubNotFoundException;
use App\Jobs\EsDeleteDocument;
use App\Jobs\EsIndexDocument;
use App\Libraries\Elasticsearch\BoolQuery;
use App\Libraries\Elasticsearch\Es;
use App\Libraries\Markdown\OsuMarkdown;
use App\Libraries\OsuWiki;
use App\Libraries\Search\BasicSearch;
use Carbon\Carbon;
use Exception;
use Sentry;

class Page
{
    // in minutes
    const REINDEX_AFTER = 300;
    const VERSION = 1;

    public $locale;
    public $requestedLocale;

    private $cache = [];
    private $defaultTitle;
    private $defaultSubtitle;

    private $source; // source document from elasticsearch;

    public static function cleanupPath($path)
    {
        return strtolower(str_replace(['-', '/', '_'], ' ', $path));
    }

    public static function searchIndexConfig($params = [])
    {
        return array_merge([
            'index' => config('osu.elasticsearch.index.wiki_pages'),
            'type' => 'wiki_page',
        ], $params);
    }

    public static function searchPath($path, $locale)
    {
        $searchPath = static::cleanupPath($path);

        $localeQuery = [
            ['constant_score' => [
                'boost' => 1000,
                'filter' => [
                    'match' => [
                        'locale' => $locale ?? App::getLocale(),
                    ],
                ],
            ]],
            ['constant_score' => [
                'filter' => [
                    'match' => [
                        'locale' => config('app.fallback_locale'),
                    ],
                ],
            ]],
        ];

        $query = (new BoolQuery())
            ->must(['match' => ['path_clean' => es_query_and_words($searchPath)]])
            ->should($localeQuery)
            ->shouldMatch(1);

        $search = (new BasicSearch(config('osu.elasticsearch.index.wiki_pages'), 'wiki_searchpath'))
            ->source('path')
            ->query($query);

        $response = $search->response();

        if ($response->total() === 0) {
            return;
        }

        foreach ($response as $hit) {
            $resultPath = static::cleanupPath($hit->source('path'));

            if ($resultPath === $searchPath) {
                return $hit->source('path');
            }
        }
    }

    public function __construct($path, $locale, $esCache = null)
    {
        $this->source = $esCache;
        if ($esCache !== null) {
            $path = $esCache['path'];
            $locale = $esCache['locale'];
            $this->cache['page'] = json_decode($esCache['page'], true);
        }

        $this->path = OsuWiki::cleanPath($path);
        $this->requestedLocale = $locale;
        $this->locale = $locale;

        $defaultTitles = explode('/', str_replace('_', ' ', $this->path));
        $this->defaultTitle = array_pop($defaultTitles);
        $this->defaultSubtitle = array_pop($defaultTitles);
    }

    public function editUrl()
    {
        return 'https://github.com/'.OsuWiki::USER.'/'.OsuWiki::REPOSITORY.'/tree/master/wiki/'.$this->pagePath();
    }

    public function esIndexDocument()
    {
        $params = static::searchIndexConfig();

        if ($this->page() === null) {
            Sentry::captureMessage(
                'wiki index document empty (%s)',
                [$this->pagePath()]
            );

            $params['body'] = [
                'locale' => null,
                'page' => null,
                'page_text' => null,
                'path' => null,
                'path_clean' => null,
                'title' => null,
                'tags' => [],
            ];
        } else {
            Sentry::captureMessage(
                'wiki index document (%s)',
                [$this->pagePath()]
            );

            $content = $this->getContent();
            $indexContent = (new OsuMarkdown('wiki', [
                'relative_url_root' => wiki_url($this->path),
            ]))->load($content)->toIndexable();

            $params['body'] = [
                'locale' => $this->locale,
                'page' => json_encode($this->page()),
                'page_text' => $indexContent,
                'path' => $this->path,
                'path_clean' => static::cleanupPath($this->path),
                'title' => $this->title(),
                'tags' => $this->tags(),
            ];
        }

        $params['id'] = $this->pagePath();
        $params['body']['indexed_at'] = json_time(Carbon::now());
        $params['body']['version'] = static::VERSION;

        return Es::getClient()->index($params);
    }

    public function esDeleteDocument()
    {
        Sentry::captureMessage(
            'wiki delete document (%s)',
            [$this->pagePath()]
        );

        return Es::getClient()->delete(static::searchIndexConfig([
            'id' => $this->pagePath(),
            'client' => ['ignore' => 404],
        ]));
    }

    /**
     * Gets the markdown content for the page from Github.
     *
     * @param bool $force Force any cached value to refresh.
     * @return string|null
     */
    public function getContent(bool $force = false)
    {
        if (!array_key_exists('content', $this->cache) || $force) {
            try {
                Sentry::captureMessage(
                    'wiki fetch (%s)',
                    [$this->pagePath()]
                );

                $this->cache['content'] = OsuWiki::fetchContent('wiki/'.$this->pagePath());
            } catch (GitHubNotFoundException $e) {
                Sentry::captureMessage(
                    'wiki fetch not found (%s)',
                    [$this->pagePath()]
                );

                $this->cache['content'] = null;
            }
        }

        return $this->cache['content'];
    }

    public function getSource()
    {
        return $this->source;
    }

    public function isOutdated()
    {
        return $this->page()['header']['outdated'] ?? false;
    }

    public function isLegalTranslation()
    {
        return $this->locale !== config('app.fallback_locale')
            && ($this->page()['header']['legal'] ?? false);
    }

    public function page()
    {
        if (!array_key_exists('page', $this->cache)) {
            foreach (array_unique([$this->requestedLocale, config('app.fallback_locale')]) as $locale) {
                $this->locale = $locale;

                $response = (new BasicSearch(config('osu.elasticsearch.index.wiki_pages'), 'wiki_page_lookup'))
                    ->source(['page', 'indexed_at', 'version'])
{'fraction_non_alphanumeric': 0.1527306168647425, 'fraction_numerical': 0.0019807583474816073, 'mean_word_length': 1.9178534571723427, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 168, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1516990', 'n_tokens_mistral': 4042, 'n_tokens_neox': 3872, 'n_words': 854}