<commit_before><?php

namespace Colligator;

use Colligator\Http\Requests\SearchDocumentsRequest;
use Elasticsearch\Common\Exceptions\BadRequest400Exception;
use Elasticsearch\Common\Exceptions\Missing404Exception;

class SearchEngine
{

    /**
     * Builds a query string query from a SearchDocumentsRequest.
     *
     * @param SearchDocumentsRequest $request
     *
     * @return string
     */
    public function queryFromRequest(SearchDocumentsRequest $request)
    {
        $query = [];
        if ($request->has('q')) {
            $query[] = $request->q;
        }
        if ($request->has('collection')) {
            $col = Collection::find($request->collection);
            $query[] = 'collections:' . $col->name;
        }
        if ($request->has('real')) {
            $query[] = 'subjects.noubomn.prefLabel:' . $request->real;
        }
        $query = count($query) ? implode(' AND ', $query) : '';

        return $query;
    }

    /**
     * Search for documents in ElasticSearch.
     *
     * @param SearchDocumentsRequest $request
     *
     * @return array
     */
    public function searchDocuments(SearchDocumentsRequest $request)
    {
        $query = $this->queryFromRequest($request);
        $payload = [
             'index' => 'documents',
             'type' => 'document',
             'body' => [],
             'from' => $request->offset ?: 0,
             'size' => $request->limit ?: 25,
         ];

        if (!empty($query)) {
            $payload['body']['query']['query_string']['query'] = $query;
        }

        $response = \Es::search($payload);
        $response['offset'] = $payload['from'];
        return $response;
    }

    /**
     * Return a single document identified by ID.
     *
     * @param int $id
     *
     * @return array
     */
    public function getDocument($id)
    {
        $payload = [
            'index' => 'documents',
            'type' => 'document',
            'id' => $id,
        ];
        try {
            $response = \Es::get($payload);
        } catch (Missing404Exception $e) {
            return;
        }

        return $response['_source'];
    }

    /**
     * Generate payload for indexing a document in ElasticSearch.
     *
     * @param Document $doc
     *
     * @return array
     */
    public function indexDocumentPayload(Document $doc)
    {
        $body = $doc->bibliographic;  // PHP makes a copy for us, right?

        $body['id'] = $doc->id;
        $body['bibsys_id'] = $doc->bibsys_id;

        // Remove some stuff we don't need
        foreach (['agency', 'catalogingRules', 'debug', 'modified', 'extent', 'cover_image'] as $key) {
            unset($body[$key]);
        }

        // Add local collections
        $body['collections'] = [];
        foreach ($doc->collections as $collection) {
            $body['collections'][] = $collection['name'];
        }

        // Add cover
        $body['cover'] = $doc->cover ? $doc->cover->toArray() : null;

        // Add subjects
        $body['subjects'] = [];
        foreach ($doc->subjects as $subject) {
            $body['subjects'][$subject['vocabulary'] ?: 'keywords'][] = [
                'id' => array_get($subject, 'id'),
                'prefLabel' => array_get($subject, 'term'),
                'type' => array_get($subject, 'type'),
            ];
        }

        // Add genres
        $body['genres'] = [];
        foreach ($doc->genres as $genre) {
            $body['genres'][$genre['vocabulary'] ?: 'keywords'][] = [
                'id' => array_get($genre, 'id'),
                'prefLabel' => array_get($genre, 'term'),
            ];
        }

        // Add holdings
        $this->addHoldings($body, $doc);

        // Add xisbns
        $body['xisbns'] = (new XisbnResponse($doc->xisbn))->getSimpleRepr();

        // Add description
        $body['description'] = $doc->description;

        // Add 'other form'
        $otherFormId = array_get($body, 'other_form.id');
        if (!empty($otherFormId)) {
            $otherFormDoc = Document::where('bibsys_id', '=', $otherFormId)->firstOrFail();
            $body['other_form'] = [
                'id' => $otherFormDoc->id,
                'bibsys_id' => $otherFormDoc->bibsys_id,
                'electronic' => $otherFormDoc->isElectronic(),
            ];
            $this->addHoldings($body['other_form'], $otherFormDoc);
        }

        return $body;
    }

    public function addHoldings(&$body, Document $doc)
    {
        if ($doc->isElectronic()) {
            $body['fulltext'] = $this->fulltextFromHoldings($doc->holdings);
        } else {
            $holdings = [];
            foreach ($doc->holdings as $holding) {
                if ($holding['location'] == 'UBO' && $holding['sublocation'] == 'UREAL') {
                    array_forget($holding, 'fulltext');
                    array_forget($holding, 'bibliographic_record');
                    array_forget($holding, 'nonpublic_notes');
                    $holdings[] = $holding;
                }
            }
            $body['holdings'] = $holdings;
        }
    }

    public function fulltextFromHoldings($holdings)
    {
        $fulltext = [
            'access' => false,
        ];
        foreach ($holdings as $holding) {
            if (!count($holding['fulltext'])) {
                continue;
            }
            if ($holding['location'] == 'UBO' || stripos($holding['fulltext'][0]['comment'], 'gratis') !== false) {
                $fulltext = $holding['fulltext'][0];
                $fulltext['access'] = true;
            }
        }
        return $fulltext;
    }

    /**
     * Add or update a document in the ElasticSearch index, making it searchable.
     *
     * @param Document $doc
     *
     * @throws \ErrorException
     */
    public function indexDocument(Document $doc)
    {
        // \Log::debug('Search engine: Indexing document ' . $doc->id);
        $payload = $this->indexDocumentPayload($doc);<commit_msg>[#28] Add sortable call code for 42 collection<commit_after><?php

namespace Colligator;

use Colligator\Http\Requests\SearchDocumentsRequest;
use Elasticsearch\Common\Exceptions\BadRequest400Exception;
use Elasticsearch\Common\Exceptions\Missing404Exception;

class SearchEngine
{

    /**
     * Builds a query string query from a SearchDocumentsRequest.
     *
     * @param SearchDocumentsRequest $request
     *
     * @return string
     */
    public function queryFromRequest(SearchDocumentsRequest $request)
    {
        $query = [];
        if ($request->has('q')) {
            $query[] = $request->q;
        }
        if ($request->has('collection')) {
            $col = Collection::find($request->collection);
            $query[] = 'collections:' . $col->name;
        }
        if ($request->has('real')) {
            $query[] = 'subjects.noubomn.prefLabel:' . $request->real;
        }
        $query = count($query) ? implode(' AND ', $query) : '';

        return $query;
    }

    /**
     * Search for documents in ElasticSearch.
     *
     * @param SearchDocumentsRequest $request
     *
     * @return array
     */
    public function searchDocuments(SearchDocumentsRequest $request)
    {
        $query = $this->queryFromRequest($request);
        $payload = [
             'index' => 'documents',
             'type' => 'document',
             'body' => [],
             'from' => $request->offset ?: 0,
             'size' => $request->limit ?: 25,
         ];

        if (!empty($query)) {
            $payload['body']['query']['query_string']['query'] = $query;
        }

        $response = \Es::search($payload);
        $response['offset'] = $payload['from'];
        return $response;
    }

    /**
     * Return a single document identified by ID.
     *
     * @param int $id
     *
     * @return array
     */
    public function getDocument($id)
    {
        $payload = [
            'index' => 'documents',
            'type' => 'document',
            'id' => $id,
        ];
        try {
            $response = \Es::get($payload);
        } catch (Missing404Exception $e) {
            return;
        }

        return $response['_source'];
    }

    /**
     * Generate payload for indexing a document in ElasticSearch.
     *
     * @param Document $doc
     *
     * @return array
     */
    public function indexDocumentPayload(Document $doc)
    {
        $body = $doc->bibliographic;  // PHP makes a copy for us, right?

        $body['id'] = $doc->id;
        $body['bibsys_id'] = $doc->bibsys_id;

        // Remove some stuff we don't need
        foreach (['agency', 'catalogingRules', 'debug', 'modified', 'extent', 'cover_image'] as $key) {
            unset($body[$key]);
        }

        // Add local collections
        $body['collections'] = [];
        foreach ($doc->collections as $collection) {
            $body['collections'][] = $collection['name'];
        }

        // Add cover
        $body['cover'] = $doc->cover ? $doc->cover->toArray() : null;

        // Add subjects
        $body['subjects'] = [];
        foreach ($doc->subjects as $subject) {
            $body['subjects'][$subject['vocabulary'] ?: 'keywords'][] = [
                'id' => array_get($subject, 'id'),
                'prefLabel' => array_get($subject, 'term'),
                'type' => array_get($subject, 'type'),
            ];
        }

        // Add genres
        $body['genres'] = [];
        foreach ($doc->genres as $genre) {
            $body['genres'][$genre['vocabulary'] ?: 'keywords'][] = [
                'id' => array_get($genre, 'id'),
                'prefLabel' => array_get($genre, 'term'),
            ];
        }

        // Add holdings
        $this->addHoldings($body, $doc);

        // Add xisbns
        $body['xisbns'] = (new XisbnResponse($doc->xisbn))->getSimpleRepr();

        // Add description
        $body['description'] = $doc->description;

        // Add 'other form'
        $otherFormId = array_get($body, 'other_form.id');
        if (!empty($otherFormId)) {
            $otherFormDoc = Document::where('bibsys_id', '=', $otherFormId)->firstOrFail();
            $body['other_form'] = [
                'id' => $otherFormDoc->id,
                'bibsys_id' => $otherFormDoc->bibsys_id,
                'electronic' => $otherFormDoc->isElectronic(),
            ];
            $this->addHoldings($body['other_form'], $otherFormDoc);
        }

        return $body;
    }

    public function sortableCallCode($holding)
    {
        if ($holding['shelvinglocation'] == 'UREAL Samling 42') {
            $m = preg_match('/FA ([0-9]+)(\/([A-Z]))?/', $holding['callcode'], $matches);
            if ($m) {
                return intval($matches[1]);
                // TODO: Også ta hensyn til undersortering i $matches[3], men
                // denne er en blanding av romertall og alfabetisk sortering
                // https://github.com/scriptotek/colligator-backend/issues/28
            }
        }
        return null;
    }

    public function addHoldings(&$body, Document $doc)
    {
        if ($doc->isElectronic()) {
            $body['fulltext'] = $this->fulltextFromHoldings($doc->holdings);
        } else {
            $body['holdings'] = [];
            foreach ($doc->holdings as $holding) {
                if ($holding['location'] == 'UBO' && $holding['sublocation'] == 'UREAL') {
                    array_forget($holding, 'fulltext');
                    array_forget($holding, 'bibliographic_record');
                    array_forget($holding, 'nonpublic_notes');
                    $s = $this->sortableCallCode($holding);
                    if (!is_null($s)) {
                        $holding['callcodeSortable'] = $s;
                    }
                    $body['holdings'][] = $holding;
                }
            }
        }
    }

    public function fulltextFromHoldings($holdings)
    {
        $fulltext = [
            'access' => false,
        ];
        foreach ($holdings as $holding) {
            if (!count($holding['fulltext'])) {
                continue;
            }
            if ($holding['location'] == 'UBO' || stripos($holding['fulltext'][0]['comment'], 'gratis') !== false) {
                $fulltext = $holding['fulltext'][0];
                $fulltext['access'] = true;
            }
        }
        return $fulltext;
    }

    /**
     * Add or update a document in the ElasticSearch index, making it searchable.
     *
     * @param Document $doc
     *
     * @throws \ErrorException
     */
    public function indexDocument(Document $doc)
    {
        // \Log::debug('Search engine: Indexing document ' . $doc->id);
        $payload = $this->indexDocumentPayload($doc);
{'fraction_non_alphanumeric': 0.16153303550434986, 'fraction_numerical': 0.0031350419311858297, 'mean_word_length': 1.823008849557522, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 106, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929832', 'n_tokens_mistral': 3772, 'n_tokens_neox': 3590, 'n_words': 716}