<commit_before>package se.lagrummet.rinfo.service

import groovy.transform.CompileStatic
import groovy.util.logging.Commons as Log

import static org.restlet.data.CharacterSet.UTF_8
import org.restlet.data.Reference

import org.elasticsearch.action.search.SearchPhaseExecutionException
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchRequestBuilder
import org.elasticsearch.index.query.FilterBuilder
import org.elasticsearch.index.query.FilterBuilders
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.query.QueryStringQueryBuilder
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.facet.FacetBuilders
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet
import org.elasticsearch.search.sort.SortOrder


@Log
class ElasticQuery {

    ElasticData elasticData
    JsonLdSettings jsonLdSettings
    String serviceAppBaseUrl

    def defaultPageSize = 50
    def sortParamKey = '_sort'
    def pageParamKey = '_page'
    def pageSizeParamKey = '_pageSize'
    def statsParamKey = '_stats'

    def segmentWildcard = "*"
    def facetStatsSubSegment = "stats"

    ElasticQuery(ElasticData elasticData, String serviceAppBaseUrl) {
        this.elasticData = elasticData
        this.jsonLdSettings = elasticData.jsonLdSettings
        this.serviceAppBaseUrl = serviceAppBaseUrl
    }

    @CompileStatic
    Map search(String docType, Reference ref) {
        boolean onlyStats = false
        int splitAt = docType.indexOf(';')
        if (splitAt > -1) {
            onlyStats = docType.substring(splitAt+1) == facetStatsSubSegment
            docType = docType.substring(0, splitAt)
        }
        if (docType == segmentWildcard) {
            docType = null
        }
        def srb = newSearchRequestBuilder()
        return onlyStats?
            getElasticStats(srb, docType, ref) :
            searchElastic(srb, docType, ref)
    }

    SearchRequestBuilder newSearchRequestBuilder() {
        return elasticData.client.prepareSearch(elasticData.indexName)
    }

    //@CompileStatic
    Map getElasticStats(SearchRequestBuilder srb, String docType, Reference ref) {
        //def qb = QueryBuilders.matchAllQuery()
        //srb.setQuery(qb)
        prepareElasticSearch(srb, ref, docType, Collections.emptyList(), false, true)
        SearchResponse esRes = srb.execute().actionGet()
        return buildStats(esRes)
    }

    def isLeaf(value) {
        if(value == null
            || value instanceof String
            || value instanceof Integer) return true
        return false
    }
    //flatten the map containing all interesting properties for the query
    //needed because fields don't accept complextypes
    def compress( Map m, String prefix = '' ) {
        prefix = prefix ? "$prefix." : ''
        m.collectEntries { k, v ->
            if(!isLeaf(v)) compress( v, "$prefix$k" )
            else
                if(k == "_boost") [ (prefix[0..-2].toString()): v ]
                else [ ("$prefix$k".toString()): v ]
        }
    }

    //@CompileStatic
    Map searchElastic(SearchRequestBuilder srb, String docType, Reference ref) {
        def showTerms = compress(jsonLdSettings.listFramesData[docType])?.keySet()

        if (!showTerms) {
            return null
        }

        Map prepSearch = null
        SearchResponse esRes = null
        try {
            prepSearch = prepareElasticSearch(srb, ref, docType, showTerms)
            esRes = srb.execute().actionGet()
        } catch (Exception e) {
            if (e.cause instanceof SearchPhaseExecutionException) {
                log.debug "Malformed query <${ref}>. Retrying with escaped query text..."
                srb = newSearchRequestBuilder()
                prepSearch = prepareElasticSearch(srb, ref, docType, showTerms, true)
                esRes = srb.execute().actionGet()
            } else {
                throw e
            }
        }

        assert esRes.failedShards == 0

        def data = [
            "@language": "sv",
            "@context": jsonLdSettings.ldContextPath,
            startIndex: prepSearch.startIndex,
            itemsPerPage: prepSearch.pageSize,
            totalResults: esRes.hits.totalHits(),<commit_msg>Create a LUT at instantiation #87<commit_after>package se.lagrummet.rinfo.service

import groovy.transform.CompileStatic
import groovy.util.logging.Commons as Log

import static org.restlet.data.CharacterSet.UTF_8
import org.restlet.data.Reference

import org.elasticsearch.action.search.SearchPhaseExecutionException
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchRequestBuilder
import org.elasticsearch.index.query.FilterBuilder
import org.elasticsearch.index.query.FilterBuilders
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.query.QueryStringQueryBuilder
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.facet.FacetBuilders
import org.elasticsearch.search.facet.datehistogram.DateHistogramFacet
import org.elasticsearch.search.sort.SortOrder


@Log
class ElasticQuery {

    ElasticData elasticData
    JsonLdSettings jsonLdSettings
    String serviceAppBaseUrl

    def defaultPageSize = 50
    def sortParamKey = '_sort'
    def pageParamKey = '_page'
    def pageSizeParamKey = '_pageSize'
    def statsParamKey = '_stats'

    def segmentWildcard = "*"
    def facetStatsSubSegment = "stats"

    Map elasticfields = [:]

    ElasticQuery(ElasticData elasticData, String serviceAppBaseUrl) {
        this.elasticData = elasticData
        this.jsonLdSettings = elasticData.jsonLdSettings
        this.serviceAppBaseUrl = serviceAppBaseUrl
        this.jsonLdSettings.listFramesData.each {
            this.elasticfields += [(it.key): compress(it.value)?.keySet()]
        }
    }

    @CompileStatic
    Map search(String docType, Reference ref) {
        boolean onlyStats = false
        int splitAt = docType.indexOf(';')
        if (splitAt > -1) {
            onlyStats = docType.substring(splitAt+1) == facetStatsSubSegment
            docType = docType.substring(0, splitAt)
        }
        if (docType == segmentWildcard) {
            docType = null
        }
        def srb = newSearchRequestBuilder()
        return onlyStats?
            getElasticStats(srb, docType, ref) :
            searchElastic(srb, docType, ref)
    }

    SearchRequestBuilder newSearchRequestBuilder() {
        return elasticData.client.prepareSearch(elasticData.indexName)
    }

    //@CompileStatic
    Map getElasticStats(SearchRequestBuilder srb, String docType, Reference ref) {
        //def qb = QueryBuilders.matchAllQuery()
        //srb.setQuery(qb)
        prepareElasticSearch(srb, ref, docType, Collections.emptyList(), false, true)
        SearchResponse esRes = srb.execute().actionGet()
        return buildStats(esRes)
    }

    def isLeaf(value) {
        if(value == null
            || value instanceof String
            || value instanceof Integer) return true
        return false
    }
    //flatten the map containing all interesting properties for the query
    //needed because fields don't accept complextypes
    def compress( Map m, String prefix = '' ) {
        prefix = prefix ? "$prefix." : ''
        m.collectEntries { k, v ->
            if(!isLeaf(v)) compress( v, "$prefix$k" )
            else
                if(k == "_boost") [ (prefix[0..-2].toString()): v ]
                else [ ("$prefix$k".toString()): v ]
        }
    }

    //@CompileStatic
    Map searchElastic(SearchRequestBuilder srb, String docType, Reference ref) {
        def showTerms = elasticfields[docType]
        if (!showTerms) {
            return null
        }

        Map prepSearch = null
        SearchResponse esRes = null
        try {
            prepSearch = prepareElasticSearch(srb, ref, docType, showTerms)
            esRes = srb.execute().actionGet()
        } catch (Exception e) {
            if (e.cause instanceof SearchPhaseExecutionException) {
                log.debug "Malformed query <${ref}>. Retrying with escaped query text..."
                srb = newSearchRequestBuilder()
                prepSearch = prepareElasticSearch(srb, ref, docType, showTerms, true)
                esRes = srb.execute().actionGet()
            } else {
                throw e
            }
        }

        assert esRes.failedShards == 0

        def data = [
            "@language": "sv",
            "@context": jsonLdSettings.ldContextPath,
            startIndex: prepSearch.startIndex,
            itemsPerPage: prepSearch.pageSize,
            totalResults: esRes.hits.totalHits(),
{'fraction_non_alphanumeric': 0.09130089599637065, 'fraction_numerical': 0.0022683452421458548, 'mean_word_length': 2.5874694873881205, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5048345', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2397, 'n_words': 544}