<commit_before>package whelk.rest.api

import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Log4j2 as Log
import io.prometheus.client.Counter
import io.prometheus.client.Gauge
import io.prometheus.client.Histogram
import io.prometheus.client.Summary
import org.apache.http.entity.ContentType
import whelk.Document
import whelk.IdGenerator
import whelk.IdType
import whelk.JsonLd
import whelk.JsonLdValidator
import whelk.TargetVocabMapper
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.exception.ElasticIOException
import whelk.exception.InvalidQueryException
import whelk.exception.LinkValidationException
import whelk.exception.ModelValidationException
import whelk.exception.StaleUpdateException
import whelk.exception.StorageCreateFailedException
import whelk.exception.UnexpectedHttpStatusException
import whelk.exception.WhelkRuntimeException
import whelk.history.History
import whelk.rest.api.CrudGetRequest.Lens
import whelk.rest.security.AccessControl
import whelk.util.WhelkFactory

import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.lang.management.ManagementFactory

import static whelk.rest.api.CrudUtils.ETag
import static whelk.rest.api.HttpTools.getBaseUri
import static whelk.rest.api.HttpTools.sendError
import static whelk.rest.api.HttpTools.sendResponse
import static whelk.util.Jackson.mapper

/**
 * Handles all GET/PUT/POST/DELETE requests against the backend.
 */
@Log
@CompileStatic
class Crud extends HttpServlet {
    final static String XL_ACTIVE_SIGEL_HEADER = 'XL-Active-Sigel'
    final static String CONTEXT_PATH = '/context.jsonld'
    final static String DATA_CONTENT_TYPE = "application/ld+json"

    static final Counter requests = Counter.build()
        .name("api_requests_total").help("Total requests to API.")
        .labelNames("method").register()

    static final Counter failedRequests = Counter.build()
        .name("api_failed_requests_total").help("Total failed requests to API.")
        .labelNames("method", "status").register()

    static final Gauge ongoingRequests = Gauge.build()
        .name("api_ongoing_requests_total").help("Total ongoing API requests.")
        .labelNames("method").register()

    static final Summary requestsLatency = Summary.build()
        .name("api_requests_latency_seconds")
        .help("API request latency in seconds.")
        .labelNames("method")
        .quantile(0.5f, 0.05f)
        .quantile(0.95f, 0.01f)
        .quantile(0.99f, 0.001f)
        .register()

    static final Histogram requestsLatencyHistogram = Histogram.build()
            .name("api_requests_latency_seconds_histogram").help("API request latency in seconds.")
            .labelNames("method")
            .register()

    Whelk whelk

    Map vocabData
    Map displayData
    JsonLd jsonld
    JsonLdValidator validator
    TargetVocabMapper targetVocabMapper

    SearchUtils search
    AccessControl accessControl = new AccessControl()
    ConverterUtils converterUtils

    Map siteConfig
    Map sitesData
    Map siteAlias

    Map<String, Tuple2<Document, String>> cachedDocs

    Crud() {
        // Do nothing - only here for Tomcat to have something to call
    }

    Crud(Whelk whelk) {
        this.whelk = whelk
    }

    @Override
    void init() {
        if (!whelk) {
            whelk = WhelkFactory.getSingletonWhelk()
        }
        displayData = whelk.displayData
        vocabData = whelk.vocabData

        jsonld = whelk.jsonld
        search = new SearchUtils(whelk)
        validator = JsonLdValidator.from(jsonld)
        converterUtils = new ConverterUtils(whelk)
        siteConfig = mapper.readValue(getClass().classLoader
                .getResourceAsStream("site_config.json").getBytes(), Map)
        sitesData = (Map) siteConfig['sites']
        siteAlias = (Map) siteConfig['site_alias']

        cachedDocs = [
                (whelk.vocabContextUri): getDocumentFromStorage(whelk.vocabContextUri, null),
                (whelk.vocabDisplayUri): getDocumentFromStorage(whelk.vocabDisplayUri, null),
                (whelk.vocabUri): getDocumentFromStorage(whelk.vocabUri, null)

        ]
        Tuple2<Document, String> docAndLoc = getDocumentFromStorage(whelk.kbvContextUri)
        Document contextDoc = docAndLoc.v1
        if (contextDoc) {
            targetVocabMapper = new TargetVocabMapper(whelk.jsonld, contextDoc.data)
        }
    }

    void handleQuery(HttpServletRequest request, HttpServletResponse response) {
        Map queryParameters = new HashMap<String, String[]>(request.getParameterMap())

        // Depending on what site/client we're serving, we might need to add extra query parameters
        // before they're sent further.
        String activeSite = request.getAttribute('activeSite')
        Map activeSiteData = (Map) sitesData[activeSite]

        if (activeSite != siteConfig['default_site']) {
            queryParameters.put('_site_base_uri', [activeSiteData['@id']] as String[])
        }

        if (!queryParameters['_statsrepr'] && activeSiteData['statsfind']) {
            queryParameters.put('_statsrepr', [mapper.writeValueAsString(activeSiteData['statsfind'])] as String[])
        }

        if (!queryParameters['_boost'] && activeSiteData['boost']) {
            queryParameters.put('_boost', [activeSiteData['boost']] as String[])
        }

        try {
            Map results = search.doSearch(queryParameters)
            String uri = request.getRequestURI()
            Map contextData = whelk.jsonld.context
            def crudReq = CrudGetRequest.parse(request)
            def dataBody = getNegotiatedDataBody(crudReq, contextData, results, uri)

            sendGetResponse(response, dataBody, null, crudReq.getPath(), crudReq.getContentType(), crudReq.getId())
        } catch (ElasticIOException | UnexpectedHttpStatusException e) {
            log.error("Attempted elastic query, but failed: $e", e)
            throw new WhelkRuntimeException("Failed to reach elastic for query.", e)
        } catch (WhelkRuntimeException e) {
            log.error("Attempted elastic query, but whelk has no elastic component configured.", e)
            throw new OtherStatusException("Attempted to use elastic for query, but no elastic component is configured.",
                    HttpServletResponse.SC_NOT_IMPLEMENTED)
        } catch (InvalidQueryException e) {
            log.warn("Invalid query: ${queryParameters}")
            throw new BadRequestException("Invalid query, please check the documentation. ${e.getMessage()}")
        }
    }

    void handleData(HttpServletRequest request, HttpServletResponse response) {
        Map queryParameters = new HashMap<String, String[]>(request.getParameterMap())
        String activeSite = request.getAttribute('activeSite')
        Map results = [:]
        results.putAll((Map) sitesData[activeSite])

        if (!queryParameters['_statsrepr']) {
            queryParameters.put('_statsrepr', [mapper.writeValueAsString(sitesData[activeSite]['statsindex'])] as String[])
        }
        if (!queryParameters['_limit']) {
            queryParameters.put('_limit', ["0"] as String[])
        }
        if (!queryParameters['q']) {
            queryParameters.put('q', ["*"] as String[])
        }
        Map searchResults = search.doSearch(queryParameters)
        results['statistics'] = searchResults['stats']

        String responseContentType = CrudUtils.getBestContentType(request)
        if (responseContentType == MimeTypes.JSONLD && !results[JsonLd.CONTEXT_KEY]) {
            results[JsonLd.CONTEXT_KEY] = CONTEXT_PATH
        }
        def jsonResult = mapper.writeValueAsString(results)<commit_msg>Filter on site_base_uri=activeSite in handleData<commit_after>package whelk.rest.api

import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Log4j2 as Log
import io.prometheus.client.Counter
import io.prometheus.client.Gauge
import io.prometheus.client.Histogram
import io.prometheus.client.Summary
import org.apache.http.entity.ContentType
import whelk.Document
import whelk.IdGenerator
import whelk.IdType
import whelk.JsonLd
import whelk.JsonLdValidator
import whelk.TargetVocabMapper
import whelk.Whelk
import whelk.component.PostgreSQLComponent
import whelk.exception.ElasticIOException
import whelk.exception.InvalidQueryException
import whelk.exception.LinkValidationException
import whelk.exception.ModelValidationException
import whelk.exception.StaleUpdateException
import whelk.exception.StorageCreateFailedException
import whelk.exception.UnexpectedHttpStatusException
import whelk.exception.WhelkRuntimeException
import whelk.history.History
import whelk.rest.api.CrudGetRequest.Lens
import whelk.rest.security.AccessControl
import whelk.util.WhelkFactory

import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.lang.management.ManagementFactory

import static whelk.rest.api.CrudUtils.ETag
import static whelk.rest.api.HttpTools.getBaseUri
import static whelk.rest.api.HttpTools.sendError
import static whelk.rest.api.HttpTools.sendResponse
import static whelk.util.Jackson.mapper

/**
 * Handles all GET/PUT/POST/DELETE requests against the backend.
 */
@Log
@CompileStatic
class Crud extends HttpServlet {
    final static String XL_ACTIVE_SIGEL_HEADER = 'XL-Active-Sigel'
    final static String CONTEXT_PATH = '/context.jsonld'
    final static String DATA_CONTENT_TYPE = "application/ld+json"

    static final Counter requests = Counter.build()
        .name("api_requests_total").help("Total requests to API.")
        .labelNames("method").register()

    static final Counter failedRequests = Counter.build()
        .name("api_failed_requests_total").help("Total failed requests to API.")
        .labelNames("method", "status").register()

    static final Gauge ongoingRequests = Gauge.build()
        .name("api_ongoing_requests_total").help("Total ongoing API requests.")
        .labelNames("method").register()

    static final Summary requestsLatency = Summary.build()
        .name("api_requests_latency_seconds")
        .help("API request latency in seconds.")
        .labelNames("method")
        .quantile(0.5f, 0.05f)
        .quantile(0.95f, 0.01f)
        .quantile(0.99f, 0.001f)
        .register()

    static final Histogram requestsLatencyHistogram = Histogram.build()
            .name("api_requests_latency_seconds_histogram").help("API request latency in seconds.")
            .labelNames("method")
            .register()

    Whelk whelk

    Map vocabData
    Map displayData
    JsonLd jsonld
    JsonLdValidator validator
    TargetVocabMapper targetVocabMapper

    SearchUtils search
    AccessControl accessControl = new AccessControl()
    ConverterUtils converterUtils

    Map siteConfig
    Map sitesData
    Map siteAlias

    Map<String, Tuple2<Document, String>> cachedDocs

    Crud() {
        // Do nothing - only here for Tomcat to have something to call
    }

    Crud(Whelk whelk) {
        this.whelk = whelk
    }

    @Override
    void init() {
        if (!whelk) {
            whelk = WhelkFactory.getSingletonWhelk()
        }
        displayData = whelk.displayData
        vocabData = whelk.vocabData

        jsonld = whelk.jsonld
        search = new SearchUtils(whelk)
        validator = JsonLdValidator.from(jsonld)
        converterUtils = new ConverterUtils(whelk)
        siteConfig = mapper.readValue(getClass().classLoader
                .getResourceAsStream("site_config.json").getBytes(), Map)
        sitesData = (Map) siteConfig['sites']
        siteAlias = (Map) siteConfig['site_alias']

        cachedDocs = [
                (whelk.vocabContextUri): getDocumentFromStorage(whelk.vocabContextUri, null),
                (whelk.vocabDisplayUri): getDocumentFromStorage(whelk.vocabDisplayUri, null),
                (whelk.vocabUri): getDocumentFromStorage(whelk.vocabUri, null)

        ]
        Tuple2<Document, String> docAndLoc = getDocumentFromStorage(whelk.kbvContextUri)
        Document contextDoc = docAndLoc.v1
        if (contextDoc) {
            targetVocabMapper = new TargetVocabMapper(whelk.jsonld, contextDoc.data)
        }
    }

    void handleQuery(HttpServletRequest request, HttpServletResponse response) {
        Map queryParameters = new HashMap<String, String[]>(request.getParameterMap())

        // Depending on what site/client we're serving, we might need to add extra query parameters
        // before they're sent further.
        String activeSite = request.getAttribute('activeSite')
        Map activeSiteData = (Map) sitesData[activeSite]

        if (activeSite != siteConfig['default_site']) {
            queryParameters.put('_site_base_uri', [activeSiteData['@id']] as String[])
        }

        if (!queryParameters['_statsrepr'] && activeSiteData['statsfind']) {
            queryParameters.put('_statsrepr', [mapper.writeValueAsString(activeSiteData['statsfind'])] as String[])
        }

        if (!queryParameters['_boost'] && activeSiteData['boost']) {
            queryParameters.put('_boost', [activeSiteData['boost']] as String[])
        }

        try {
            Map results = search.doSearch(queryParameters)
            String uri = request.getRequestURI()
            Map contextData = whelk.jsonld.context
            def crudReq = CrudGetRequest.parse(request)
            def dataBody = getNegotiatedDataBody(crudReq, contextData, results, uri)

            sendGetResponse(response, dataBody, null, crudReq.getPath(), crudReq.getContentType(), crudReq.getId())
        } catch (ElasticIOException | UnexpectedHttpStatusException e) {
            log.error("Attempted elastic query, but failed: $e", e)
            throw new WhelkRuntimeException("Failed to reach elastic for query.", e)
        } catch (WhelkRuntimeException e) {
            log.error("Attempted elastic query, but whelk has no elastic component configured.", e)
            throw new OtherStatusException("Attempted to use elastic for query, but no elastic component is configured.",
                    HttpServletResponse.SC_NOT_IMPLEMENTED)
        } catch (InvalidQueryException e) {
            log.warn("Invalid query: ${queryParameters}")
            throw new BadRequestException("Invalid query, please check the documentation. ${e.getMessage()}")
        }
    }

    void handleData(HttpServletRequest request, HttpServletResponse response) {
        Map queryParameters = new HashMap<String, String[]>(request.getParameterMap())
        String activeSite = request.getAttribute('activeSite')
        Map activeSiteData = (Map) sitesData[activeSite]

        Map results = [:]
        results.putAll((Map) activeSiteData)

        if (activeSite != siteConfig['default_site']) {
            queryParameters.put('_site_base_uri', [activeSiteData['@id']] as String[])
        }

        if (!queryParameters['_statsrepr']) {
            queryParameters.put('_statsrepr', [mapper.writeValueAsString(activeSiteData['statsindex'])] as String[])
        }
        if (!queryParameters['_limit']) {
            queryParameters.put('_limit', ["0"] as String[])
        }
        if (!queryParameters['q']) {
            queryParameters.put('q', ["*"] as String[])
        }
        Map searchResults = search.doSearch(queryParameters)
        results['statistics'] = searchResults['stats']

        String responseContentType = CrudUtils.getBestContentType(request)
        if (responseContentType == MimeTypes.JSONLD && !results[JsonLd.CONTEXT_KEY]) {
            results[JsonLd.CONTEXT_KEY] = CONTEXT_PATH
        }
        def jsonResult = mapper.writeValueAsString(results)
{'fraction_non_alphanumeric': 0.0971565475064624, 'fraction_numerical': 0.0030262909022129754, 'mean_word_length': 3.3114976895895625, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '126920', 'n_tokens_mistral': 4496, 'n_tokens_neox': 4309, 'n_words': 954}