<commit_before>/*
 * Copyright (C) 2016 Language Technology Group and Interactive Graphics Systems Group, Technische Universität Darmstadt, Germany
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

package util.es

import com.google.inject.Inject
import org.elasticsearch.action.search.{ SearchRequestBuilder, SearchResponse }
import util.DateUtils
// scalastyle:off
import org.elasticsearch.index.query.QueryStringQueryBuilder._
import org.elasticsearch.index.query._
// scalastyle:on
import org.joda.time.LocalDateTime

import models.services.SearchClientService
import models.Facets

/** Common helper to create and parse elasticsearch queries. Further provides elasticsearch field mappings. */
class ESRequestUtils @Inject() (dateUtils: DateUtils) {

  /** Elasticsearch field storing the document content. */
  val docContentField = "Content"
  /** Elasticsearch field storing the document creation date. */
  val docDateField = "Created"
  /** Elasticsearch field storing the time expression occurring in the document. */
  val docTimeExpressionField = "SimpleTimeExpresion"

  /** Elasticsearch field for running aggregation queries using entity ids. */
  val entityIdsField = "Entities" -> "Entities.EntId"
  /** Elasticsearch field for running aggregation queries using document keywords. */
  val keywordsField = "Keywords" -> "Keywords.Keyword.raw"

  /** Elasticsearch fields for running aggregation queries using entity ids from separate entity types. */
  def convertEntityTypeToField = (t: String) => s"Entities${t.toLowerCase}.EntId"

  /** Executes the given search request. */
  def executeRequest(request: SearchRequestBuilder, cache: Boolean = true): SearchResponse = request.setRequestCache(cache).execute().actionGet()

  /**
   * Converts an internal search query represented as [[models.Facets]] to an elasticsearch query.
   *
   * @param facets the internal search query.
   * @param documentSize the number of documents to return for the given request. Pass ''0'', if only interested in aggregation.
   * @param index the data source index or database name to query.
   * @param client the elasticsearch client.
   * @return an elasticsearch query builder.
   */
  def createSearchRequest(facets: Facets, documentSize: Int, index: String, client: SearchClientService): SearchRequestBuilder = {
    val requestBuilder = client.client.prepareSearch(index)
      .setQuery(createQuery(facets))
      .setSize(documentSize)
      // We are only interested in the document id
      .addFields("id")

    requestBuilder
  }
<commit_msg>INIT document whitelisting ESRequestUtils scala<commit_after>/*
 * Copyright (C) 2016 Language Technology Group and Interactive Graphics Systems Group, Technische Universität Darmstadt, Germany
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

package util.es

import com.google.inject.Inject
import org.elasticsearch.action.search.{ SearchRequestBuilder, SearchResponse }

import util.DateUtils
// scalastyle:off
import org.elasticsearch.index.query.QueryStringQueryBuilder._
import org.elasticsearch.index.query._
// scalastyle:on
import org.joda.time.LocalDateTime

import models.services.SearchClientService
import models.Facets

/** Common helper to create and parse elasticsearch queries. Further provides elasticsearch field mappings. */
class ESRequestUtils @Inject() (dateUtils: DateUtils) {

  /** Elasticsearch field storing the document content. */
  val docContentField = "Content"
  /** Elasticsearch field storing the document creation date. */
  val docDateField = "Created"
  /** Elasticsearch field storing the time expression occurring in the document. */
  val docTimeExpressionField = "SimpleTimeExpresion"

  /** Elasticsearch field for running aggregation queries using entity ids. */
  val entityIdsField = "Entities" -> "Entities.EntId"
  /** Elasticsearch field for running aggregation queries using document keywords. */
  val keywordsField = "Keywords" -> "Keywords.Keyword.raw"

  /** Elasticsearch fields for running aggregation queries using entity ids from separate entity types. */
  def convertEntityTypeToField = (t: String) => s"Entities${t.toLowerCase}.EntId"

  /** Executes the given search request. */
  def executeRequest(request: SearchRequestBuilder, cache: Boolean = true): SearchResponse = request.setRequestCache(cache).execute().actionGet()

  /**
   * Converts an internal search query represented as [[models.Facets]] to an elasticsearch query.
   *
   * @param facets the internal search query.
   * @param documentSize the number of documents to return for the given request. Pass ''0'', if only interested in aggregation.
   * @param index the data source index or database name to query.
   * @param client the elasticsearch client.
   * @return an elasticsearch query builder.
   */
  def createSearchRequest(facets: Facets, documentSize: Int, index: String, client: SearchClientService): SearchRequestBuilder = {
    val requestBuilder = client.client.prepareSearch(index)
      .setQuery(createQuery(facets))
      .setSize(documentSize)
      // We are only interested in the document id
      .addFields("id")

    requestBuilder
  }


  /** newsleak version 2.0.0: document whitelisting */
  def checkEntities(index: String, client: SearchClientService): AnyRef = {
    val requestBuilder = client.client.prepareGet(index, "document", "11")
      .get()
      .getSource()
      .get("Entities")

    requestBuilder
  }

{'fraction_non_alphanumeric': 0.08365758754863813, 'fraction_numerical': 0.0025441484585453458, 'mean_word_length': 4.245682888540031, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188081', 'n_tokens_mistral': 1688, 'n_tokens_neox': 1674, 'n_words': 684}