<commit_before>'use strict';

const
  debug = require('debug')('kuzzle:services:elasticsearch'),
  _ = require('lodash'),
  Promise = require('bluebird'),
  Service = require('./service'),
  BadRequestError = require('kuzzle-common-objects').errors.BadRequestError,
  InternalError = require('kuzzle-common-objects').errors.InternalError,
  ServiceUnavailableError = require('kuzzle-common-objects').errors.ServiceUnavailableError,
  NotFoundError = require('kuzzle-common-objects').errors.NotFoundError,
  KuzzleError = require('kuzzle-common-objects').errors.KuzzleError,
  ExternalServiceError = require('kuzzle-common-objects').errors.ExternalServiceError,
  Request = require('kuzzle-common-objects').Request,
  es = require('elasticsearch'),
  ms = require('ms'),
  highwayhash = require('highwayhash'),
  compareVersions = require('compare-versions');

const scrollCachePrefix = '_docscroll_';

const errorMessagesMapping = [
  {
    // [illegal_argument_exception] object mapping [titi] can't be changed from nested to non-nested
    regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from nested to non-nested$/,
    replacement: 'Can not change mapping for field "$1" from nested to another type'
  },
  {
    // [illegal_argument_exception] object mapping [baz] can't be changed from non-nested to nested
    regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from non-nested to nested$/,
    replacement: 'Can not change mapping for field "$1" from object to another type'
  },
  {
    // [illegal_argument_exception] Can't merge a non object mapping [aeaze] with an object mapping [aeaze]
    regex: /^\[illegal_argument_exception] Can't merge a non object mapping \[(.*?)] with an object mapping \[(.*?)]$/,
    replacement: 'Can not change mapping for field "$1" from object to a scalar type'
  },
  {
    // [illegal_argument_exception] [tutu.tutu] is defined as an object in mapping [aze] but this name is already used for a field in other types
    regex: /^\[illegal_argument_exception] \[(.*?)] is defined as an object in mapping \[(.*?)] but this name is already used for a field in other types$/,
    replacement: 'Can not set mapping for field "$1" on collection "$2" because the field name is already used in another collection with a different type'
  },
  {
    // [illegal_argument_exception] mapper [source.flags] of different type, current_type [string], merged_type [long]
    regex: /^\[illegal_argument_exception] mapper \[(.*?)] of different type, current_type \[(.*?)], merged_type \[(.*?)]$/,
    replacement: 'Can not change type of field "$1" from "$2" to "$3"'
  },
  {
    // [mapper_parsing_exception] Mapping definition for [flags] has unsupported parameters:  [index : not_analyzed]
    // eslint-disable-next-line no-regex-spaces
    regex: /^\[mapper_parsing_exception] Mapping definition for \[(.*?)] has unsupported parameters:  \[(.*?)]$/,
    replacement: 'Parameter "$2" is not supported for field "$1"'
  },
  {
    // [mapper_parsing_exception] No handler for type [booleasn] declared on field [not]
    regex: /^\[mapper_parsing_exception] No handler for type \[(.*?)] declared on field \[(.*?)]$/,
    replacement: 'Can not set mapping for field "$2" because type "$1" does not exist'
  },
  {
    // [mapper_parsing_exception] failed to parse [conditions.host.flags]
    regex: /^\[mapper_parsing_exception] failed to parse \[(.*?)]$/,
    replacement: 'Failed to validate value of field "$1". Are you trying to insert nested value in a non-nested field ?'
  },
  {
    // [index_not_found_exception] no such index, with { resource.type=index_or_alias resource.id=foso index=foso }
    regex: /^\[index_not_found_exception] no such index, with { resource\.type=([^\s]+) resource\.id=([^\s]+) (index_uuid=.* )?index=([^\s]+) }$/,
    replacement: 'Index "$2" does not exist, please create it first'
  },
  {
    // [mapper_parsing_exception] Expected map for property [fields] on field [foo] but got a class java.lang.String
    regex: /^\[mapper_parsing_exception] Expected map for property \[fields] on field \[(.*?)] but got a class java\.lang\.String$/,
    replacement: 'Mapping for field "$1" must be an object with a property "type"'
  },
  {
    regex: /^\[version_conflict_engine_exception] \[data]\[(.*?)]: version conflict.*$/,
    replacement: 'Unable to modify document "$1": cluster sync failed (too many simultaneous changes applied)'
  }
];

/**
 * @property {Kuzzle} kuzzle
 * @property {object} settings
 * @property {object} client
 * @param {Kuzzle} kuzzle kuzzle instance
 * @param {object} options used to start the service
 * @param {object} config used to start the service
 * @constructor
 */
class ElasticSearch extends Service {
  constructor(kuzzle, options, config) {
    super();
    this.kuzzle = kuzzle;
    this.config = config;
    this.settings = {
      service: options.service,
      autoRefresh: options.autoRefresh || {}
    };
    this.client = null;
    this.esVersion = null;
  }

  /**
   * Initialize the elasticsearch client
   *
   * @returns {Promise}
   */
  init() {
    if (this.client) {
      return Promise.resolve(this);
    }

    this.client = buildClient({
      host: this.config.host + ':' + this.config.port,
      apiVersion: this.config.apiVersion
    });

    return Promise.resolve(this.client.info())
      .then(response => {
        this.esVersion = response.version;

        if (this.esVersion && compareVersions(this.esVersion.number, '5.0.0') < 0) {
          return Promise.reject(`Your elasticsearch version is ${this.esVersion.number}; Only elasticsearch version 5.0.0 and above are supported.`);
        }

        return Promise.resolve(this);
      });
  }

  /**
   * Return some basic information about this service
   *
   * @returns {Promise} service informations
   */
  getInfos() {
    const response = {
      type: 'elasticsearch',
      api: this.config.apiVersion
    };

    return this.client.info()
      .then(res => {
        /** @type {{version: {number: Number, lucene_version: String}}} res */
        response.version = res.version.number;
        response.lucene = res.version.lucene_version;

        return this.client.cluster.health();
      })
      .then(res => {
        /** @type {{status: String, number_of_nodes: Number}} res */
        response.status = res.status;
        response.nodes = res.number_of_nodes;
        return this.client.cluster.stats({human: true});
      })
      .then(res => {
        response.spaceUsed = res.indices.store.size;
        response.nodes = res.nodes;
        return response;
      })
      .catch(error => Promise.reject(this.formatESError(error)));
  }

  /**
   * Scroll results from previous elasticsearch query
   * @param {Request} request
   * @returns {Promise} resolve documents matching the scroll id
   */
  scroll(request) {
    const esRequest = getElasticsearchRequest(request, ['scroll', 'scrollId']);

    if (!esRequest.scroll) {
      esRequest.scroll = this.config.defaults.scrollTTL;
    }

    const cacheKey = scrollCachePrefix + highwayhash.asHexString(this.kuzzle.hashKey, Buffer.from(esRequest.scrollId));

    return this.kuzzle.services.list.internalCache.exists(cacheKey)
      .then(exists => {
        if (exists === 0) {
          throw new NotFoundError('Non-existing or expired scroll identifier');
        }

        // ms(scroll) may return undefined if in microseconds or in nanoseconds
        const ttl = ms(esRequest.scroll) || ms(this.config.defaults.scrollTTL);

        return this.kuzzle.services.list.internalCache.pexpire(cacheKey, ttl);
      })
      .then(() => this.client.scroll(esRequest))
      .then(result => {

        return flattenSearchResults(result);
      })
      .catch(error => Promise.reject(this.formatESError(error)));
  }

  /**
   * Search documents from elasticsearch with a query
   * @param {Request} request
   * @returns {Promise} resolve documents matching the filter
   */
  search(request) {
    const esRequest = getElasticsearchRequest(request, ['from', 'size', 'scroll']);

    // todo add condition once the 'trash' feature has been implemented
    addActiveFilter(esRequest);

    return this.client.search(esRequest)
      .then(result => {
        let flattened = flattenSearchResults(result);

        if (flattened._scroll_id !== undefined) {
          const
            // ms(scroll) may return undefined if in microseconds or in nanoseconds
            ttl = ms(esRequest.scroll) || ms(this.config.defaults.scrollTTL),
            key = scrollCachePrefix + highwayhash.asHexString(this.kuzzle.hashKey, Buffer.from(flattened._scroll_id));

          return this.kuzzle.services.list.internalCache.psetex(key, ttl, 0)
            .then(() => flattened);
        }

        return flattened;<commit_msg>better code<commit_after>'use strict';

const
  debug = require('debug')('kuzzle:services:elasticsearch'),
  _ = require('lodash'),
  Promise = require('bluebird'),
  Service = require('./service'),
  BadRequestError = require('kuzzle-common-objects').errors.BadRequestError,
  InternalError = require('kuzzle-common-objects').errors.InternalError,
  ServiceUnavailableError = require('kuzzle-common-objects').errors.ServiceUnavailableError,
  NotFoundError = require('kuzzle-common-objects').errors.NotFoundError,
  KuzzleError = require('kuzzle-common-objects').errors.KuzzleError,
  ExternalServiceError = require('kuzzle-common-objects').errors.ExternalServiceError,
  Request = require('kuzzle-common-objects').Request,
  es = require('elasticsearch'),
  ms = require('ms'),
  highwayhash = require('highwayhash'),
  compareVersions = require('compare-versions');

const scrollCachePrefix = '_docscroll_';

const errorMessagesMapping = [
  {
    // [illegal_argument_exception] object mapping [titi] can't be changed from nested to non-nested
    regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from nested to non-nested$/,
    replacement: 'Can not change mapping for field "$1" from nested to another type'
  },
  {
    // [illegal_argument_exception] object mapping [baz] can't be changed from non-nested to nested
    regex: /^\[illegal_argument_exception] object mapping \[(.*?)] can't be changed from non-nested to nested$/,
    replacement: 'Can not change mapping for field "$1" from object to another type'
  },
  {
    // [illegal_argument_exception] Can't merge a non object mapping [aeaze] with an object mapping [aeaze]
    regex: /^\[illegal_argument_exception] Can't merge a non object mapping \[(.*?)] with an object mapping \[(.*?)]$/,
    replacement: 'Can not change mapping for field "$1" from object to a scalar type'
  },
  {
    // [illegal_argument_exception] [tutu.tutu] is defined as an object in mapping [aze] but this name is already used for a field in other types
    regex: /^\[illegal_argument_exception] \[(.*?)] is defined as an object in mapping \[(.*?)] but this name is already used for a field in other types$/,
    replacement: 'Can not set mapping for field "$1" on collection "$2" because the field name is already used in another collection with a different type'
  },
  {
    // [illegal_argument_exception] mapper [source.flags] of different type, current_type [string], merged_type [long]
    regex: /^\[illegal_argument_exception] mapper \[(.*?)] of different type, current_type \[(.*?)], merged_type \[(.*?)]$/,
    replacement: 'Can not change type of field "$1" from "$2" to "$3"'
  },
  {
    // [mapper_parsing_exception] Mapping definition for [flags] has unsupported parameters:  [index : not_analyzed]
    // eslint-disable-next-line no-regex-spaces
    regex: /^\[mapper_parsing_exception] Mapping definition for \[(.*?)] has unsupported parameters:  \[(.*?)]$/,
    replacement: 'Parameter "$2" is not supported for field "$1"'
  },
  {
    // [mapper_parsing_exception] No handler for type [booleasn] declared on field [not]
    regex: /^\[mapper_parsing_exception] No handler for type \[(.*?)] declared on field \[(.*?)]$/,
    replacement: 'Can not set mapping for field "$2" because type "$1" does not exist'
  },
  {
    // [mapper_parsing_exception] failed to parse [conditions.host.flags]
    regex: /^\[mapper_parsing_exception] failed to parse \[(.*?)]$/,
    replacement: 'Failed to validate value of field "$1". Are you trying to insert nested value in a non-nested field ?'
  },
  {
    // [index_not_found_exception] no such index, with { resource.type=index_or_alias resource.id=foso index=foso }
    regex: /^\[index_not_found_exception] no such index, with { resource\.type=([^\s]+) resource\.id=([^\s]+) (index_uuid=.* )?index=([^\s]+) }$/,
    replacement: 'Index "$2" does not exist, please create it first'
  },
  {
    // [mapper_parsing_exception] Expected map for property [fields] on field [foo] but got a class java.lang.String
    regex: /^\[mapper_parsing_exception] Expected map for property \[fields] on field \[(.*?)] but got a class java\.lang\.String$/,
    replacement: 'Mapping for field "$1" must be an object with a property "type"'
  },
  {
    regex: /^\[version_conflict_engine_exception] \[data]\[(.*?)]: version conflict.*$/,
    replacement: 'Unable to modify document "$1": cluster sync failed (too many simultaneous changes applied)'
  }
];

/**
 * @property {Kuzzle} kuzzle
 * @property {object} settings
 * @property {object} client
 * @param {Kuzzle} kuzzle kuzzle instance
 * @param {object} options used to start the service
 * @param {object} config used to start the service
 * @constructor
 */
class ElasticSearch extends Service {
  constructor(kuzzle, options, config) {
    super();
    this.kuzzle = kuzzle;
    this.config = config;
    this.settings = {
      service: options.service,
      autoRefresh: options.autoRefresh || {}
    };
    this.client = null;
    this.esVersion = null;
  }

  /**
   * Initialize the elasticsearch client
   *
   * @returns {Promise}
   */
  init() {
    if (this.client) {
      return Promise.resolve(this);
    }

    this.client = buildClient({
      host: this.config.host + ':' + this.config.port,
      apiVersion: this.config.apiVersion
    });

    return Promise.resolve(this.client.info())
      .then(response => {
        this.esVersion = response.version;

        if (this.esVersion && compareVersions(this.esVersion.number, '5.0.0') < 0) {
          return Promise.reject(`Your elasticsearch version is ${this.esVersion.number}; Only elasticsearch version 5.0.0 and above are supported.`);
        }

        return Promise.resolve(this);
      });
  }

  /**
   * Return some basic information about this service
   *
   * @returns {Promise} service informations
   */
  getInfos() {
    const response = {
      type: 'elasticsearch',
      api: this.config.apiVersion
    };

    return this.client.info()
      .then(res => {
        /** @type {{version: {number: Number, lucene_version: String}}} res */
        response.version = res.version.number;
        response.lucene = res.version.lucene_version;

        return this.client.cluster.health();
      })
      .then(res => {
        /** @type {{status: String, number_of_nodes: Number}} res */
        response.status = res.status;
        response.nodes = res.number_of_nodes;
        return this.client.cluster.stats({human: true});
      })
      .then(res => {
        response.spaceUsed = res.indices.store.size;
        response.nodes = res.nodes;
        return response;
      })
      .catch(error => Promise.reject(this.formatESError(error)));
  }

  /**
   * Scroll results from previous elasticsearch query
   * @param {Request} request
   * @returns {Promise} resolve documents matching the scroll id
   */
  scroll(request) {
    const esRequest = getElasticsearchRequest(request, ['scroll', 'scrollId']);

    if (!esRequest.scroll) {
      esRequest.scroll = this.config.defaults.scrollTTL;
    }

    const cacheKey = scrollCachePrefix + highwayhash.asHexString(this.kuzzle.hashKey, Buffer.from(esRequest.scrollId));

    return this.kuzzle.services.list.internalCache.exists(cacheKey)
      .then(exists => {
        if (exists === 0) {
          throw new NotFoundError('Non-existing or expired scroll identifier');
        }

        // ms(scroll) may return undefined if in microseconds or in nanoseconds
        const ttl = ms(esRequest.scroll) || ms(this.config.defaults.scrollTTL);

        return this.kuzzle.services.list.internalCache.pexpire(cacheKey, ttl);
      })
      .then(() => this.client.scroll(esRequest))
      .then(result => flattenSearchResults(result))
      .catch(error => Promise.reject(this.formatESError(error)));
  }

  /**
   * Search documents from elasticsearch with a query
   * @param {Request} request
   * @returns {Promise} resolve documents matching the filter
   */
  search(request) {
    const esRequest = getElasticsearchRequest(request, ['from', 'size', 'scroll']);

    // todo add condition once the 'trash' feature has been implemented
    addActiveFilter(esRequest);

    return this.client.search(esRequest)
      .then(result => {
        let flattened = flattenSearchResults(result);

        if (flattened._scroll_id !== undefined) {
          const
            // ms(scroll) may return undefined if in microseconds or in nanoseconds
            ttl = ms(esRequest.scroll) || ms(this.config.defaults.scrollTTL),
            key = scrollCachePrefix + highwayhash.asHexString(this.kuzzle.hashKey, Buffer.from(flattened._scroll_id));

          return this.kuzzle.services.list.internalCache.psetex(key, ttl, 0)
            .then(() => flattened);
        }

        return flattened;
{'fraction_non_alphanumeric': 0.13560953513391089, 'fraction_numerical': 0.0028310967668874923, 'mean_word_length': 3.345964566929134, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5082998', 'n_tokens_mistral': 5253, 'n_tokens_neox': 5111, 'n_words': 1429}