<commit_before># Copyright 2015 Hewlett-Packard Corporation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
from elasticsearch import exceptions as es_exc
from elasticsearch import helpers
import logging
from oslo_utils import fnmatch
import six

from searchlight.elasticsearch import ROLE_USER_FIELD
from searchlight.i18n import _LE, _LW, _LI

# Refer to ROLE_USER_FIELD
ADMIN_ID_SUFFIX = "_ADMIN"
USER_ID_SUFFIX = "_USER"


def strip_role_suffix(strip_from, suffix=None):
    if suffix:
        if strip_from.endswith(suffix):
            return strip_from[:-len(suffix)]
    else:
        if strip_from.endswith(ADMIN_ID_SUFFIX):
            return strip_from[:-len(ADMIN_ID_SUFFIX)]
        elif strip_from.endswith(USER_ID_SUFFIX):
            return strip_from[:-len(USER_ID_SUFFIX)]
    return strip_from


def get_metafield(doc, metafield):
    """For fields like _parent and _routing es1.x requires you to request them
    in 'fields', and they're returned as such. ES2.x always returns them but
    at the root document level (like _id). This function copes with both.
    """
    return doc.get(metafield, doc.get('fields', {}).get(metafield))


# String from Alias Failure Exception. See _is_multiple_alias_exception().
ALIAS_EXCEPTION_STRING = "has more than one indices associated with it"

DOC_VALUE_TYPES = ('long', 'integer', 'short', 'boolean', 'byte',
                   'double', 'float', 'date', 'binary', 'geo_point', 'ip')

LOG = logging.getLogger(__name__)


class IndexingHelper(object):
    """Class to abstract away some of the details of indexing documents
    including versioning, parent-child links and separation by role.

    Role separation is reasonably simple; documents with admin-only fields
    will be indexed twice, once for admins and once for normal users, with some
    fields excluded for the second case. The document IDs use the suffixes
    defined in ADMIN_ID_SUFFIX and USER_ID_SUFFIX.

    In the case of parent child relationships things are more complicated. The
    ids in that case follow these rules:
    * parent is role separated
      * child is role separated: the _USER and _ADMIN docs each refer to a
            parent with the same suffix
      * child is NOT separated: this case is not encouraged, but where it
            exists the child documents will use the _USER parent document as
            their _parent to avoid security issues with has_parent query
            fishing
    * parent is not role separated
      * child is role separated: both child docs will use the parent id
            without any prefix
      * child is not role separated: the simple case; no suffix in either
            case
    """

    def __init__(self, plugin):
        self.plugin = plugin

        # Retain some state here for convenience
        self.engine = self.plugin.engine
        self.alias_name = plugin.alias_name_listener
        self.document_type = plugin.document_type

    @property
    def index_chunk_size(self):
        return 200

    def _index_alias_multiple_indexes_bulk(self, documents=None, actions=None,
                                           versions=None):
        """A bulk operation failed by trying to access an alias that has
           multiple indexes. To rememdy this we will need to iterate on all
           indexes within the alias and retry the bulk operation individually.
        """
        indexes = self.engine.indices.get_alias(index=self.alias_name)
        for index_name in indexes:
            try:
                if documents:
                    result = helpers.bulk(
                        client=self.engine,
                        index=index_name,
                        doc_type=self.document_type,
                        chunk_size=self.index_chunk_size,
                        actions=self._prepare_actions(documents,
                                                      versions))
                if actions:
                    result = helpers.bulk(
                        client=self.engine,
                        index=index_name,
                        doc_type=self.document_type,
                        chunk_size=self.index_chunk_size,
                        actions=actions)
                LOG.debug("Indexing result: %s", result)
            except Exception as e:
                # Log the error and continue to the next index.
                format_msg = {
                    'doc': self.document_type,
                    'msg': str(e)
                }
                LOG.error(_LE("Failed Indexing %(doc)s: %(msg)s") % format_msg)

    def _index_alias_multiple_indexes_get(self, doc_id, routing=None):
        """Getting a document from an alias with multiple indexes will fail.
           We need to retrive it from one of the indexes. We will choose
           the latest index. Since the indexes are named with a timestamp,
           a reverse sort will bring the latest to the front.
        """
        indexes = self.engine.indices.get_alias(index=self.alias_name)
        index_list = indexes.keys()
        index_list.sort(reverse=True)<commit_msg>Fix typo in helper.py<commit_after># Copyright 2015 Hewlett-Packard Corporation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import copy
from elasticsearch import exceptions as es_exc
from elasticsearch import helpers
import logging
from oslo_utils import fnmatch
import six

from searchlight.elasticsearch import ROLE_USER_FIELD
from searchlight.i18n import _LE, _LW, _LI

# Refer to ROLE_USER_FIELD
ADMIN_ID_SUFFIX = "_ADMIN"
USER_ID_SUFFIX = "_USER"


def strip_role_suffix(strip_from, suffix=None):
    if suffix:
        if strip_from.endswith(suffix):
            return strip_from[:-len(suffix)]
    else:
        if strip_from.endswith(ADMIN_ID_SUFFIX):
            return strip_from[:-len(ADMIN_ID_SUFFIX)]
        elif strip_from.endswith(USER_ID_SUFFIX):
            return strip_from[:-len(USER_ID_SUFFIX)]
    return strip_from


def get_metafield(doc, metafield):
    """For fields like _parent and _routing es1.x requires you to request them
    in 'fields', and they're returned as such. ES2.x always returns them but
    at the root document level (like _id). This function copes with both.
    """
    return doc.get(metafield, doc.get('fields', {}).get(metafield))


# String from Alias Failure Exception. See _is_multiple_alias_exception().
ALIAS_EXCEPTION_STRING = "has more than one indices associated with it"

DOC_VALUE_TYPES = ('long', 'integer', 'short', 'boolean', 'byte',
                   'double', 'float', 'date', 'binary', 'geo_point', 'ip')

LOG = logging.getLogger(__name__)


class IndexingHelper(object):
    """Class to abstract away some of the details of indexing documents
    including versioning, parent-child links and separation by role.

    Role separation is reasonably simple; documents with admin-only fields
    will be indexed twice, once for admins and once for normal users, with some
    fields excluded for the second case. The document IDs use the suffixes
    defined in ADMIN_ID_SUFFIX and USER_ID_SUFFIX.

    In the case of parent child relationships things are more complicated. The
    ids in that case follow these rules:
    * parent is role separated
      * child is role separated: the _USER and _ADMIN docs each refer to a
            parent with the same suffix
      * child is NOT separated: this case is not encouraged, but where it
            exists the child documents will use the _USER parent document as
            their _parent to avoid security issues with has_parent query
            fishing
    * parent is not role separated
      * child is role separated: both child docs will use the parent id
            without any prefix
      * child is not role separated: the simple case; no suffix in either
            case
    """

    def __init__(self, plugin):
        self.plugin = plugin

        # Retain some state here for convenience
        self.engine = self.plugin.engine
        self.alias_name = plugin.alias_name_listener
        self.document_type = plugin.document_type

    @property
    def index_chunk_size(self):
        return 200

    def _index_alias_multiple_indexes_bulk(self, documents=None, actions=None,
                                           versions=None):
        """A bulk operation failed by trying to access an alias that has
           multiple indexes. To rememdy this we will need to iterate on all
           indexes within the alias and retry the bulk operation individually.
        """
        indexes = self.engine.indices.get_alias(index=self.alias_name)
        for index_name in indexes:
            try:
                if documents:
                    result = helpers.bulk(
                        client=self.engine,
                        index=index_name,
                        doc_type=self.document_type,
                        chunk_size=self.index_chunk_size,
                        actions=self._prepare_actions(documents,
                                                      versions))
                if actions:
                    result = helpers.bulk(
                        client=self.engine,
                        index=index_name,
                        doc_type=self.document_type,
                        chunk_size=self.index_chunk_size,
                        actions=actions)
                LOG.debug("Indexing result: %s", result)
            except Exception as e:
                # Log the error and continue to the next index.
                format_msg = {
                    'doc': self.document_type,
                    'msg': str(e)
                }
                LOG.error(_LE("Failed Indexing %(doc)s: %(msg)s") % format_msg)

    def _index_alias_multiple_indexes_get(self, doc_id, routing=None):
        """Getting a document from an alias with multiple indexes will fail.
           We need to retrieve it from one of the indexes. We will choose
           the latest index. Since the indexes are named with a timestamp,
           a reverse sort will bring the latest to the front.
        """
        indexes = self.engine.indices.get_alias(index=self.alias_name)
        index_list = indexes.keys()
        index_list.sort(reverse=True)
{'fraction_non_alphanumeric': 0.06421210252796618, 'fraction_numerical': 0.0026424733550603364, 'mean_word_length': 2.290060851926978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4813161', 'n_tokens_mistral': 2990, 'n_tokens_neox': 2864, 'n_words': 1106}