<commit_before>"""Search backends used by django-watson."""

from __future__ import unicode_literals

import abc
import re

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db import transaction, connections, router
from django.db.models import Q, FloatField
from django.db.models.expressions import RawSQL, Value
from django.utils.encoding import force_text
from django.utils import six

from watson.models import SearchEntry, has_int_pk


def regex_from_word(word):
    """Generates a regext from the given search word."""
    return "(\s{word})|(^{word})".format(
        word=re.escape(word),
    )


# PostgreSQL to_tsquery operators: ! & : ( ) |
RE_POSTGRES_ESCAPE_CHARS = re.compile(r'[&:(|)!><]', re.UNICODE)
# MySQL boolean full-text search operators: > < ( ) " ~ * + -
RE_MYSQL_ESCAPE_CHARS = re.compile(r'["()><~*+-]', re.UNICODE)

RE_SPACE = re.compile(r"[\s]+", re.UNICODE)


def escape_query(text, re_escape_chars):
    """
    normalizes the query text to a format that can be consumed
    by the backend database
    """
    text = force_text(text)
    text = RE_SPACE.sub(" ", text)  # Standardize spacing.
    text = re_escape_chars.sub(" ", text)  # Replace harmful characters with space.
    text = text.strip()
    return text


class SearchBackend(six.with_metaclass(abc.ABCMeta)):
    """Base class for all search backends."""

    def is_installed(self):
        """Checks whether django-watson is installed."""
        return True<commit_msg>[TASK] Fix flake8 error that regex is not marked as regex<commit_after>"""Search backends used by django-watson."""

from __future__ import unicode_literals

import abc
import re

from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db import transaction, connections, router
from django.db.models import Q, FloatField
from django.db.models.expressions import RawSQL, Value
from django.utils.encoding import force_text
from django.utils import six

from watson.models import SearchEntry, has_int_pk


def regex_from_word(word):
    """Generates a regex from the given search word."""
    return r"(\s{word})|(^{word})".format(
        word=re.escape(word),
    )


# PostgreSQL to_tsquery operators: ! & : ( ) |
RE_POSTGRES_ESCAPE_CHARS = re.compile(r'[&:(|)!><]', re.UNICODE)
# MySQL boolean full-text search operators: > < ( ) " ~ * + -
RE_MYSQL_ESCAPE_CHARS = re.compile(r'["()><~*+-]', re.UNICODE)

RE_SPACE = re.compile(r"[\s]+", re.UNICODE)


def escape_query(text, re_escape_chars):
    """
    normalizes the query text to a format that can be consumed
    by the backend database
    """
    text = force_text(text)
    text = RE_SPACE.sub(" ", text)  # Standardize spacing.
    text = re_escape_chars.sub(" ", text)  # Replace harmful characters with space.
    text = text.strip()
    return text


class SearchBackend(six.with_metaclass(abc.ABCMeta)):
    """Base class for all search backends."""

    def is_installed(self):
        """Checks whether django-watson is installed."""
        return True
{'fraction_non_alphanumeric': 0.12370466321243523, 'fraction_numerical': 0.0003238341968911917, 'mean_word_length': 4.055646481178396, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6230406', 'n_tokens_mistral': 995, 'n_tokens_neox': 965, 'n_words': 277}