<commit_before># -*- coding: utf-8 -*-

from __future__ import division

import copy
import functools
import logging
import math
import re
import unicodedata

from elasticsearch import (
    ConnectionError,
    Elasticsearch,
    NotFoundError,
    RequestError,
    TransportError,
    helpers,
)
from modularodm import Q
import six

from framework import sentry
from framework.celery_tasks import app as celery_app
from framework.mongo.utils import paginated

from website import settings
from website.files.models import FileNode
from website.filters import gravatar
from website.models import User, Node
from website.project.licenses import serialize_node_license_record
from website.search import exceptions
from website.search.util import build_query, clean_splitters
from website.util import sanitize
from website.views import validate_page_num

logger = logging.getLogger(__name__)


# These are the doc_types that exist in the search database
ALIASES = {
    'component': 'Components',
    'file': 'Files',
    'institution': 'Institutions',
    'project': 'Projects',
    'preprint': 'Preprints',
    'registration': 'Registrations',
    'total': 'Total',
    'user': 'Users',
}

DOC_TYPE_TO_MODEL = {
    'component': Node,
    'project': Node,
    'registration': Node,
    'user': User,
    'file': FileNode,
}

# Prevent tokenizing and stop word removal.
NOT_ANALYZED_PROPERTY = {'type': 'string', 'index': 'not_analyzed'}

# Perform stemming on the field it's applied to.
ENGLISH_ANALYZER_PROPERTY = {'type': 'string', 'analyzer': 'english'}

INDEX = settings.ELASTIC_INDEX

try:
    es = Elasticsearch(
        settings.ELASTIC_URI,
        request_timeout=settings.ELASTIC_TIMEOUT
    )<commit_msg>Add preprint to DOC_TYPE_TO_MODEL dict.<commit_after># -*- coding: utf-8 -*-

from __future__ import division

import copy
import functools
import logging
import math
import re
import unicodedata

from elasticsearch import (
    ConnectionError,
    Elasticsearch,
    NotFoundError,
    RequestError,
    TransportError,
    helpers,
)
from modularodm import Q
import six

from framework import sentry
from framework.celery_tasks import app as celery_app
from framework.mongo.utils import paginated

from website import settings
from website.files.models import FileNode
from website.filters import gravatar
from website.models import User, Node
from website.project.licenses import serialize_node_license_record
from website.search import exceptions
from website.search.util import build_query, clean_splitters
from website.util import sanitize
from website.views import validate_page_num

logger = logging.getLogger(__name__)


# These are the doc_types that exist in the search database
ALIASES = {
    'component': 'Components',
    'file': 'Files',
    'institution': 'Institutions',
    'project': 'Projects',
    'preprint': 'Preprints',
    'registration': 'Registrations',
    'total': 'Total',
    'user': 'Users',
}

DOC_TYPE_TO_MODEL = {
    'component': Node,
    'project': Node,
    'registration': Node,
    'preprint': Node,
    'user': User,
    'file': FileNode,
}

# Prevent tokenizing and stop word removal.
NOT_ANALYZED_PROPERTY = {'type': 'string', 'index': 'not_analyzed'}

# Perform stemming on the field it's applied to.
ENGLISH_ANALYZER_PROPERTY = {'type': 'string', 'analyzer': 'english'}

INDEX = settings.ELASTIC_INDEX

try:
    es = Elasticsearch(
        settings.ELASTIC_URI,
        request_timeout=settings.ELASTIC_TIMEOUT
    )
{'fraction_non_alphanumeric': 0.09224137931034483, 'fraction_numerical': 0.0005747126436781609, 'mean_word_length': 4.456112852664577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5303423', 'n_tokens_mistral': 1048, 'n_tokens_neox': 1035, 'n_words': 320}