<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
import uuid

from gevent.pywsgi import WSGIServer

import rasa_core
import shutil
import tempfile
import time
import zipfile
from threading import Thread

import six
import typing
from requests.exceptions import ConnectionError, InvalidURL
from six import string_types
from typing import Text, List, Optional, Callable, Any, Dict, Union

from rasa_core import training, constants
from rasa_core.channels import UserMessage, OutputChannel, InputChannel
from rasa_core.dispatcher import Dispatcher
from rasa_core.domain import Domain, check_domain_sanity
from rasa_core.interpreter import NaturalLanguageInterpreter
from rasa_core.nlg import NaturalLanguageGenerator
from rasa_core.policies import Policy
from rasa_core.policies.ensemble import SimplePolicyEnsemble, PolicyEnsemble
from rasa_core.policies.memoization import MemoizationPolicy
from rasa_core.processor import MessageProcessor
from rasa_core.tracker_store import InMemoryTrackerStore, TrackerStore
from rasa_core.trackers import DialogueStateTracker
from rasa_core.utils import EndpointConfig
from rasa_nlu.utils import is_url

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_core.nlg import NaturalLanguageGenerator as NLG

if six.PY2:
    from StringIO import StringIO as IOReader
else:
    from io import BytesIO as IOReader


def load_from_server(interpreter=None,  # type: NaturalLanguageInterpreter
                     generator=None,  # type: Union[EndpointConfig, NLG]
                     tracker_store=None,  # type: Optional[TrackerStore]
                     action_endpoint=None,  # type: Optional[EndpointConfig]
                     model_server=None,  # type: Optional[EndpointConfig]
                     wait_time_between_pulls=None,  # type: Optional[int]
                     ):
    # type: (...) -> Agent
    """Load a persisted model from a server."""

    agent = Agent(interpreter=interpreter,
                  generator=generator,
                  tracker_store=tracker_store,
                  action_endpoint=action_endpoint)

    try:
        _update_model_from_server(model_server, agent)
    except ConnectionError as e:
        logger.warn("Could not retrieve model from server. Connection Error.  "
                    "Running without a model. Server URL: `{}`\n".format(e))

    if wait_time_between_pulls:
        # continuously pull the model every `wait_time_between_pulls` seconds
        start_model_pulling_in_worker(model_server,
                                      wait_time_between_pulls,
                                      agent)
    return agent


def _init_model_from_server(model_server):
    # type: (EndpointConfig) -> Optional[(Domain, PolicyEnsemble, Text)]
    """Initialise a Rasa Core model from a URL."""

    if not is_url(model_server.url):
        raise InvalidURL(model_server.url)

    model_directory = tempfile.mkdtemp()

    fingerprint = _pull_model_and_fingerprint(model_server,
                                              model_directory,
                                              fingerprint=None)

    return fingerprint, model_directory


def _update_model_from_server(
        model_server,  # type: EndpointConfig
        agent,  # type: Agent
):
    # type: (...) -> None
    """Load a zipped Rasa Core model from a URL and update the passed agent."""

    if not is_url(model_server.url):
        raise InvalidURL(model_server.url)

    model_directory = tempfile.mkdtemp()

    new_model_fingerprint = _pull_model_and_fingerprint(
            model_server, model_directory, agent.fingerprint)
    if new_model_fingerprint:
        domain_path = os.path.join(os.path.abspath(model_directory),
                                   "domain.yml")
        domain = Domain.load(domain_path)
        policy_ensemble = PolicyEnsemble.load(model_directory)
        agent.update_model(domain, policy_ensemble, new_model_fingerprint)
    else:
        logger.debug("No new model found at "
                     "URL {}".format(model_server.url))


def _pull_model_and_fingerprint(model_server, model_directory, fingerprint):
    # type: (EndpointConfig, Text, Optional[Text]) -> Optional[Text]
    """Queries the model server and returns the value of the response's

    <ETag> header which contains the model hash."""
    header = {"If-None-Match": fingerprint}
    response = model_server.request(method="GET", headers=header)
<commit_msg>fixed error handling on model retrieval<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import logging
import os
import shutil
import tempfile
import time
import uuid
import zipfile
from threading import Thread

import six
import typing
from gevent.pywsgi import WSGIServer
from requests.exceptions import InvalidURL, RequestException
from six import string_types
from typing import Text, List, Optional, Callable, Any, Dict, Union

import rasa_core
from rasa_core import training, constants
from rasa_core.channels import UserMessage, OutputChannel, InputChannel
from rasa_core.dispatcher import Dispatcher
from rasa_core.domain import Domain, check_domain_sanity
from rasa_core.interpreter import NaturalLanguageInterpreter
from rasa_core.nlg import NaturalLanguageGenerator
from rasa_core.policies import Policy
from rasa_core.policies.ensemble import SimplePolicyEnsemble, PolicyEnsemble
from rasa_core.policies.memoization import MemoizationPolicy
from rasa_core.processor import MessageProcessor
from rasa_core.tracker_store import InMemoryTrackerStore, TrackerStore
from rasa_core.trackers import DialogueStateTracker
from rasa_core.utils import EndpointConfig
from rasa_nlu.utils import is_url

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa_core.nlg import NaturalLanguageGenerator as NLG

if six.PY2:
    from StringIO import StringIO as IOReader
else:
    from io import BytesIO as IOReader


def load_from_server(interpreter=None,  # type: NaturalLanguageInterpreter
                     generator=None,  # type: Union[EndpointConfig, NLG]
                     tracker_store=None,  # type: Optional[TrackerStore]
                     action_endpoint=None,  # type: Optional[EndpointConfig]
                     model_server=None,  # type: Optional[EndpointConfig]
                     wait_time_between_pulls=None,  # type: Optional[int]
                     ):
    # type: (...) -> Agent
    """Load a persisted model from a server."""

    agent = Agent(interpreter=interpreter,
                  generator=generator,
                  tracker_store=tracker_store,
                  action_endpoint=action_endpoint)

    try:
        _update_model_from_server(model_server, agent)
    except RequestException as e:
        logger.warn("Could not retrieve model from server. Connection Error.  "
                    "Running without a model. Server URL: `{}`\n".format(e))

    if wait_time_between_pulls:
        # continuously pull the model every `wait_time_between_pulls` seconds
        start_model_pulling_in_worker(model_server,
                                      wait_time_between_pulls,
                                      agent)
    return agent


def _init_model_from_server(model_server):
    # type: (EndpointConfig) -> Optional[(Domain, PolicyEnsemble, Text)]
    """Initialise a Rasa Core model from a URL."""

    if not is_url(model_server.url):
        raise InvalidURL(model_server.url)

    model_directory = tempfile.mkdtemp()

    fingerprint = _pull_model_and_fingerprint(model_server,
                                              model_directory,
                                              fingerprint=None)

    return fingerprint, model_directory


def _update_model_from_server(
        model_server,  # type: EndpointConfig
        agent,  # type: Agent
):
    # type: (...) -> None
    """Load a zipped Rasa Core model from a URL and update the passed agent."""

    if not is_url(model_server.url):
        raise InvalidURL(model_server.url)

    model_directory = tempfile.mkdtemp()

    new_model_fingerprint = _pull_model_and_fingerprint(
            model_server, model_directory, agent.fingerprint)
    if new_model_fingerprint:
        domain_path = os.path.join(os.path.abspath(model_directory),
                                   "domain.yml")
        domain = Domain.load(domain_path)
        policy_ensemble = PolicyEnsemble.load(model_directory)
        agent.update_model(domain, policy_ensemble, new_model_fingerprint)
    else:
        logger.debug("No new model found at "
                     "URL {}".format(model_server.url))


def _pull_model_and_fingerprint(model_server, model_directory, fingerprint):
    # type: (EndpointConfig, Text, Optional[Text]) -> Optional[Text]
    """Queries the model server and returns the value of the response's

    <ETag> header which contains the model hash."""
    header = {"If-None-Match": fingerprint}
    try:
        response = model_server.request(method="GET", headers=header)
    except RequestException as e:
        logger.warn("Tried to fetch model from server, but couldn't reach "
                    "server. We'll retry later... Error: {}."
                    "".format(e))
        return None

{'fraction_non_alphanumeric': 0.0673290349857309, 'fraction_numerical': 0.00021139414438220062, 'mean_word_length': 3.031529612270984, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1311263', 'n_tokens_mistral': 2614, 'n_tokens_neox': 2528, 'n_words': 703}