<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import json
import logging
import os
import random
import re
import time

from watsononlinestore.tests.fake_discovery import FAKE_DISCOVERY

logging.basicConfig(level=logging.DEBUG)
LOG = logging.getLogger(__name__)

# Limit the result count when calling Discovery query.
DISCOVERY_QUERY_COUNT = 10
# Limit more when formatting and filtering out "weak" results.
# Also useful for allowing us to log more results for dev/test even
# though we return fewer to the client.
DISCOVERY_KEEP_COUNT = 5
# Truncate the Discovery 'text'. It can be a lot. We'll add "..." if truncated.
DISCOVERY_TRUNCATE = 500
# Available data sources for Discovery
DISCOVERY_AMAZON_STORE = "AMAZON"
DISCOVERY_IBM_STORE = "IBM_STORE"


class SlackSender:

    def __init__(self, slack_client, channel):
        self.slack_client = slack_client
        self.channel = channel

    def send_message(self, message):
        """Sends message via Slack API.

        :param str message: The message to be sent to slack
        """
        self.slack_client.api_call("chat.postMessage",
                                   channel=self.channel,
                                   text=message,
                                   as_user=True)

    def get_user_json(self, user_id):
        """Get slack user information from user_id.

        :param str user_id: user ID to look up
        """
        try:
            # Get the authenticated user profile from Slack
            user_json = self.slack_client.api_call("users.info", user=user_id)
        except Exception as e:
            LOG.exception("Slack client call exception:")
            user_json = {'error': repr(e)}

        return user_json


class OnlineStoreCustomer:
    def __init__(self, email=None, first_name=None, last_name=None,
                 shopping_cart=None):

        self.email = email
        self.first_name = first_name
        self.last_name = last_name
        self.shopping_cart = shopping_cart

    def get_customer_dict(self):
        """Returns a dict in form usable by our cloudant_online_store DB

        :returns: customer dict of customer data for noSQL doc
        :rtype: dict
        """
        customer = {
            'type': 'customer',
            'email': self.email,
            'first_name': self.first_name,
            'last_name': self.last_name,
            'shopping_cart': self.shopping_cart
        }
        return customer


class WatsonOnlineStore:
    def __init__(self, bot_id, slack_client,
                 assistant_client, discovery_client,
                 cloudant_online_store):

        # specific for Slack as UI
        self.slack_client = slack_client
        self.bot_id = bot_id or 'unknown bot_id'
        self.at_bot = "<@" + self.bot_id + ">"

        # IBM Watson Assistant
        self.assistant_client = assistant_client
        self.workspace_id = self.setup_assistant_workspace(
            assistant_client, os.environ)

        # IBM Cloudant noSQL database
        self.cloudant_online_store = cloudant_online_store

        # IBM Watson Discovery Service
        self.discovery_client = discovery_client
        self.discovery_data_source = os.environ.get(
            'DISCOVERY_DATA_SOURCE', DISCOVERY_IBM_STORE)
        try:
            self.discovery_score_filter = float(
                os.environ.get(self.discovery_data_source +
                               '_DISCO_SCORE_FILTER', 0))
        except ValueError:
            LOG.debug(self.discovery_data_source +
                      "_DISCO_SCORE_FILTER must " +
                      "be a number between 0.0 and 1.0. " +
                      "Using default value of 0.0")
            self.discovery_score_filter = 0.0
            pass

        self.discovery_environment_id, self.discovery_collection_id = (
            self.setup_discovery_collection(discovery_client,
                                            self.discovery_data_source,
                                            os.environ))

        self.context = {}
        self.customer = None
        self.response_tuple = None
        self.delay = 0.5  # second
        response = self.assistant_client.message(
            workspace_id=self.workspace_id,
            context=None).get_result()
        self.context = self.context_merge(self.context, response['context'])

    @staticmethod
    def setup_assistant_workspace(assistant_client, environ):
        """Verify and/or initialize the Assistant workspace.

        If a WORKSPACE_ID is specified in the runtime environment,
        make sure that workspace exists. If no WORKSTATION_ID is
        specified then try to find it using a lookup by name.
        Name will be 'watson-online-store' unless overridden
        using the WORKSPACE_NAME environment variable.

        If a workspace is not found by ID or name, then try to
        create one from the JSON in data/workspace.json. Use the
        name as mentioned above so future lookup will find what
        was created.

        :param assistant_client: Assistant service client
        :param object environ: runtime environment variables
        :return: ID of Assistant workspace to use
        :rtype: str
        :raise Exception: When workspace is not found and cannot be created
        """

        # Get the actual workspaces
        workspaces = assistant_client.list_workspaces().get_result()[
            'workspaces']

        env_workspace_id = environ.get('WORKSPACE_ID')
        if env_workspace_id:
            # Optionally, we have an env var to give us a WORKSPACE_ID.
            # If one was set in the env, require that it can be found.
            LOG.debug("Using WORKSPACE_ID=%s" % env_workspace_id)
            for workspace in workspaces:
                if workspace['workspace_id'] == env_workspace_id:
                    ret = env_workspace_id
                    break
            else:
                raise Exception("WORKSPACE_ID=%s is specified in a runtime "
                                "environment variable, but that workspace "
                                "does not exist." % env_workspace_id)
        else:
            # Find it by name. We may have already created it.
            name = environ.get('WORKSPACE_NAME', 'watson-online-store')
            for workspace in workspaces:
                if workspace['name'] == name:
                    ret = workspace['workspace_id']
                    LOG.debug("Found WORKSPACE_ID=%(id)s using lookup by "
                              "name=%(name)s" % {'id': ret, 'name': name})
                    break
            else:
                # Not found, so create it.
                LOG.debug("Creating workspace from data/workspace.json...")
                workspace = WatsonOnlineStore.get_workspace_json()
                created = assistant_client.create_workspace(
                    name,
                    "Assistant workspace created by watson-online-store.",
                    workspace['language'],
                    intents=workspace['intents'],
                    entities=workspace['entities'],
                    dialog_nodes=workspace['dialog_nodes'],
                    counterexamples=workspace['counterexamples'],
                    metadata=workspace['metadata']).get_result()
                ret = created['workspace_id']
                LOG.debug("Created WORKSPACE_ID=%(id)s with "
                          "name=%(name)s" % {'id': ret, 'name': name})
        return ret

    @staticmethod
    def setup_discovery_collection(discovery_client,
                                   data_source,
                                   environ):
        """ Ensure that the collection exists in the Watson Discovery service.

        :param discovery_client: discovery service client
        :param str data_source: name of the discovery data source
        :param object environ: runtime environment variables
        :return: ID of discovery environment and collection to use
        :rtype: str
        :raise Exception: When collection is not found and cannot be created
        """

        # If environment id exists, ensure it is valid.
        environment_id = environ.get('DISCOVERY_ENVIRONMENT_ID')
        if environment_id:
            try:
                LOG.debug("Using DISCOVERY_ENVIRONMENT_ID=%s" % environment_id)
                discovery_client.get_environment(environment_id)
            except Exception as e:
                print(e)
                raise Exception("Environment with DISCOVERY_ENVIRONMENT_ID=%s "
                                "not found." % environment_id)
        else:
            # Try to find the environment by name.
            name = environ.get('DISCOVERY_ENVIRONMENT_NAME',
                               'watson-online-store')

            environments = discovery_client.list_environments().get_result()

            for environment in environments['environments']:
                if environment['name'] == name:
                    environment_id = environment['environment_id']
                    LOG.debug("Found DISCOVERY_ENVIRONMENT_ID=%(id)s using "
                              "lookup by name=%(name)s" %
                              {'id': environment_id, 'name': name})
                    break
                elif not environment['read_only']:
                    # Last resort will be to use an available one, but
                    # cannot use/modify a read-only environment.
                    environment_id = environment['environment_id']

            if not environment_id:
                # No existing environment found, so create it.
                # NOTE that the number of environments that can be created
                # under a trial Bluemix account is limited to one environment
                # per organization.
                try:
                    LOG.debug("Creating discovery environment...")
                    created = discovery_client.create_environment(
                        name,
                        "Discovery environment created by "
                        "watson-online-store.").get_result()
                    environment_id = created['environment_id']
                    LOG.debug("Created DISCOVERY_ENVIRONMENT_ID=%(id)s with "
                              "name=%(name)s" %
                              {'id': environment_id, 'name': name})
                except Exception as e:
                    raise Exception("Error creating Discovery "
                                    "Error: %s" % repr(e))

        # Determine if collection exists.
        collection_id = environ.get('DISCOVERY_COLLECTION_ID')
        if collection_id:
            try:
                LOG.debug("Using DISCOVERY_COLLECTION_ID=%s" % collection_id)
                discovery_client.get_collection(environment_id,
                                                collection_id)
                return environment_id, collection_id
            except Exception:
                raise Exception("Collection with DISCOVERY_COLLECTION_ID=%s "
                                "does not exist." % collection_id)
        else:
            # Try to find collection by name. Search all collections
            # that exist in the discovery environment.

            # Discovery collection names
            amazon_collection_name = "amazon-shopping"
            ibm_collection_name = "ibm-logo-store"
            # File path location to discovery files
            amazon_data_path = "data/amazon_data_html/"
            ibm_data_path = "data/ibm_store/"

            collections = discovery_client.list_collections(
                environment_id).get_result()['collections']
            for coll in collections:
                if ((data_source == DISCOVERY_AMAZON_STORE and
                        coll['name'] == amazon_collection_name) or
                        (data_source == DISCOVERY_IBM_STORE and
                            coll['name'] == ibm_collection_name)):
                    return environment_id, coll['collection_id']

        # Doesn't exist, so create it.
        LOG.debug("Creating collection from data files...")
        try:
            if data_source == DISCOVERY_AMAZON_STORE:
                name = amazon_collection_name
                path = amazon_data_path
            elif data_source == DISCOVERY_IBM_STORE:
                name = ibm_collection_name
                path = ibm_data_path
            if name:
                collection = discovery_client.create_collection(
                    environment_id,
                    name).get_result()

                # Add documents to collection
                if collection:
                    collection_id = collection['collection_id']
                    for _, _, files in os.walk(path):
                        for fname in files:
                            if fname.endswith('.html') or \
                                    fname.endswith('.json'):
                                with open(os.path.join(path, fname), 'r') as f:
                                    data = f.read()
                                discovery_client.add_document(environment_id,
                                                              collection_id,
                                                              file=data,
                                                              filename=fname)

        except Exception as e:
            raise Exception("Discovery Collection could not be created. "
                            "Error: %s" % repr(e))

        if not collection_id:
            raise Exception("Discovery Collection could not be found "
                            "or created.")

        return environment_id, collection_id

    @staticmethod
    def get_workspace_json():
        with open('data/workspace.json') as workspace_file:
            workspace = json.load(workspace_file)
        return workspace

    def context_merge(self, dict1, dict2):
        """Combine 2 dicts into one for Watson Assistant context.

        Common data in dict2 will override data in dict1

        :param dict dict1: original context dictionary
        :param dict dict2: new context dictionary - will override fields
        :returns: new_dict for context
        :rtype: dict
        """
        new_dict = dict1.copy()
        if dict2:
            new_dict.update(dict2)

        return new_dict

    def parse_slack_output(self, output_dict):
        """Prepare output when using Slack as UI.

        :param dict output_dict: text, channel, user, etc from slack posting
        :returns: text, channel, user
        :rtype: str, str, str
        """
        if output_dict and len(output_dict) > 0:
            for output in output_dict:
                if output and 'text' in output and 'user' in output and (
                        'user_profile' not in output):
                    if self.at_bot in output['text']:
                        return (
                            ''.join(output['text'].split(self.at_bot
                                                         )).strip().lower(),
                            output['channel'],
                            output['user'])
                    elif (output['channel'].startswith('D') and
                          output['user'] != self.bot_id):
                        # Direct message!
                        return (output['text'].strip().lower(),
                                output['channel'],
                                output['user'])
        return None, None, None

    def add_customer_to_context(self):
        """Send Customer info to Watson using context.

        The customer data from the UI is in the Cloudant DB, or has
        been added. Now add it to the context and pass back to Watson.
        """
        self.context = self.context_merge(self.context,
                                          self.customer.get_customer_dict())

    def customer_from_db(self, user_data):
        """Set the customer using data from Cloudant DB.

        We have the Customer in the Cloudant DB. Create a Customer object from
        this data and set for this instance of WatsonOnlineStore

        :param dict user_data: email, first_name, and last_name
        """

        email_addr = user_data['email']
        first = user_data['first_name']
        last = user_data['last_name']
        self.customer = OnlineStoreCustomer(email=email_addr,
                                            first_name=first,
                                            last_name=last,
                                            shopping_cart=[])

    def create_user_from_ui(self, user_json):
        """Set the customer using data from Slack.

        Authenticated user in slack will have email, First, and Last
        names. Create a user in the DB for this. Note that a different
        UI will require different code here.
        json info in ['user']['profile']

        :param dict user_json: email, first_name, and last_name
        """

        email_addr = user_json['user']['profile']['email']
        try:
            first = user_json['user']['profile']['first_name']
        except KeyError:
            first = email_addr
        try:
            last = user_json['user']['profile']['last_name']
        except KeyError:
            last = email_addr
        self.customer = OnlineStoreCustomer(email=email_addr,
                                            first_name=first,
                                            last_name=last,
                                            shopping_cart=[])

    def init_customer(self, sender, user_id):
        """Get user from DB, or create entry for user.

        :param object sender: Client-specific implementation
        :param str user_id: User ID
        """
        assert user_id

        user_json = sender.get_user_json(user_id)

        # Not found returns json with error.
        LOG.debug("user_from_slack:\n{}\n".format(user_json))

        if user_json and 'user' in user_json:
            cust = user_json['user'].get('profile', {}).get('email')
            if cust:
                user_data = self.cloudant_online_store.find_customer(cust)
                if user_data:
                    # We found this Slack user in our Cloudant DB
                    LOG.debug("user_from_DB\n{}\n".format(user_data))
                    self.customer_from_db(user_data)
                else:
                    # Didn't find Slack user in DB, so add them
                    self.create_user_from_ui(user_json)
                    self.cloudant_online_store.add_customer_obj(self.customer)

            if self.customer:
                # Now Watson will have customer info
                self.add_customer_to_context()

    def get_fake_discovery_response(self):
        """Returns fake response from IBM Discovery for testing purposes.

        :returns: list of Urls
        :rtype: list
        """
        index = random.randint(0, len(FAKE_DISCOVERY)-1)
        ret_string = {'discovery_result': FAKE_DISCOVERY[index]}
        return ret_string

    def handle_discovery_query(self):
        """Take query string from Watson Context and send to Discovery.

        Discovery response will be merged into context in order to allow it to
        be returned to Watson. In the case where there is no discovery client,
        a fake response will be returned, for testing purposes.

        :returns: False indicating no need for UI input, just return to Watson
        :rtype: Bool
        """
        query_string = self.context['discovery_string']
        if self.discovery_client:
            try:
                response = self.get_discovery_response(query_string)
            except Exception as e:
                response = {'discovery_result': repr(e)}
        else:
            response = self.get_fake_discovery_response()

        self.context = self.context_merge(self.context, response)
        LOG.debug("watson_discovery:\n{}\ncontext:\n{}".format(response,
                                                               self.context))

        # no need for user input, return to Watson Dialogue
        return False

    def get_watson_response(self, message):
        """Sends text and context to Watson and gets reply.

        Message input is text, self.context is also added and sent to Watson.

        :param str message: text to send to Watson
        :returns: json dict from Watson
        :rtype: dict
        """
        response = self.assistant_client.message(
            workspace_id=self.workspace_id,
            input={'text': message},
            context=self.context)
        return response

    @staticmethod
    def format_discovery_response(response, data_source):
        """Format data for Slack based on discovery data source.

        This method handles the different data source data and formats
        it specifically for Slack.

        The following functions are specific to the data source that has
        been fed into the Watson Discovery service. This example has two
        data sources to choose from: "IBM_STORE" and "AMAZON'. Which data
        source is being used is specified in the ".env" file by setting
        the following key values:

        <data_source>_DISCO_COLLECTION_ID=<collection id of data source>
        <data_source>_DISCO_SCORE_FILTER=<float value betweem 0.0. and 1.0>
        DISCOVERY_DATA_SOURCE="<data source string name>"
<commit_msg>Fix Issue with bad input.<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import json
import logging
import os
import random
import re
import time

from watsononlinestore.tests.fake_discovery import FAKE_DISCOVERY

logging.basicConfig(level=logging.DEBUG)
LOG = logging.getLogger(__name__)

# Limit the result count when calling Discovery query.
DISCOVERY_QUERY_COUNT = 10
# Limit more when formatting and filtering out "weak" results.
# Also useful for allowing us to log more results for dev/test even
# though we return fewer to the client.
DISCOVERY_KEEP_COUNT = 5
# Truncate the Discovery 'text'. It can be a lot. We'll add "..." if truncated.
DISCOVERY_TRUNCATE = 500
# Available data sources for Discovery
DISCOVERY_AMAZON_STORE = "AMAZON"
DISCOVERY_IBM_STORE = "IBM_STORE"


class SlackSender:

    def __init__(self, slack_client, channel):
        self.slack_client = slack_client
        self.channel = channel

    def send_message(self, message):
        """Sends message via Slack API.

        :param str message: The message to be sent to slack
        """
        self.slack_client.api_call("chat.postMessage",
                                   channel=self.channel,
                                   text=message,
                                   as_user=True)

    def get_user_json(self, user_id):
        """Get slack user information from user_id.

        :param str user_id: user ID to look up
        """
        try:
            # Get the authenticated user profile from Slack
            user_json = self.slack_client.api_call("users.info", user=user_id)
        except Exception as e:
            LOG.exception("Slack client call exception:")
            user_json = {'error': repr(e)}

        return user_json


class OnlineStoreCustomer:
    def __init__(self, email=None, first_name=None, last_name=None,
                 shopping_cart=None):

        self.email = email
        self.first_name = first_name
        self.last_name = last_name
        self.shopping_cart = shopping_cart

    def get_customer_dict(self):
        """Returns a dict in form usable by our cloudant_online_store DB

        :returns: customer dict of customer data for noSQL doc
        :rtype: dict
        """
        customer = {
            'type': 'customer',
            'email': self.email,
            'first_name': self.first_name,
            'last_name': self.last_name,
            'shopping_cart': self.shopping_cart
        }
        return customer


class WatsonOnlineStore:
    def __init__(self, bot_id, slack_client,
                 assistant_client, discovery_client,
                 cloudant_online_store):

        # specific for Slack as UI
        self.slack_client = slack_client
        self.bot_id = bot_id or 'unknown bot_id'
        self.at_bot = "<@" + self.bot_id + ">"

        # IBM Watson Assistant
        self.assistant_client = assistant_client
        self.workspace_id = self.setup_assistant_workspace(
            assistant_client, os.environ)

        # IBM Cloudant noSQL database
        self.cloudant_online_store = cloudant_online_store

        # IBM Watson Discovery Service
        self.discovery_client = discovery_client
        self.discovery_data_source = os.environ.get(
            'DISCOVERY_DATA_SOURCE', DISCOVERY_IBM_STORE)
        try:
            self.discovery_score_filter = float(
                os.environ.get(self.discovery_data_source +
                               '_DISCO_SCORE_FILTER', 0))
        except ValueError:
            LOG.debug(self.discovery_data_source +
                      "_DISCO_SCORE_FILTER must " +
                      "be a number between 0.0 and 1.0. " +
                      "Using default value of 0.0")
            self.discovery_score_filter = 0.0
            pass

        self.discovery_environment_id, self.discovery_collection_id = (
            self.setup_discovery_collection(discovery_client,
                                            self.discovery_data_source,
                                            os.environ))

        self.context = {}
        self.customer = None
        self.response_tuple = None
        self.delay = 0.5  # second

    @staticmethod
    def setup_assistant_workspace(assistant_client, environ):
        """Verify and/or initialize the Assistant workspace.

        If a WORKSPACE_ID is specified in the runtime environment,
        make sure that workspace exists. If no WORKSTATION_ID is
        specified then try to find it using a lookup by name.
        Name will be 'watson-online-store' unless overridden
        using the WORKSPACE_NAME environment variable.

        If a workspace is not found by ID or name, then try to
        create one from the JSON in data/workspace.json. Use the
        name as mentioned above so future lookup will find what
        was created.

        :param assistant_client: Assistant service client
        :param object environ: runtime environment variables
        :return: ID of Assistant workspace to use
        :rtype: str
        :raise Exception: When workspace is not found and cannot be created
        """

        # Get the actual workspaces
        workspaces = assistant_client.list_workspaces().get_result()[
            'workspaces']

        env_workspace_id = environ.get('WORKSPACE_ID')
        if env_workspace_id:
            # Optionally, we have an env var to give us a WORKSPACE_ID.
            # If one was set in the env, require that it can be found.
            LOG.debug("Using WORKSPACE_ID=%s" % env_workspace_id)
            for workspace in workspaces:
                if workspace['workspace_id'] == env_workspace_id:
                    ret = env_workspace_id
                    break
            else:
                raise Exception("WORKSPACE_ID=%s is specified in a runtime "
                                "environment variable, but that workspace "
                                "does not exist." % env_workspace_id)
        else:
            # Find it by name. We may have already created it.
            name = environ.get('WORKSPACE_NAME', 'watson-online-store')
            for workspace in workspaces:
                if workspace['name'] == name:
                    ret = workspace['workspace_id']
                    LOG.debug("Found WORKSPACE_ID=%(id)s using lookup by "
                              "name=%(name)s" % {'id': ret, 'name': name})
                    break
            else:
                # Not found, so create it.
                LOG.debug("Creating workspace from data/workspace.json...")
                workspace = WatsonOnlineStore.get_workspace_json()
                created = assistant_client.create_workspace(
                    name,
                    "Assistant workspace created by watson-online-store.",
                    workspace['language'],
                    intents=workspace['intents'],
                    entities=workspace['entities'],
                    dialog_nodes=workspace['dialog_nodes'],
                    counterexamples=workspace['counterexamples'],
                    metadata=workspace['metadata']).get_result()
                ret = created['workspace_id']
                LOG.debug("Created WORKSPACE_ID=%(id)s with "
                          "name=%(name)s" % {'id': ret, 'name': name})
        return ret

    @staticmethod
    def setup_discovery_collection(discovery_client,
                                   data_source,
                                   environ):
        """ Ensure that the collection exists in the Watson Discovery service.

        :param discovery_client: discovery service client
        :param str data_source: name of the discovery data source
        :param object environ: runtime environment variables
        :return: ID of discovery environment and collection to use
        :rtype: str
        :raise Exception: When collection is not found and cannot be created
        """

        # If environment id exists, ensure it is valid.
        environment_id = environ.get('DISCOVERY_ENVIRONMENT_ID')
        if environment_id:
            try:
                LOG.debug("Using DISCOVERY_ENVIRONMENT_ID=%s" % environment_id)
                discovery_client.get_environment(environment_id)
            except Exception as e:
                print(e)
                raise Exception("Environment with DISCOVERY_ENVIRONMENT_ID=%s "
                                "not found." % environment_id)
        else:
            # Try to find the environment by name.
            name = environ.get('DISCOVERY_ENVIRONMENT_NAME',
                               'watson-online-store')

            environments = discovery_client.list_environments().get_result()

            for environment in environments['environments']:
                if environment['name'] == name:
                    environment_id = environment['environment_id']
                    LOG.debug("Found DISCOVERY_ENVIRONMENT_ID=%(id)s using "
                              "lookup by name=%(name)s" %
                              {'id': environment_id, 'name': name})
                    break
                elif not environment['read_only']:
                    # Last resort will be to use an available one, but
                    # cannot use/modify a read-only environment.
                    environment_id = environment['environment_id']

            if not environment_id:
                # No existing environment found, so create it.
                # NOTE that the number of environments that can be created
                # under a trial Bluemix account is limited to one environment
                # per organization.
                try:
                    LOG.debug("Creating discovery environment...")
                    created = discovery_client.create_environment(
                        name,
                        "Discovery environment created by "
                        "watson-online-store.").get_result()
                    environment_id = created['environment_id']
                    LOG.debug("Created DISCOVERY_ENVIRONMENT_ID=%(id)s with "
                              "name=%(name)s" %
                              {'id': environment_id, 'name': name})
                except Exception as e:
                    raise Exception("Error creating Discovery "
                                    "Error: %s" % repr(e))

        # Determine if collection exists.
        collection_id = environ.get('DISCOVERY_COLLECTION_ID')
        if collection_id:
            try:
                LOG.debug("Using DISCOVERY_COLLECTION_ID=%s" % collection_id)
                discovery_client.get_collection(environment_id,
                                                collection_id)
                return environment_id, collection_id
            except Exception:
                raise Exception("Collection with DISCOVERY_COLLECTION_ID=%s "
                                "does not exist." % collection_id)
        else:
            # Try to find collection by name. Search all collections
            # that exist in the discovery environment.

            # Discovery collection names
            amazon_collection_name = "amazon-shopping"
            ibm_collection_name = "ibm-logo-store"
            # File path location to discovery files
            amazon_data_path = "data/amazon_data_html/"
            ibm_data_path = "data/ibm_store/"

            collections = discovery_client.list_collections(
                environment_id).get_result()['collections']
            for coll in collections:
                if ((data_source == DISCOVERY_AMAZON_STORE and
                        coll['name'] == amazon_collection_name) or
                        (data_source == DISCOVERY_IBM_STORE and
                            coll['name'] == ibm_collection_name)):
                    return environment_id, coll['collection_id']

        # Doesn't exist, so create it.
        LOG.debug("Creating collection from data files...")
        try:
            if data_source == DISCOVERY_AMAZON_STORE:
                name = amazon_collection_name
                path = amazon_data_path
            elif data_source == DISCOVERY_IBM_STORE:
                name = ibm_collection_name
                path = ibm_data_path
            if name:
                collection = discovery_client.create_collection(
                    environment_id,
                    name).get_result()

                # Add documents to collection
                if collection:
                    collection_id = collection['collection_id']
                    for _, _, files in os.walk(path):
                        for fname in files:
                            if fname.endswith('.html') or \
                                    fname.endswith('.json'):
                                with open(os.path.join(path, fname), 'r') as f:
                                    data = f.read()
                                discovery_client.add_document(environment_id,
                                                              collection_id,
                                                              file=data,
                                                              filename=fname)

        except Exception as e:
            raise Exception("Discovery Collection could not be created. "
                            "Error: %s" % repr(e))

        if not collection_id:
            raise Exception("Discovery Collection could not be found "
                            "or created.")

        return environment_id, collection_id

    @staticmethod
    def get_workspace_json():
        with open('data/workspace.json') as workspace_file:
            workspace = json.load(workspace_file)
        return workspace

    def context_merge(self, dict1, dict2):
        """Combine 2 dicts into one for Watson Assistant context.

        Common data in dict2 will override data in dict1

        :param dict dict1: original context dictionary
        :param dict dict2: new context dictionary - will override fields
        :returns: new_dict for context
        :rtype: dict
        """
        new_dict = dict1.copy()
        if dict2:
            new_dict.update(dict2)

        return new_dict

    def parse_slack_output(self, output_dict):
        """Prepare output when using Slack as UI.

        :param dict output_dict: text, channel, user, etc from slack posting
        :returns: text, channel, user
        :rtype: str, str, str
        """
        if output_dict and len(output_dict) > 0:
            for output in output_dict:
                if (output and 'text' in output and 'user' in output and
                   ('bot_id' not in output) and
                   ('user_profile' not in output)):
                    if self.at_bot in output['text']:
                        return (
                            ''.join(output['text'].split(self.at_bot
                                                         )).strip().lower(),
                            output['channel'],
                            output['user'])
                    elif (output['channel'].startswith('D') and
                          output['user'] != self.bot_id):
                        # Direct message!
                        return (output['text'].strip().lower(),
                                output['channel'],
                                output['user'])
        return None, None, None

    def add_customer_to_context(self):
        """Send Customer info to Watson using context.

        The customer data from the UI is in the Cloudant DB, or has
        been added. Now add it to the context and pass back to Watson.
        """
        self.context = self.context_merge(self.context,
                                          self.customer.get_customer_dict())

    def customer_from_db(self, user_data):
        """Set the customer using data from Cloudant DB.

        We have the Customer in the Cloudant DB. Create a Customer object from
        this data and set for this instance of WatsonOnlineStore

        :param dict user_data: email, first_name, and last_name
        """

        email_addr = user_data['email']
        first = user_data['first_name']
        last = user_data['last_name']
        self.customer = OnlineStoreCustomer(email=email_addr,
                                            first_name=first,
                                            last_name=last,
                                            shopping_cart=[])

    def create_user_from_ui(self, user_json):
        """Set the customer using data from Slack.

        Authenticated user in slack will have email, First, and Last
        names. Create a user in the DB for this. Note that a different
        UI will require different code here.
        json info in ['user']['profile']

        :param dict user_json: email, first_name, and last_name
        """

        email_addr = user_json['user']['profile']['email']
        try:
            first = user_json['user']['profile']['first_name']
        except KeyError:
            first = email_addr
        try:
            last = user_json['user']['profile']['last_name']
        except KeyError:
            last = email_addr
        self.customer = OnlineStoreCustomer(email=email_addr,
                                            first_name=first,
                                            last_name=last,
                                            shopping_cart=[])

    def init_customer(self, sender, user_id):
        """Get user from DB, or create entry for user.

        :param object sender: Client-specific implementation
        :param str user_id: User ID
        """
        assert user_id

        user_json = sender.get_user_json(user_id)

        # Not found returns json with error.
        LOG.debug("user_from_slack:\n{}\n".format(user_json))

        if user_json and 'user' in user_json:
            cust = user_json['user'].get('profile', {}).get('email')
            if cust:
                user_data = self.cloudant_online_store.find_customer(cust)
                if user_data:
                    # We found this Slack user in our Cloudant DB
                    LOG.debug("user_from_DB\n{}\n".format(user_data))
                    self.customer_from_db(user_data)
                else:
                    # Didn't find Slack user in DB, so add them
                    self.create_user_from_ui(user_json)
                    self.cloudant_online_store.add_customer_obj(self.customer)

            if self.customer:
                # Now Watson will have customer info
                self.add_customer_to_context()

    def get_fake_discovery_response(self):
        """Returns fake response from IBM Discovery for testing purposes.

        :returns: list of Urls
        :rtype: list
        """
        index = random.randint(0, len(FAKE_DISCOVERY)-1)
        ret_string = {'discovery_result': FAKE_DISCOVERY[index]}
        return ret_string

    def handle_discovery_query(self):
        """Take query string from Watson Context and send to Discovery.

        Discovery response will be merged into context in order to allow it to
        be returned to Watson. In the case where there is no discovery client,
        a fake response will be returned, for testing purposes.

        :returns: False indicating no need for UI input, just return to Watson
        :rtype: Bool
        """
        query_string = self.context['discovery_string']
        if self.discovery_client:
            try:
                response = self.get_discovery_response(query_string)
            except Exception as e:
                response = {'discovery_result': repr(e)}
        else:
            response = self.get_fake_discovery_response()

        self.context = self.context_merge(self.context, response)
        LOG.debug("watson_discovery:\n{}\ncontext:\n{}".format(response,
                                                               self.context))

        # no need for user input, return to Watson Dialogue
        return False

    def get_watson_response(self, message):
        """Sends text and context to Watson and gets reply.

        Message input is text, self.context is also added and sent to Watson.

        :param str message: text to send to Watson
        :returns: json dict from Watson
        :rtype: dict
        """
        response = self.assistant_client.message(
            workspace_id=self.workspace_id,
            input={'text': message},
            context=self.context)
        if self.context is None:
            LOG.debug("Initializing context")
            self.context = self.context_merge(self.context,
                                              response['context'])
        return response

    @staticmethod
    def format_discovery_response(response, data_source):
        """Format data for Slack based on discovery data source.

        This method handles the different data source data and formats
        it specifically for Slack.

        The following functions are specific to the data source that has
        been fed into the Watson Discovery service. This example has two
        data sources to choose from: "IBM_STORE" and "AMAZON'. Which data
        source is being used is specified in the ".env" file by setting
        the following key values:

        <data_source>_DISCO_COLLECTION_ID=<collection id of data source>
        <data_source>_DISCO_SCORE_FILTER=<float value betweem 0.0. and 1.0>
        DISCOVERY_DATA_SOURCE="<data source string name>"

{'fraction_non_alphanumeric': 0.07311984033384741, 'fraction_numerical': 0.0017236686927333757, 'mean_word_length': 1.704594246457707, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 80, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1775455', 'n_tokens_mistral': 11044, 'n_tokens_neox': 10547, 'n_words': 3307}