<commit_before>import os
from typing import List

from pymongo import MongoClient

from esm.models import Manifest
from esm.models import ServiceType
from esm.models import ServiceInstance
from esm.models import LastOperation

from adapters.log import LOG

# TODO standardise on return type of None or tuple for relevant store operations


class Store(object):
    def add_service(self, service: ServiceType) -> None:
        raise NotImplementedError

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        raise NotImplementedError

    def delete_service(self, service_id: str=None) -> None:
        raise NotImplementedError

    def add_service_instance(self, service_instance: ServiceInstance) -> None:
        raise NotImplementedError

    def get_service_instance(self, instance_id: str=None) -> List[ServiceInstance]:
        raise NotImplementedError

    def delete_service_instance(self, service_instance_id: str=None) -> None:
        raise NotImplementedError

    def add_manifest(self, manifest: Manifest) -> None:
        raise NotImplementedError

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:
        raise NotImplementedError

    def delete_manifest(self, manifest_id: str=None) -> None:
        raise NotImplementedError

    def add_last_operation(self, instance_id: str, last_operation: LastOperation) -> None:
        raise NotImplementedError

    def delete_last_operation(self, instance_id: str=None) -> None:
        raise NotImplementedError

    def get_last_operation(self, instance_id: str=None) -> List[LastOperation]:
        raise NotImplementedError


class MongoDBStore(Store):

    def __init__(self, host: str) -> None:
        _client = MongoClient(host, 27017)
        self.ESM_DB = _client.esm
        LOG.info('Using the MongoDBStore.')
        LOG.info('MongoDBStore is persistent.')

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        if service_id:
            if self.ESM_DB.services.count({'id': service_id}) == 1:
                return [
                    ServiceType.from_dict(
                        self.ESM_DB.services.find_one({'id': service_id})
                    )
                ]
            else:
                LOG.warn('Requested service type not found: {id}'.format(id=service_id))
                return []
        else:
            services = []

            for service in self.ESM_DB.services.find():
                services.append(
                    ServiceType().from_dict(service)
                )

            return services

    def add_service(self, service: ServiceType) -> tuple:
        if self.ESM_DB.services.count({'id': service.id}) == 0:
            result = self.ESM_DB.services.insert_one(service.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the supplied service type to the DB', 500
        else:
            return 'the service already exists in the catalog. you should resubmit with a different service.', 409

    def delete_service(self, service_id: str=None) -> None:
        if service_id:
            self.ESM_DB.services.delete_one({'id': service_id})
        else:
            self.ESM_DB.services.delete_many({})

    def add_manifest(self, manifest: Manifest) -> tuple:
        # ensure the manifest-defined service and plan id exist?
        # if self.ESM_DB.services.count({'id': manifest.service_id}) > 0:
        #     # get the plans
        #     plans = self.ESM_DB.services.find_one({'id': manifest.service_id})['plans']
        #     # filter the plans to find the plan to be associated with
        #     plan = [p for p in plans if p['id'] == manifest.plan_id]
        #     if len(plan) != 1:
        #         return 'no plan or duplicate plan found.', 401
        # else:
        #     return 'the service id in the supplied manifest does not exist.', 404

        if self.ESM_DB.manifests.count({'id': manifest.id}) == 0:
            result = self.ESM_DB.manifests.insert_one(manifest.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the supplied manifest to the DB', 500
        else:
            print('client side error - 4XX')
            return 'the manifest already exists in the catalog', 409

        return 'ok', 200

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:

        if manifest_id and plan_id:
            raise Exception('Query manifests only by manifest_id OR plan_id')

        if plan_id:
            if self.ESM_DB.manifests.count({'plan_id': plan_id}) == 1:
                return [
                    Manifest.from_dict(
                        self.ESM_DB.manifests.find_one({'plan_id': plan_id})
                    )
                ]
            else:
                LOG.warn('Requested manifest by plan ID not found: {id}'.format(id=plan_id))
                return []
        elif manifest_id:
            if self.ESM_DB.manifests.count({'id': manifest_id}) == 1:
                return [
                    Manifest.from_dict(
                        self.ESM_DB.manifests.find_one({'id': manifest_id})
                    )
                ]
            else:
                LOG.warn('Requested manifest not found: {id}'.format(id=manifest_id))
                return []
        else:
            manifests = []

            for manifest in self.ESM_DB.manifests.find():
                manifests.append(
                    Manifest().from_dict(manifest)
                )

            return manifests

    def delete_manifest(self, manifest_id: str=None) -> None:
        if manifest_id:
            self.ESM_DB.manifests.delete_one({'id': manifest_id})
        else:
            self.ESM_DB.manifests.delete_many({})

    def get_service_instance(self, instance_id: str=None) -> List[ServiceInstance]:
        if instance_id:
            if self.ESM_DB.instances.count({'context.id': instance_id}) == 1:
                return [
                    ServiceInstance.from_dict(
                        self.ESM_DB.instances.find_one({'context.id': instance_id})
                    )
                ]
            else:
                LOG.warn('Requested service instance not found: {id}'.format(id=instance_id))
                return []
        else:
            instances = []

            for inst in self.ESM_DB.instances.find():
                instances.append(
                    ServiceInstance().from_dict(inst)
                )

            return instances

    def add_service_instance(self, service_instance: ServiceInstance) -> tuple:
        result = self.ESM_DB.instances.count({'context.id': service_instance.context['id']})
        if result == 1:
            LOG.info('A duplicate service instance was attempted to be stored. '
                     'Updating the existing service instance {id}.'
                     'Content supplied:\n{content}'.format(id=service_instance.context['id'], content=service_instance.to_str()))
            # Update the service instance
            raw_svc_inst = self.ESM_DB.instances.find_one({'context.id': service_instance.context['id']})
            record_id = raw_svc_inst['_id']
            result = self.ESM_DB.instances.update_one({'_id': record_id}, {"$set": service_instance.to_dict()},
                                                      upsert=False)
            if not result.acknowledged:
                return 'there was an issue updating the service instance to the DB', 500
        else:
            result = self.ESM_DB.instances.insert_one(service_instance.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the service instance to the DB', 500

    def delete_service_instance(self, service_instance_id: str=None) -> None:
        if service_instance_id:
            self.ESM_DB.instances.delete_one({'context.id': service_instance_id})
        else:
            self.ESM_DB.instances.delete_many({})

    def add_last_operation(self, instance_id: str, last_operation: LastOperation) -> tuple:
        result = self.ESM_DB.last_operations.insert_one({'id': instance_id, 'last_op': last_operation.to_dict()})
        if not result.acknowledged:
            return 'there was an issue saving the service status to the DB', 500

    def get_last_operation(self, instance_id: str=None) -> List[LastOperation]:
        if instance_id:
            if self.ESM_DB.last_operations.count({'id': instance_id}) == 1:
                return [
                    LastOperation.from_dict(
                        self.ESM_DB.last_operations.find_one({'id': instance_id})
                    )
                ]
            else:
                LOG.warn('Requested last operation not found: {id}'.format(id=instance_id))
                return []
        else:
            last_ops = []

            for lo in self.ESM_DB.last_operations.find():
                last_ops.append(
                    LastOperation().from_dict(lo)
                )

            return last_ops

    def delete_last_operation(self, instance_id: str=None) -> None:
        if instance_id:
            self.ESM_DB.last_operations.delete_one({'id': instance_id})
        else:
            self.ESM_DB.last_operations.delete_many({})


class DotDict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__


class InMemoryStore(Store):

    def __init__(self) -> None:
        LOG.info('Using the InMemoryStore.')
        LOG.warn('InMemoryStore is not persistent.')
        self.ESM_DB = dict()
        self.ESM_DB['services'] = list()
        self.ESM_DB['instances'] = list()
        self.ESM_DB['manifests'] = list()
        self.ESM_DB['last_operations'] = list()
        self.ESM_DB = DotDict(self.ESM_DB)

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        if not service_id:
            LOG.info('Returning registered services. Count: '.format(count=len(self.ESM_DB.services)))
            return self.ESM_DB.services
        else:
            return [s for s in self.ESM_DB.services if s.id == service_id]

    def add_service(self, service: ServiceType) -> None:
        if service not in self.ESM_DB.services:
            self.ESM_DB.services.append(service)
            LOG.info('Adding a new service type to the catalog. '
                     'Content supplied: {content}'.format(content=service.to_str()))
        else:
            LOG.warn('A duplicate service type was attempted to be registered. '
                     'Ignoring the request. '
                     'Content supplied:\n{content}'.format(content=service.to_str()))

    def delete_service(self, service_id: str=None) -> None:
        if not service_id:
            LOG.warn('Deleting ALL registered service types in the catalog.')
            self.ESM_DB.services = list()
        else:
            service_to_delete = [s for s in self.ESM_DB.services if s.id == service_id][0]
            LOG.info('Deleting the service {id} from the catalog. Content:\n{content}'.format(
                id=service_id, content=service_to_delete.to_str()))
            # TODO this will fail if service_to_delete is None
            self.ESM_DB.services.remove(service_to_delete)

    def add_manifest(self, manifest: Manifest) -> None:
        if manifest not in self.ESM_DB.manifests:
            self.ESM_DB.manifests.append(manifest)
        else:
            LOG.warn('A duplicate manifest was attempted to be registered. '
                     'Ignoring the request. '
                     'Content supplied:\n{content}'.format(content=manifest.to_str()))

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:<commit_msg>temporary solution to error handling in add_manifest for InMemoryStore<commit_after>import os
from typing import List

from pymongo import MongoClient

from esm.models import Manifest
from esm.models import ServiceType
from esm.models import ServiceInstance
from esm.models import LastOperation

from adapters.log import LOG

# TODO standardise on return type of None or tuple for relevant store operations
# TODO implement exception handling


class Store(object):
    def add_service(self, service: ServiceType) -> None:
        raise NotImplementedError

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        raise NotImplementedError

    def delete_service(self, service_id: str=None) -> None:
        raise NotImplementedError

    def add_service_instance(self, service_instance: ServiceInstance) -> None:
        raise NotImplementedError

    def get_service_instance(self, instance_id: str=None) -> List[ServiceInstance]:
        raise NotImplementedError

    def delete_service_instance(self, service_instance_id: str=None) -> None:
        raise NotImplementedError

    def add_manifest(self, manifest: Manifest) -> None:
        raise NotImplementedError

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:
        raise NotImplementedError

    def delete_manifest(self, manifest_id: str=None) -> None:
        raise NotImplementedError

    def add_last_operation(self, instance_id: str, last_operation: LastOperation) -> None:
        raise NotImplementedError

    def delete_last_operation(self, instance_id: str=None) -> None:
        raise NotImplementedError

    def get_last_operation(self, instance_id: str=None) -> List[LastOperation]:
        raise NotImplementedError


class MongoDBStore(Store):

    def __init__(self, host: str) -> None:
        _client = MongoClient(host, 27017)
        self.ESM_DB = _client.esm
        LOG.info('Using the MongoDBStore.')
        LOG.info('MongoDBStore is persistent.')

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        if service_id:
            if self.ESM_DB.services.count({'id': service_id}) == 1:
                return [
                    ServiceType.from_dict(
                        self.ESM_DB.services.find_one({'id': service_id})
                    )
                ]
            else:
                LOG.warn('Requested service type not found: {id}'.format(id=service_id))
                return []
        else:
            services = []

            for service in self.ESM_DB.services.find():
                services.append(
                    ServiceType().from_dict(service)
                )

            return services

    def add_service(self, service: ServiceType) -> tuple:
        if self.ESM_DB.services.count({'id': service.id}) == 0:
            result = self.ESM_DB.services.insert_one(service.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the supplied service type to the DB', 500
        else:
            return 'the service already exists in the catalog. you should resubmit with a different service.', 409

    def delete_service(self, service_id: str=None) -> None:
        if service_id:
            self.ESM_DB.services.delete_one({'id': service_id})
        else:
            self.ESM_DB.services.delete_many({})

    def add_manifest(self, manifest: Manifest) -> tuple:
        # ensure the manifest-defined service and plan id exist?
        # if self.ESM_DB.services.count({'id': manifest.service_id}) > 0:
        #     # get the plans
        #     plans = self.ESM_DB.services.find_one({'id': manifest.service_id})['plans']
        #     # filter the plans to find the plan to be associated with
        #     plan = [p for p in plans if p['id'] == manifest.plan_id]
        #     if len(plan) != 1:
        #         return 'no plan or duplicate plan found.', 401
        # else:
        #     return 'the service id in the supplied manifest does not exist.', 404

        if self.ESM_DB.manifests.count({'id': manifest.id}) == 0:
            result = self.ESM_DB.manifests.insert_one(manifest.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the supplied manifest to the DB', 500
        else:
            print('client side error - 4XX')
            return 'the manifest already exists in the catalog', 409

        return 'ok', 200

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:

        if manifest_id and plan_id:
            raise Exception('Query manifests only by manifest_id OR plan_id')

        if plan_id:
            if self.ESM_DB.manifests.count({'plan_id': plan_id}) == 1:
                return [
                    Manifest.from_dict(
                        self.ESM_DB.manifests.find_one({'plan_id': plan_id})
                    )
                ]
            else:
                LOG.warn('Requested manifest by plan ID not found: {id}'.format(id=plan_id))
                return []
        elif manifest_id:
            if self.ESM_DB.manifests.count({'id': manifest_id}) == 1:
                return [
                    Manifest.from_dict(
                        self.ESM_DB.manifests.find_one({'id': manifest_id})
                    )
                ]
            else:
                LOG.warn('Requested manifest not found: {id}'.format(id=manifest_id))
                return []
        else:
            manifests = []

            for manifest in self.ESM_DB.manifests.find():
                manifests.append(
                    Manifest().from_dict(manifest)
                )

            return manifests

    def delete_manifest(self, manifest_id: str=None) -> None:
        if manifest_id:
            self.ESM_DB.manifests.delete_one({'id': manifest_id})
        else:
            self.ESM_DB.manifests.delete_many({})

    def get_service_instance(self, instance_id: str=None) -> List[ServiceInstance]:
        if instance_id:
            if self.ESM_DB.instances.count({'context.id': instance_id}) == 1:
                return [
                    ServiceInstance.from_dict(
                        self.ESM_DB.instances.find_one({'context.id': instance_id})
                    )
                ]
            else:
                LOG.warn('Requested service instance not found: {id}'.format(id=instance_id))
                return []
        else:
            instances = []

            for inst in self.ESM_DB.instances.find():
                instances.append(
                    ServiceInstance().from_dict(inst)
                )

            return instances

    def add_service_instance(self, service_instance: ServiceInstance) -> tuple:
        result = self.ESM_DB.instances.count({'context.id': service_instance.context['id']})
        if result == 1:
            LOG.info('A duplicate service instance was attempted to be stored. '
                     'Updating the existing service instance {id}.'
                     'Content supplied:\n{content}'.format(id=service_instance.context['id'], content=service_instance.to_str()))
            # Update the service instance
            raw_svc_inst = self.ESM_DB.instances.find_one({'context.id': service_instance.context['id']})
            record_id = raw_svc_inst['_id']
            result = self.ESM_DB.instances.update_one({'_id': record_id}, {"$set": service_instance.to_dict()},
                                                      upsert=False)
            if not result.acknowledged:
                return 'there was an issue updating the service instance to the DB', 500
        else:
            result = self.ESM_DB.instances.insert_one(service_instance.to_dict())
            if not result.acknowledged:
                return 'there was an issue saving the service instance to the DB', 500

    def delete_service_instance(self, service_instance_id: str=None) -> None:
        if service_instance_id:
            self.ESM_DB.instances.delete_one({'context.id': service_instance_id})
        else:
            self.ESM_DB.instances.delete_many({})

    def add_last_operation(self, instance_id: str, last_operation: LastOperation) -> tuple:
        result = self.ESM_DB.last_operations.insert_one({'id': instance_id, 'last_op': last_operation.to_dict()})
        if not result.acknowledged:
            return 'there was an issue saving the service status to the DB', 500

    def get_last_operation(self, instance_id: str=None) -> List[LastOperation]:
        if instance_id:
            if self.ESM_DB.last_operations.count({'id': instance_id}) == 1:
                return [
                    LastOperation.from_dict(
                        self.ESM_DB.last_operations.find_one({'id': instance_id})
                    )
                ]
            else:
                LOG.warn('Requested last operation not found: {id}'.format(id=instance_id))
                return []
        else:
            last_ops = []

            for lo in self.ESM_DB.last_operations.find():
                last_ops.append(
                    LastOperation().from_dict(lo)
                )

            return last_ops

    def delete_last_operation(self, instance_id: str=None) -> None:
        if instance_id:
            self.ESM_DB.last_operations.delete_one({'id': instance_id})
        else:
            self.ESM_DB.last_operations.delete_many({})


class DotDict(dict):
    """dot.notation access to dictionary attributes"""
    __getattr__ = dict.get
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__


class InMemoryStore(Store):

    def __init__(self) -> None:
        LOG.info('Using the InMemoryStore.')
        LOG.warn('InMemoryStore is not persistent.')
        self.ESM_DB = dict()
        self.ESM_DB['services'] = list()
        self.ESM_DB['instances'] = list()
        self.ESM_DB['manifests'] = list()
        self.ESM_DB['last_operations'] = list()
        self.ESM_DB = DotDict(self.ESM_DB)

    def get_service(self, service_id: str=None) -> List[ServiceType]:
        if not service_id:
            LOG.info('Returning registered services. Count: '.format(count=len(self.ESM_DB.services)))
            return self.ESM_DB.services
        else:
            return [s for s in self.ESM_DB.services if s.id == service_id]

    def add_service(self, service: ServiceType) -> None:
        if service not in self.ESM_DB.services:
            self.ESM_DB.services.append(service)
            LOG.info('Adding a new service type to the catalog. '
                     'Content supplied: {content}'.format(content=service.to_str()))
        else:
            LOG.warn('A duplicate service type was attempted to be registered. '
                     'Ignoring the request. '
                     'Content supplied:\n{content}'.format(content=service.to_str()))

    def delete_service(self, service_id: str=None) -> None:
        if not service_id:
            LOG.warn('Deleting ALL registered service types in the catalog.')
            self.ESM_DB.services = list()
        else:
            service_to_delete = [s for s in self.ESM_DB.services if s.id == service_id][0]
            LOG.info('Deleting the service {id} from the catalog. Content:\n{content}'.format(
                id=service_id, content=service_to_delete.to_str()))
            # TODO this will fail if service_to_delete is None
            self.ESM_DB.services.remove(service_to_delete)

    def add_manifest(self, manifest: Manifest) -> tuple:
        if manifest not in self.ESM_DB.manifests:
            self.ESM_DB.manifests.append(manifest)
            return 'ok', 200
        else:
            LOG.warn('A duplicate manifest was attempted to be registered. '
                     'Ignoring the request. '
                     'Content supplied:\n{content}'.format(content=manifest.to_str()))
            return 'duplicate', 400

    def get_manifest(self, manifest_id: str=None, plan_id: str=None) -> List[Manifest]:
{'fraction_non_alphanumeric': 0.09967737880755856, 'fraction_numerical': 0.00418988561612268, 'mean_word_length': 2.0997402597402597, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3269087', 'n_tokens_mistral': 6932, 'n_tokens_neox': 6444, 'n_words': 1609}