#!/usr/bin/env python
#
# @file    ValidatorClassFunctions.py
# @brief   class to create functions for validator classes
# @author  Frank Bergmann
# @author  Sarah Keating
#
# <!--------------------------------------------------------------------------
#
# Copyright (c) 2013-2018 by the California Institute of Technology
# (California, USA), the European Bioinformatics Institute (EMBL-EBI, UK)
# and the University of Heidelberg (Germany), with support from the National
# Institutes of Health (USA) under grant R01GM070923.  All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#
# Neither the name of the California Institute of Technology (Caltech), nor
# of the European Bioinformatics Institute (EMBL-EBI), nor of the University
# of Heidelberg, nor the names of any contributors, may be used to endorse
# or promote products derived from this software without specific prior
# written permission.
# ------------------------------------------------------------------------ -->

from ...util import strFunctions, global_variables

class ValidatorClassFunctions():
    """Class for all protected functions"""

    def __init__(self, language, class_name, is_header, elements=None,
                 package='', std_base=''):
        self.language = language
        self.up_language = strFunctions.upper_first(language)
        self.cap_language = language.upper()

        self.class_name = class_name
        self.std_base = std_base

        self.is_header = is_header
        self.elements = elements
        self.package = package
        self.up_package = strFunctions.upper_first(package)
    ########################################################################

    # Function for writing constructor/destructor functions

    def write_constructor(self, name=''):
        # create comment parts
        if name == 'visitor':
            title_line = ''
            params = []
        else:
            title_line = 'Creates a new {0} object for the given category of ' \
                         'validation.'.format(self.class_name)
            params = ['@param category code indicating the type of validation '
                      'that this validator will perform.']
        return_lines = []
        additional = []

        # create the function declaration
        function = '{0}'.format(self.class_name)
        return_type = ''
        if name == 'visitor':
            arguments = ['{0}Validator& v'.format(self.up_package),
                         'const Model& m']
            constructor_args = [': v(v)', ', m(m)']
            arguments_no_defaults = arguments
        else:
            arguments = ['{0}ErrorCategory_t category = '
                         'LIB{0}_CAT_{0}'.format(self.cap_language)]
            arguments_no_defaults = ['{0}ErrorCategory_t '
                                     'category'.format(self.cap_language)]
            constructor_args = [': Validator(category)']
        code = []
        if not self.is_header:
            if name == 'visitor':
                code = [self.create_code_block('blank', [])]
            else:
                line = ['m{0}Constraints = new {0}Validator'
                        'Constraints()'.format(self.up_package)]
                code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': False,
                     'object_name': self.class_name,
                     'implementation': code,
                     'args_no_defaults': arguments_no_defaults,
                     'constructor_args': constructor_args})

    def write_destructor(self, constraints=False):
        # create comment parts
        title_line = 'Destroys this {0} object.'.format(self.class_name)
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = '~{0}'.format(self.class_name)
        return_type = ''
        arguments = []
        code = []
        if not self.is_header:
            if constraints:
                line = 'map<VConstraint*, bool>::iterator it = ptrMap.begin()'
                code.append(self.create_code_block('line', [line]))
                nested_if = self.create_code_block('if', ['it->second',
                                                          'delete it->first'])
                code.append(self.create_code_block('while',
                                                   ['it != ptrMap.end()',
                                                    nested_if, '++it']))
            else:
                line = ['delete m{0}Constraints'.format(self.up_package)]
                code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    ########################################################################

    # Function for writing init/addConstraints functions

    def write_init_function(self):
        # create comment parts
        title_line = 'Initializes this {0} object.'.format(self.class_name)
        params = []
        return_lines = []
        additional = ['When creating a subclass of {0}, override this method '
                      'to add your own validation '
                      'code.'.format(self.class_name)]

        # create the function declaration
        function = 'init'
        return_type = 'void'
        arguments = []
        code = []
        if not self.is_header:
            line = ['To DO'.format()]
            code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code,
                     'pure_abstract': True})

    def write_add_constraint_function(self):
        # create comment parts
        title_line = 'Adds the given VConstraint object to this ' \
                     '{0}.'.format(self.class_name)
        params = ['@param c the VConstraint (\"validator constraint\") object '
                  'to add.']
        return_lines = []
        additional = []

        # create the function declaration
        function = 'addConstraint'
        return_type = 'void'
        arguments = ['VConstraint* c']
        code = []
        if not self.is_header:
            line = ['m{0}Constraints->add(c)'.format(self.up_package)]
            code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    ########################################################################

    # Function for writing init/addConstraints functions

    def write_validate_function(self, object_type):
        # create comment parts
        if object_type == 'doc':
            title_line = 'Validates the given ' \
                         '{0}'.format(global_variables.document_class)
            params = ['@param d the {0} object to be '
                      'validated.'.format(global_variables.document_class)]
        else:
            title_line = 'Validates the {0} located at the given ' \
                         'filename'.format(global_variables.document_class)
            params = ['@param filename the path to the file to be read '
                      'and validated.']
        return_lines = [' @return the number of validation failures that '
                        'occurred.  The objects describing the actual failures '
                        'can be retrieved using getFailures().']
        additional = []

        # create the function declaration
        function = 'validate'
        return_type = 'unsigned int'
        if object_type == 'doc':
            arguments = ['const {0}& d'.format(global_variables.document_class)]
        else:
            arguments = ['const std::string& filename']
        code = []
        if not self.is_header and object_type == 'doc':
            code.append(self.create_code_block('line', ['const Model* m = '
                                                        'd.getModel()']))
            nested_if = self.create_code_block('if', ['plugin != NULL',
                                                      'plugin->accept(vv)'])
            lines = ['m != NULL',
                     '{0}ValidatingVisitor vv(*this, '
                     '*m)'.format(self.up_package),
                     'const {0}SBMLDocumentPlugin* plugin = static_cast<const '
                     '{0}SBMLDocumentPlugin*>(d.getPlugin(\"{1}\")'
                     ')'.format(self.up_package, self.package),
                     nested_if]
            code.append(self.create_code_block('if', lines))
            code.append(self.create_code_block('comment',
                                               ['ADD ANY OTHER OBJECTS '
                                                'THAT HAS PLUGINS']))
            code.append(self.create_code_block('line',
                                               ['return (unsigned int)'
                                                '(mFailures.size())']))
        elif not self.is_header and object_type != 'doc':
            line = ['{0}Reader  reader'.format(self.cap_language),
                    '{0}* d = reader.read{1}'
                    '(filename)'.format(global_variables.document_class,
                                        self.cap_language)]
            code = [self.create_code_block('line', line),
                    self.create_code_block('blank', []),
                    self.create_code_block('line',
                                           ['unsigned int numErrors = '
                                            'd->getNumErrors()']),
                    self.create_code_block('blank', [])]

            lines = ['unsigned int n=0; n < numErrors; ++n',
                     'logFailure(*d->getError(n))']
            code.append(self.create_code_block('for', lines))
            lines = ['numErrors = validate(*d)', 'delete d',
                     'return numErrors']
            code.append(self.create_code_block('line', lines))

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    ########################################################################

    # Function for writing the validator structure functions

    def write_apply_function(self):
        # create comment parts
        title_line = ''
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'Apply'
        return_type = ''
        arguments = ['const Model& m, const T& o']
        constructor_args = [': model(m)', ', object(o)']
        code = []
        if not self.is_header:
            line = []
            code = [self.create_code_block('blank', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code,
                     'constructor_args': constructor_args})

    def write_operator_function(self):
        # create comment parts
        title_line = ''
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'operator()'
        return_type = 'void'
        arguments = ['TConstraint<T>* constraint']
        line = ['constraint->check(model, object)']
        code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    def write_constraint_add_function(self):
        # create comment parts
        title_line = 'Adds a Constraint to this ConstraintSet'
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'add'
        return_type = 'void'
        arguments = ['TConstraint<T>* c']
        line = ['constraints.push_back(c)']
        code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    def write_apply_to_function(self):
        # create comment parts
        title_line = 'Applies all Constraints in this ConstraintSet to the ' \
                     'given {0} object of type T. Constraint violations are ' \
                     'logged to Validator.'.format(self.cap_language)
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'applyTo'
        return_type = 'void'
        arguments = ['const Model& model', 'const T& object']
        line = ['for_each(constraints.begin(), constraints.end(), '
                'Apply<T>(model, object))']
        code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': False,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    def write_empty_function(self):
        # create comment parts
        title_line = 'Returns true if the ConstraintSet is empty, ' \
                     'false otherwise'
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'empty'
        return_type = 'bool'
        arguments = []
        line = ['return constraints.empty()']
        code = [self.create_code_block('line', line)]

        # return the parts
        return dict({'title_line': title_line,
                     'params': params,
                     'return_lines': return_lines,
                     'additional': additional,
                     'function': function,
                     'return_type': return_type,
                     'arguments': arguments,
                     'constant': True,
                     'virtual': True,
                     'object_name': self.class_name,
                     'implementation': code})

    def write_v_constraint_add_function(self):
        # create comment parts
        title_line = 'Adds the given Constraint to the appropriate ' \
                     'ConstraintSet'
        params = []
        return_lines = []
        additional = []

        # create the function declaration
        function = 'add'
        return_type = 'void'
        arguments = ['VConstraint* c']
        line = ['if (c == NULL) return']
        code = [self.create_code_block('line', line)]
        line = 'ptrMap.insert(pair<VConstraint*, bool>(c, true))'
        code.append(self.create_code_block('line', [line]))


#
# uchroma - Copyright (C) 2021 Stefanie Kondik
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
#

# pylint: disable=too-many-function-args, invalid-name, too-many-instance-attributes

import asyncio
import time

from typing import NamedTuple

from uchroma.log import LOG_TRACE
from uchroma.util import clamp

_KeyInputEvent = NamedTuple('KeyInputEvent', \
    [('timestamp', float),
     ('expire_time', float),
     ('keycode', str),
     ('scancode', str),
     ('keystate', int),
     ('coords', list),
     ('data', dict)])

class KeyInputEvent(_KeyInputEvent, object):
    """
    Container of all values of a keyboard input event
    with optional expiration time.
    """

    @property
    def time_remaining(self) -> float:
        """
        The number of seconds until this event expires
        """
        return max(0.0, self.expire_time - time.time())

    @property
    def percent_complete(self) -> float:
        """
        Percentage of elapsed time until this event expires
        """
        duration = self.expire_time - self.timestamp
        return clamp(self.time_remaining / duration, 0.0, 1.0)

class InputQueue(object):
    """
    Asynchronous input event queue

    After calling attach(), users of this class may await get_events()
    until a new event is produced from the keyboard. If an expiration time
    is set, events will not be purged until the time has lapsed-
    subsequents yield/await on get_events will return all active events.
    """

    KEY_UP = 1
    KEY_DOWN = 2
    KEY_HOLD = 4

    def __init__(self, driver, expire_time=None):

        self._logger = driver.logger
        self._input_manager = driver.input_manager
        self._key_mapping = driver.hardware.key_mapping
        self._expire_time = 0
        if expire_time is not None:
            self._expire_time = expire_time
        self._attached = False

        self._q = asyncio.Queue()
        self._events = []
        self._keystates = InputQueue.KEY_DOWN

    def attach(self) -> bool:
        """
        Start listening for input events

        :return: True if successful
        """
        if self._attached:
            return True

        if self._input_manager is None:
            raise ValueError('Input events are not supported on this device')

        if not self._input_manager.add_callback(self._input_callback):
            return False

        self._attached = True
        self._logger.debug("InputQueue attached")
        return True

    async def detach(self):
        """
        Stop listening for input events
        """
        if not self._attached:
            return

        await self._input_manager.remove_callback(self._input_callback)
        self._attached = False
        self._logger.debug("InputQueue detached")

    async def get_events(self):
        """
        Get all active (new and unexpired) events from the queue.

        This is a coroutine, and will yield until new data is available.
        """
        if not self._attached:
            self._logger.error("InputQueue is not attached!")
            return None

        if self._expire_time is None or self._expire_time <= 0:
            event = await self._q.get()
            return event

        self._expire()

        while len(self._events) == 0:
            await self._q.get()

        return self._events[:]

    @property
    def keystates(self) -> int:
        """
        The keystates to report as a bitmask
        """
        return self._keystates

    @keystates.setter
    def keystates(self, mask: int):
        """
        Set the keystates to report.

        By default, only KEY_DOWN is reported.
        """
        self._keystates = mask

    def get_events_nowait(self):
        """
        Version of get_events which returns immediately
        """
        return self._events[:]

    async def _input_callback(self, ev):
        """
        Coroutine called by the evdev module when data is available
        """
        self._expire()

        if ev.keystate == ev.key_up and not self._keystates & InputQueue.KEY_UP:
            return

        if ev.keystate == ev.key_down and not self.keystates & InputQueue.KEY_DOWN:
            return

        if ev.keystate == ev.key_hold and not self.keystates & InputQueue.KEY_HOLD:
            return

        coords = None
        if self._key_mapping is not None:
            coords = self._key_mapping.get(ev.keycode, None)

        event = KeyInputEvent(ev.event.timestamp(),
                              ev.event.timestamp() + self._expire_time,
                              ev.keycode, ev.scancode,
                              ev.keystate, coords, {})

        if self._logger.isEnabledFor(LOG_TRACE):
            self._logger.debug('Input event: %s', event)

        if self._expire_time is not None:
            for idx in range(0, len(self._events)):
                if self._events[idx].keycode == event.keycode:
                    self._events[idx] = None
            self._events = [x for x in self._events if x is not None]

            self._events.append(event)

            await self._q.put(event)

    def _expire(self):
        """
        Clear all events which have passed the deadline
        """
        if self._expire_time is None:
            return

        now = time.time()

        # Remove expired events from queue
        try:


# -*- coding: utf-8
# pylint: disable=line-too-long
"""
    A module to dealing with genome storages.

    Pangenomic workflow heavily uses this module.

    Ad hoc access to make sene of internal or external genome descriptions is also welcome.
"""

import time
import hashlib

import anvio
import anvio.db as db
import anvio.tables as t
import anvio.utils as utils
import anvio.fastalib as fastalib
import anvio.terminal as terminal
import anvio.filesnpaths as filesnpaths

from anvio.errors import ConfigError

__author__ = "Developers of anvi'o (see AUTHORS.txt)"
__copyright__ = "Copyleft 2015-2018, the Meren Lab (http://merenlab.org/)"
__credits__ = []
__license__ = "GPL 3.0"
__version__ = anvio.__version__
__maintainer__ = "A. Murat Eren"
__email__ = "a.murat.eren@gmail.com"

run = terminal.Run()
progress = terminal.Progress()
pp = terminal.pretty_print

class GenomeStorage(object):
    def __init__(self, storage_path, storage_hash=None, genome_names_to_focus=None, create_new=False, skip_init=False, skip_init_functions=False, function_annotation_sources=None, run=run, progress=progress):
        self.db_type = 'genomestorage'
        self.version = anvio.__genomes_storage_version__
        self.run = run
        self.storage_hash = storage_hash
        self.progress = progress
        self.storage_path = storage_path
        self.genome_names_to_focus = genome_names_to_focus
        self.skip_init_functions = skip_init_functions
        self.function_annotation_sources = function_annotation_sources

        if self.function_annotation_sources and not isinstance(self.function_annotation_sources, type(list())):
            raise ConfigError("The `function_annotation_sources` must be of type `list`.")

        if self.function_annotation_sources and self.skip_init_functions:
            raise ConfigError("You can't provide a function annotation source and then also ask initializing of functions to be skipped :/")

        if create_new:
            self.check_storage_path_for_create_new()
        else:
            utils.is_genome_storage(self.storage_path)
            self.check_storage_path_for_load()

        self.db = db.DB(self.storage_path, self.version, new_database=create_new)

        self.genome_info_entries = []
        self.gene_info_entries = []
        self.gene_functions_entries = []

        if create_new:
            self.create_tables()
        else:
            if skip_init:
                pass
            else:
                self.init()

    def check_storage_path_for_create_new(self):
        if not self.storage_path.endswith('GENOMES.db'):
            raise ConfigError("The genomes storage file must end with '-GENOMES.db'. Anvi'o developers do know how ridiculous "
                               "this requirement sounds like, but if you have seen the things they did, you would totally "
                               "understand why this is necessary.")

        filesnpaths.is_output_file_writable(self.storage_path)

    def check_storage_path_for_load(self):
        if not self.storage_path:
            raise ConfigError("Anvi'o genomes storage is speaking. Someone called the init function,\
                               yet there is nothing to initialize since genome storage path variable\
                               (args.genomes_storage) is None. If you are an end user, please make sure\
                               you provide the genomes storage paramater to whatever program you were\
                               running. If you are a developer, you probably already figured what is\
                               wrong. If you are a cat, you need to send us an e-mail immediately.")

        if self.storage_path.endswith('.h5'):
            raise ConfigError("We recenlty switched from HD5 files (.h5) to Sqlite (.db) files for the genome storage, \
                              you can upgrade your genome storage by running 'anvi-migrate %s'." % self.storage_path)

        filesnpaths.is_file_exists(self.storage_path)

    def get_gene_functions_in_genomes_dict(self):
        # ------8<------8<------8<------8<------8<------8<------8<------8<------8<------8<------8<------8<------8<------
        # If the user wants functions to be initialized, anvi'o will have to make a very challenging lookup
        # to fill into self.gene_info dictionary information for each functional source for each gene callers
        # id in each genome in the pangenome. The following code prepares a dictionary and a lookup table
        # for fast access in the expense of some memory space. the previous code was making SQL queries for
        # each gene callers id / genome pair. which is not too bad for small pangenomes with small number of
        # functional sources. but when the pangenome and the number of functional sources grow, the time
        # required for each call grows exponentially. just to give an example, the prochlorococcus pangenome
        # took 34 minutes to initialize functions following the `anvi-display-pan` command. the following code
        # reduces that 2 seconds.
        if self.skip_init_functions:
            return {}, {}

        function_annotation_sources_in_db = self.db.get_single_column_from_table(t.genome_gene_function_calls_table_name, 'source', unique=True)

        self.run.info("Functions found", ', '.join(function_annotation_sources_in_db))

        # if user requested function annotation sources, make sure they're in the db
        if self.function_annotation_sources:
            missing_soures = [s for s in self.function_annotation_sources if s not in function_annotation_sources_in_db]

            if len(missing_soures):
                raise ConfigError(f"The following function annotation sources are not in genomes storage: {', '.join(missing_soures)}")
            else:
                self.run.info("User requested annotation sources", ', '.join(self.function_annotation_sources))

        self.progress.new('Functions stuff')
        if self.function_annotation_sources:
            self.progress.update("Loading SOME functions into memory")
            where_clause = """source IN (%s)""" % ",".join('"' + item + '"' for item in self.function_annotation_sources)
            gene_functions_in_genomes_dict = self.db.get_some_rows_from_table_as_dict(t.genome_gene_function_calls_table_name, where_clause)
        else:
            self.progress.update("Loading ALL functions into memory")
            gene_functions_in_genomes_dict = self.db.get_table_as_dict(t.genome_gene_function_calls_table_name)

        self.progress.update("Preparing the lookup dictionary")
        # probably the following would have been much more elegant with pandas, but I am not sure
        # if the lookup performance would be comparable. sorry for not trying first.
        gene_functions_lookup_dict = {}
        for entry_id in gene_functions_in_genomes_dict:
            genome_name, gene_callers_id = gene_functions_in_genomes_dict[entry_id]['genome_name'], gene_functions_in_genomes_dict[entry_id]['gene_callers_id']

            if genome_name not in gene_functions_lookup_dict:
                gene_functions_lookup_dict[genome_name] = {}

            if gene_callers_id not in gene_functions_lookup_dict[genome_name]:
                gene_functions_lookup_dict[genome_name][gene_callers_id] = set([])

            gene_functions_lookup_dict[genome_name][gene_callers_id].add(entry_id)

        self.progress.end()

        return (gene_functions_in_genomes_dict, gene_functions_lookup_dict)

    def init(self):
        if self.db_type != self.db.get_meta_value('db_type'):
            raise ConfigError('The database "%s" does not look like a genome storage :/' % self.storage_path)

        if self.storage_hash:
            if self.storage_hash != self.get_storage_hash():
                raise ConfigError("The requested genome storage hash ('%s') does not match with the one read from the database ('%s')." %
                    (self.storage_hash, self.get_storage_hash()))

        self.genome_names_in_db = self.get_all_genome_names()

        if self.genome_names_to_focus:
            genome_names_to_focus_missing_from_db = [g for g in self.genome_names_to_focus if g not in self.genome_names_in_db]

            # make sure the user knows what they're doing
            if genome_names_to_focus_missing_from_db:
                raise ConfigError("%d of %d genome names you wanted to focus are missing from the genomes sotrage. "
                                "Although this may not be a show-stopper, anvi'o likes to be explicit, so here we "
                                "are. Not going anywhere until you fix this. For instance this is one of the missing "
                                "genome names: '%s', and this is one random genome name from the database: '%s'" % \
                                         (len(genome_names_to_focus_missing_from_db), len(self.genome_names_to_focus),\
                                         genome_names_to_focus_missing_from_db[0], ', '.join(self.genome_names_in_db)))

            self.genome_names = self.genome_names_to_focus
        else:
            self.genome_names = self.genome_names_in_db

        self.progress.new('Recovering data from the db')
        self.progress.update('Initializing...')

        self.num_genomes = len(self.genome_names)
        self.functions_are_available = self.db.get_meta_value('functions_are_available')

        ## load the data
        self.progress.update('Loading genomes basic info...')
        where_clause = """genome_name IN (%s)""" % ",".join('"' + item + '"' for item in self.genome_names)
        self.genomes_info = self.db.get_some_rows_from_table_as_dict(t.genome_info_table_name, where_clause)

        self.progress.end()

        self.gene_info = {}
        num_genes = self.db.get_row_counts_from_table(t.gene_info_table_name, where_clause)

        # get functions
        gene_functions_in_genomes_dict, gene_functions_lookup_dict = self.get_gene_functions_in_genomes_dict()

        self.progress.new('Loading genes info', progress_total_items=num_genes)
        self.progress.update('...')

        # main loop that fills in `self.gene_info` dictionary:
        for gene_num, gene_info_tuple in enumerate(self.db.get_some_rows_from_table(t.gene_info_table_name, where_clause)):
            genome_name, gene_callers_id, aa_sequence, dna_sequence, partial, length = gene_info_tuple

            if gene_num % 100 == 0:
                self.progress.increment(increment_to = gene_num)
                self.progress.update('Completed %d/%d' % (gene_num, num_genes))

            if genome_name not in self.gene_info:
                self.gene_info[genome_name] = {}

            self.gene_info[genome_name][gene_callers_id] = {
                'aa_sequence': aa_sequence,
                'dna_sequence': dna_sequence,
                'partial': partial,
                'length': length,
                'functions': {}
            }

            if not self.skip_init_functions:
                try:
                    entry_ids_for_genome_gene = gene_functions_lookup_dict[genome_name][gene_callers_id]
                except:
                    # probably we are looking one without any functions
                    entry_ids_for_genome_gene = []

                for entry_id in entry_ids_for_genome_gene:
                    source, accession, function = gene_functions_in_genomes_dict[entry_id]['source'], \
                                                  gene_functions_in_genomes_dict[entry_id]['accession'], \
                                                  gene_functions_in_genomes_dict[entry_id]['function']
                    self.gene_info[genome_name][gene_callers_id]['functions'][source] = "%s|||%s" % (accession, function)

        self.progress.end()

        self.run.info('Genomes storage', 'Initialized (storage hash: %s)' % (self.get_storage_hash()))
        self.run.info('Num genomes in storage', len(self.get_all_genome_names()))
        self.run.info('Num genomes will be used', len(self.genome_names), mc='green')

    def create_tables(self):
        self.db.create_table(t.genome_info_table_name, t.genome_info_table_structure, t.genome_info_table_types)
        self.db.create_table(t.gene_info_table_name, t.gene_info_table_structure, t.gene_info_table_types)
        self.db.create_table(t.genome_gene_function_calls_table_name, t.genome_gene_function_calls_table_structure, t.genome_gene_function_calls_table_types)
        self.db._exec("CREATE INDEX covering_index ON %s (gene_callers_id, genome_name);" % t.genome_gene_function_calls_table_name)

        self.db.set_meta_value('db_type', self.db_type)
        self.db.set_meta_value('creation_date', time.time())
        self.db.set_meta_value('functions_are_available', False)

    def get_genomes_dict(self):
        # we retrieve all table at once to avoid seperate sql queries
        all_genomes_dict = self.db.get_table_as_dict(t.genome_info_table_name)
        result = {}

        # copy genomes requested by user to result dictionary
        for genome_name in self.genome_names:
            result[genome_name] = all_genomes_dict[genome_name]

        return result

    def update_storage_hash(self):
        # here we create a signature for the storage itself by concatenating all hash values from all genomes. even if one
        # split is added or removed to any of these genomes will change this signature. since we will tie this information
        # to the profile database we will generate for the pangenome analysis, even if one split is added or removed from any
        # of the genomes will make sure that the profile databases from this storage and storage itself are not compatible:

        concatenated_genome_hashes = '_'.join(sorted(map(str, self.db.get_single_column_from_table(t.genome_info_table_name, 'genome_hash'))))
        new_hash = 'hash' + str(hashlib.sha224(concatenated_genome_hashes.encode('utf-8')).hexdigest()[0:8])

        self.db.set_meta_value('hash', new_hash)

    def get_storage_hash(self):
        return str(self.db.get_meta_value('hash'))

    def store_genomes(self, genome_descriptions):
        self.functions_are_available = genome_descriptions.functions_are_available
        self.db.set_meta_value('functions_are_available', self.functions_are_available)
        self.db.set_meta_value('gene_function_sources', ','.join(genome_descriptions.function_annotation_sources))

        num_gene_calls_added_total = 0
        num_partial_gene_calls_total = 0

        genome_names_to_go_through = sorted(genome_descriptions.genomes.keys())

        for genome_name in genome_names_to_go_through:
            self.progress.new('Initializing genomes')
            self.progress.update('%s ...' % genome_name)
            num_gene_calls_added = 0
            num_partial_gene_calls = 0


from django.db import models
from django.utils import timezone
from geopy import Point

from edc_map.exceptions import MapperError

from .site_mappers import site_mappers
from .validators import is_valid_map_area

class LandmarkMixin(models.Model):

    map_area = models.CharField(max_length=25)

    label = models.CharField(max_length=50)

    latitude = models.DecimalField(
        max_digits=20,
        null=True,
        decimal_places=15)

    longitude = models.DecimalField(
        max_digits=20,
        null=True,
        decimal_places=15)

    def __str__(self):
        return '{}: {}'.format(self.map_area, self.label)

    @property
    def point(self):
        return Point(self.latitude, self.longitude)

    @property
    def name(self):
        return self.label

    class Meta:
        abstract = True

class MapperDataModelMixin(models.Model):

    center_lat = models.DecimalField(
        max_digits=20,
        null=True,
        decimal_places=15)

    center_lon = models.DecimalField(
        max_digits=20,
        null=True,
        decimal_places=15)

    radius = models.DecimalField(
        max_digits=10,
        null=True,
        decimal_places=2)

    map_area = models.CharField(
        max_length=25)

    class Meta:
        abstract = True

class MapperModelMixin(models.Model):

    gps_confirmed_latitude = models.DecimalField(
        verbose_name='latitude',
        max_digits=20,
        null=True,
        blank=True,
        decimal_places=15)

    gps_confirmed_longitude = models.DecimalField(
        verbose_name='longitude',
        max_digits=20,
        null=True,
        blank=True,
        decimal_places=15)

    gps_target_lat = models.DecimalField(
        verbose_name='target waypoint latitude',
        max_digits=20,
        default=0.0,
        null=True,
        blank=True,
        decimal_places=15)

    gps_target_lon = models.DecimalField(
        verbose_name='target waypoint longitude',
        max_digits=20,
        default=0.0,
        null=True,
        blank=True,
        decimal_places=15)

    target_radius = models.FloatField(
        default=.025,
        null=True,
        blank=True,
        help_text='km')

    distance_from_target = models.FloatField(
        null=True,
        editable=False,
        help_text='distance in meters')

    map_area = models.CharField(
        max_length=25,
        validators=[is_valid_map_area])

    location_name = models.CharField(
        max_length=25,
        null=True,
        blank=True)

    confirmed = models.BooleanField(
        default=False,
        editable=False,
        help_text="gps target is confirmed")

    section = models.CharField(
        max_length=25,
        null=True,
        editable=False,
        verbose_name='Section')

    sub_section = models.CharField(
        max_length=25,
        null=True,
        editable=False,
        verbose_name='Sub-section',
        help_text='')

    def save(self, *args, **kwargs):
        if self.gps_confirmed_longitude and self.gps_confirmed_latitude:
            self.confirmed = self.get_confirmed()
        else:
            self.distance_from_target = None
            self.confirmed = False
        super().save(*args, **kwargs)

    def common_clean(self):
        """Attempt to trigger MapperError exception if gps is not valid.
        """
        site_mappers.get_mapper(self.map_area)
        self.is_valid_target_or_raise()
        if self.gps_confirmed_longitude and self.gps_confirmed_latitude:
            self.get_confirmed()
        super().common_clean()

    @property
    def point(self):
        """Alias for confirmed_point.
        """
        return self.confirmed_point

    def is_valid_target_or_raise(self):
        mapper = site_mappers.get_mapper(self.map_area)
        try:
            mapper.raise_if_not_in_map_area(self.target_point)
        except MapperError as e:
            raise MapperError(
                'Invalid target GPS point. Got {}'.format(str(e)))

    @property
    def confirmed_point(self):
        """Returns a geopy point of the confirmed gps.
        """
        return Point(self.gps_confirmed_latitude, self.gps_confirmed_longitude)

    @property
    def target_point(self):
        """Returns a geopy point of the target gps.
        """
        return Point(self.gps_target_lat, self.gps_target_lon)

    def get_confirmed(self):
        """Returns True if plot is considered "confirmed" or raises
        an exception.
        """
        mapper = site_mappers.get_mapper(self.map_area)
        try:
            mapper.raise_if_not_in_map_area(self.confirmed_point)
        except MapperError as e:
            raise MapperError(f'Invalid confirmation GPS point. Got {e}')
        mapper.raise_if_not_in_radius(
            self.confirmed_point, self.target_point, self.target_radius,
            units='km', label='target location')
        self.distance_from_target = mapper.distance_between_points(
            self.confirmed_point, self.target_point, units='m')
        return True

    class Meta:
        abstract = True

class CustomRadiusMixin(models.Model):
    """A model completed by the user to allow a plot\'s GPS
    target radius to be changed.

    An instance is auto created once the criteria is met.
    See method plot.increase_plot_radius.
    """
    identifier = models.CharField(max_length=50, unique=True)

    radius = models.FloatField(
        default=25.0,
        help_text='meters')

    reason = models.CharField(max_length=25)


# -*- coding: utf-8 -*-
#-----------------------------------------------------------------------------
# (C) British Crown Copyright 2012-5 Met Office.
#
# This file is part of Rose, a framework for meteorological suites.
#
# Rose is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Rose is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Rose. If not, see <http://www.gnu.org/licenses/>.
#-----------------------------------------------------------------------------

import re

import rose.config
import rose.macro
import rose.variable

_OPTIONS_KEY = "options"
_REPORTED_SECTION_KEY = "reported config sect"
_SECTION_IS_COMPULSORY_KEY = "section is compulsory"

class CompulsoryChecker(rose.macro.MacroBaseRoseEdit):

    """Returns sections and options that are compulsory but missing.

    It also returns sections or options that are compulsory but
    user-ignored.

    """

    WARNING_COMPULSORY_SECT_MISSING = ('Section set as compulsory, but '
                                       'not in configuration.')
    WARNING_COMPULSORY_OPT_MISSING = ('Variable set as compulsory, but '
                                      'not in configuration.')
    WARNING_COMPULSORY_USER_IGNORED = ('Compulsory settings should not be '
                                       'user-ignored.')

    def __init__(self, *args, **kwargs):
        self.basic_section_aliases = {}
        self.alias_section_to_basics = {}
        self.duplicate_section_default = {}
        self.compulsory_data = None
        super(CompulsoryChecker, self).__init__(*args, **kwargs)

    def get_compulsory_data(self, meta_config):
        """Return a list of compulsory=true basic (no duplicate info) ids."""
        compulsory_data = {}
        for setting_id, sect_node in meta_config.value.items():
            if sect_node.is_ignored() or isinstance(sect_node.value, str):
                continue
            if (sect_node.get_value([rose.META_PROP_COMPULSORY]) ==
                    rose.META_PROP_VALUE_TRUE):
                config_sect, config_opt = (
                    self._get_section_option_from_id(setting_id))
                compulsory_data.setdefault(
                    config_sect,
                    {_SECTION_IS_COMPULSORY_KEY: False,
                     _REPORTED_SECTION_KEY: config_sect,
                     _OPTIONS_KEY: []}
                )
                if config_opt is None:
                    compulsory_data[config_sect][
                        _SECTION_IS_COMPULSORY_KEY] = True
                else:
                    compulsory_data[config_sect][_OPTIONS_KEY].append(
                        config_opt)
                if (sect_node.get_value([rose.META_PROP_DUPLICATE]) ==
                        rose.META_PROP_VALUE_TRUE):
                    compulsory_data[config_sect][_REPORTED_SECTION_KEY] = (
                        config_sect + "({0})".format(
                            rose.CONFIG_SETTING_INDEX_DEFAULT)
                    )
        return compulsory_data

    def validate(self, config_data, meta_config):
        """Return a list of compulsory-related errors, if any.

        config_data - a rose.config.ConfigNode or a dictionary that
        looks like this:
        {"sections":
            {"namelist:foo": rose.section.Section instance,
             "env": rose.section.Section instance},
         "variables":
            {"namelist:foo": [rose.variable.Variable instance,
                              rose.variable.Variable instance],
             "env": [rose.variable.Variable instance]
            }
        }
        meta_config - a rose.config.ConfigNode.

        """
        return self.validate_settings(config_data, meta_config)

    def validate_settings(self, config_data, meta_config,
                          only_these_sections=None):
        """Return a list of compulsory-related errors, if any.

        config_data - a rose.config.ConfigNode or a dictionary that
        looks like this:
        {"sections":
            {"namelist:foo": rose.section.Section instance,
             "env": rose.section.Section instance},
         "variables":
            {"namelist:foo": [rose.variable.Variable instance,
                              rose.variable.Variable instance],
             "env": [rose.variable.Variable instance]
            }
        }
        meta_config - a rose.config.ConfigNode.
        only_these_sections (default None) - a list of sections to
        examine. If specified, checking for other sections will be
        skipped.

        """
        self.reports = []
        if self.compulsory_data is None:
            self.compulsory_data = self.get_compulsory_data(meta_config)
        self._generate_aliases_for_sections(config_data)
        if only_these_sections is None:
            basic_sections_to_check = self.compulsory_data.keys()
        else:
            basic_sections_to_check = []
            alias_sections_to_check = []
            for section in list(only_these_sections):
                basic_sections_to_check.extend(
                    self.alias_section_to_basics.get(section, [section]))
                alias_sections_to_check.extend(
                    self.basic_section_aliases.get(section, [section]))
        check_user_ignored_ids = []
        for basic_section in basic_sections_to_check:
            section_data = self.compulsory_data.get(basic_section)
            if section_data is None:
                # This happens when an only_these_section is not compulsory.
                continue

            # Find all sections in config_data that belong to basic_section.
            present_section_aliases = []
            for alias_section in self.basic_section_aliases.get(
                    basic_section, []):
                if self._get_config_has_id(config_data, alias_section):
                    present_section_aliases.append(alias_section)
                    if section_data[_SECTION_IS_COMPULSORY_KEY]:
                        check_user_ignored_ids.append(alias_section)

            if not present_section_aliases:
                # No sections in config_data that belong to basic_section.
                if section_data[_SECTION_IS_COMPULSORY_KEY]:
                    self.add_report(
                        section_data[_REPORTED_SECTION_KEY], None, None,
                        self.WARNING_COMPULSORY_SECT_MISSING
                    )
                continue

            if not section_data[_OPTIONS_KEY]:
                # There are no compulsory options set for basic_section.
                continue

            # Check for compulsory options in the present sections.
            for alias_section in present_section_aliases:
                if (only_these_sections is not None and
                        alias_section not in alias_sections_to_check):
                    continue
                for option in section_data[_OPTIONS_KEY]:
                    present_option_aliases = []
                    for alias_option in self._get_config_section_options(
                            config_data, alias_section):
                        if (option == alias_option or
                            (alias_option.startswith(option) and
                             rose.macro.REC_ID_STRIP_DUPL.sub(
                                 "", alias_option) == option)):
                            setting_id = self._get_id_from_section_option(
                                alias_section, alias_option)
                            check_user_ignored_ids.append(setting_id)
                            present_option_aliases.append(alias_option)
                    if not present_option_aliases:
                        self.add_report(
                            alias_section, option, None,
                            self.WARNING_COMPULSORY_OPT_MISSING
                        )
        # Check that ids that we have found are not user-ignored.
        for setting_id in check_user_ignored_ids:
            if (self._get_config_id_state(config_data, setting_id) ==
                    rose.config.ConfigNode.STATE_USER_IGNORED):
                value = self._get_config_id_value(config_data, setting_id)
                section, option = self._get_section_option_from_id(setting_id)
                self.add_report(section, option, value,
                                self.WARNING_COMPULSORY_USER_IGNORED)
        return self.reports

    def _generate_aliases_for_sections(self, config_data):
        """Generate maps of duplicate-vs-basic sections in config_data."""
        for section in self._get_config_sections(config_data):
            if section not in self.alias_section_to_basics:
                basic_section_no_modifier = (
                    rose.macro.REC_ID_STRIP.sub('', section))
                basic_section_keep_modifier = (
                    rose.macro.REC_ID_STRIP_DUPL.sub('', section))


# RPi Meteor Station
# Copyright (C) 2015  Dario Zubovic
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import print_function, division, absolute_import

import re
import time
import logging
from multiprocessing import Process, Event

import cv2

from RMS.Misc import ping

# Get the logger from the main module
log = logging.getLogger("logger")

class BufferedCapture(Process):
    """ Capture from device to buffer in memory.
    """

    running = False

    def __init__(self, array1, startTime1, array2, startTime2, config, video_file=None):
        """ Populate arrays with (startTime, frames) after startCapture is called.

        Arguments:
            array1: numpy array in shared memory that is going to be filled with frames
            startTime1: float in shared memory that holds time of first frame in array1
            array2: second numpy array in shared memory
            startTime2: float in shared memory that holds time of first frame in array2

        Keyword arguments:
            video_file: [str] Path to the video file, if it was given as the video source. None by default.

        """

        super(BufferedCapture, self).__init__()
        self.array1 = array1
        self.startTime1 = startTime1
        self.array2 = array2
        self.startTime2 = startTime2

        self.startTime1.value = 0
        self.startTime2.value = 0

        self.config = config

        self.video_file = video_file

        # A frame will be considered dropped if it was late more then half a frame
        self.time_for_drop = 1.5*(1.0/config.fps)

        self.dropped_frames = 0

    def startCapture(self, cameraID=0):
        """ Start capture using specified camera.

        Arguments:
            cameraID: ID of video capturing device (ie. ID for /dev/video3 is 3). Default is 0.

        """

        self.cameraID = cameraID
        self.exit = Event()
        self.start()

    def stopCapture(self):
        """ Stop capture.
        """

        self.exit.set()

        time.sleep(1)

        log.info("Joining capture...")

        # Wait for the capture to join for 60 seconds, then terminate
        for i in range(60):
            if self.is_alive():
                time.sleep(1)
            else:
                break

        if self.is_alive():
            log.info('Terminating capture...')
            self.terminate()

    def initVideoDevice(self):
        """ Initialize the video device. """

        device = None

        # use a file as the video source
        if self.video_file is not None:
            device = cv2.VideoCapture(self.video_file)

        # Use a device as the video source
        else:

            # If an analog camera is used, skip the ping
            ip_cam = False
            if "rtsp" in str(self.config.deviceID):
                ip_cam = True

            if ip_cam:

                ### If the IP camera is used, check first if it can be pinged

                # Extract the IP address
                ip = re.findall(r"[0-9]+(?:\.[0-9]+){3}", self.config.deviceID)

                # Check if the IP address was found
                if ip:
                    ip = ip[0]

                    # Try pinging 5 times
                    ping_success = False

                    for i in range(500):

                        print('Trying to ping the IP camera...')
                        ping_success = ping(ip)

                        if ping_success:
                            log.info("Camera IP ping successful!")
                            break

                        time.sleep(5)

                    if not ping_success:
                        log.error("Can't ping the camera IP!")
                        return None

                else:
                    return None

            # Init the video device
            log.info("Initializing the video device...")
            log.info("Device: " + str(self.config.deviceID))
            if self.config.force_v4l2:
                device = cv2.VideoCapture(self.config.deviceID, cv2.CAP_V4L2)
                device.set(cv2.CAP_PROP_CONVERT_RGB, 0)
            else:
                device = cv2.VideoCapture(self.config.deviceID)

            # Try setting the resultion if using a video device, not gstreamer
            try:

                # This will fail if the video device is a gstreamer pipe
                int(self.config.deviceID)

                # Set the resolution (crashes if using an IP camera and gstreamer!)
                device.set(3, self.config.width_device)
                device.set(4, self.config.height_device)

            except:
                pass

        return device

    def run(self):
        """ Capture frames.
        """

        # Init the video device
        device = self.initVideoDevice()

        if device is None:

            log.info('The video source could not be opened!')
            self.exit.set()
            return False

        # Wait until the device is opened
        device_opened = False
        for i in range(20):
            time.sleep(1)
            if device.isOpened():
                device_opened = True
                break

        # If the device could not be opened, stop capturing
        if not device_opened:
            log.info('The video source could not be opened!')
            self.exit.set()
            return False

        else:
            log.info('Video device opened!')

        # Throw away first 10 frame
        for i in range(10):
            device.read()

        first = True

        wait_for_reconnect = False

        # Run until stopped from the outside
        while not self.exit.is_set():

            lastTime = 0

            if first:
                self.startTime1.value = 0
            else:
                self.startTime2.value = 0

            # If the video device was disconnected, wait 5s for reconnection
            if wait_for_reconnect:

                print('Reconnecting...')

                while not self.exit.is_set():

                    log.info('Waiting for the video device to be reconnected...')

                    time.sleep(5)

                    # Reinit the video device
                    device = self.initVideoDevice()

                    if device is None:
                        print("The video device couldn't be connected! Retrying...")
                        continue

                    if self.exit.is_set():
                        break

                    # Read the frame
                    log.info("Reading frame...")
                    ret, frame = device.read()
                    log.info("Frame read!")

                    # If the connection was made and the frame was retrieved, continue with the capture
                    if ret:
                        log.info('Video device reconnected successfully!')
                        wait_for_reconnect = False
                        break

                wait_for_reconnect = False

            t_frame = 0
            t_assignment = 0
            t_convert = 0
            t_block = time.time()
            block_frames = 256

            log.info('Grabbing a new block of {} frames...'.format(block_frames))


# -*- coding: utf-8 -*-
"""
Tests for course access
"""
import itertools

import ddt
from django.conf import settings
from django.test.utils import override_settings
from django.core.urlresolvers import reverse
from django.test.client import RequestFactory
import mock
from nose.plugins.attrib import attr

from courseware.courses import (
    get_cms_block_link,
    get_cms_course_link,
    get_courses,
    get_course_about_section,
    get_course_by_id,
    get_course_info_section,
    get_course_overview_with_access,
    get_course_with_access,
)
from courseware.module_render import get_module_for_descriptor
from courseware.model_data import FieldDataCache
from lms.djangoapps.courseware.courseware_access_exception import CoursewareAccessException
from openedx.core.djangolib.testing.utils import get_mock_request
from openedx.core.lib.courses import course_image_url
from student.tests.factories import UserFactory
from xmodule.modulestore.django import _get_modulestore_branch_setting, modulestore
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.xml_importer import import_course_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import (
    CourseFactory, ItemFactory, check_mongo_calls
)
from xmodule.tests.xml import factories as xml
from xmodule.tests.xml import XModuleXmlImportTest

CMS_BASE_TEST = 'testcms'
TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT

@attr(shard=1)
@ddt.ddt
class CoursesTest(ModuleStoreTestCase):
    """Test methods related to fetching courses."""
    ENABLED_SIGNALS = ['course_published']

    @override_settings(CMS_BASE=CMS_BASE_TEST)
    def test_get_cms_course_block_link(self):
        """
        Tests that get_cms_course_link_by_id and get_cms_block_link_by_id return the right thing
        """
        self.course = CourseFactory.create(
            org='org', number='num', display_name='name'
        )

        cms_url = u"//{}/course/{}".format(CMS_BASE_TEST, unicode(self.course.id))
        self.assertEqual(cms_url, get_cms_course_link(self.course))
        cms_url = u"//{}/course/{}".format(CMS_BASE_TEST, unicode(self.course.location))
        self.assertEqual(cms_url, get_cms_block_link(self.course, 'course'))

    @ddt.data(get_course_with_access, get_course_overview_with_access)
    def test_get_course_func_with_access_error(self, course_access_func):
        user = UserFactory.create()
        course = CourseFactory.create(visible_to_staff_only=True)

        with self.assertRaises(CoursewareAccessException) as error:
            course_access_func(user, 'load', course.id)
        self.assertEqual(error.exception.message, "Course not found.")
        self.assertEqual(error.exception.access_response.error_code, "not_visible_to_user")
        self.assertFalse(error.exception.access_response.has_access)

    @ddt.data(
        (get_course_with_access, 1),
        (get_course_overview_with_access, 0),
    )
    @ddt.unpack
    def test_get_course_func_with_access(self, course_access_func, num_mongo_calls):
        user = UserFactory.create()
        course = CourseFactory.create(emit_signals=True)
        with check_mongo_calls(num_mongo_calls):
            course_access_func(user, 'load', course.id)

    def test_get_courses_by_org(self):
        """
        Verify that org filtering performs as expected, and that an empty result
        is returned if the org passed by the caller does not match the designated
        org.
        """
        primary = 'primary'
        alternate = 'alternate'

        def _fake_get_value(value, default=None):
            """Used to stub out site_configuration.helpers.get_value()."""
            if value == 'course_org_filter':
                return alternate

            return default

        user = UserFactory.create()

        # Pass `emit_signals=True` so that these courses are cached with CourseOverviews.
        primary_course = CourseFactory.create(org=primary, emit_signals=True)
        alternate_course = CourseFactory.create(org=alternate, emit_signals=True)

        self.assertNotEqual(primary_course.org, alternate_course.org)

        unfiltered_courses = get_courses(user)
        for org in [primary_course.org, alternate_course.org]:
            self.assertTrue(
                any(course.org == org for course in unfiltered_courses)
            )

        filtered_courses = get_courses(user, org=primary)
        self.assertTrue(
            all(course.org == primary_course.org for course in filtered_courses)
        )

        with mock.patch(
            'openedx.core.djangoapps.site_configuration.helpers.get_value',
            autospec=True,
        ) as mock_get_value:
            mock_get_value.side_effect = _fake_get_value

            # Request filtering for an org distinct from the designated org.
            no_courses = get_courses(user, org=primary)
            self.assertEqual(no_courses, [])

            # Request filtering for an org matching the designated org.
            site_courses = get_courses(user, org=alternate)
            self.assertTrue(
                all(course.org == alternate_course.org for course in site_courses)
            )

    def test_get_courses_with_filter(self):
        """
        Verify that filtering performs as expected.
        """
        user = UserFactory.create()
        non_mobile_course = CourseFactory.create(emit_signals=True)
        mobile_course = CourseFactory.create(mobile_available=True, emit_signals=True)

        test_cases = (
            (None, {non_mobile_course.id, mobile_course.id}),
            (dict(mobile_available=True), {mobile_course.id}),
            (dict(mobile_available=False), {non_mobile_course.id}),
        )
        for filter_, expected_courses in test_cases:
            self.assertEqual(
                {
                    course.id
                    for course in
                    get_courses(user, filter_=filter_)
                },
                expected_courses,
                "testing get_courses with filter_={}".format(filter_),
            )

@attr(shard=1)
class ModuleStoreBranchSettingTest(ModuleStoreTestCase):
    """Test methods related to the modulestore branch setting."""
    @mock.patch(
        'xmodule.modulestore.django.get_current_request_hostname',
        mock.Mock(return_value='preview.localhost')
    )
    @override_settings(
        HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={r'preview\.': ModuleStoreEnum.Branch.draft_preferred},
        MODULESTORE_BRANCH='fake_default_branch',
    )
    def test_default_modulestore_preview_mapping(self):
        self.assertEqual(_get_modulestore_branch_setting(), ModuleStoreEnum.Branch.draft_preferred)

    @mock.patch(
        'xmodule.modulestore.django.get_current_request_hostname',
        mock.Mock(return_value='localhost')
    )
    @override_settings(
        HOSTNAME_MODULESTORE_DEFAULT_MAPPINGS={r'preview\.': ModuleStoreEnum.Branch.draft_preferred},
        MODULESTORE_BRANCH='fake_default_branch',
    )
    def test_default_modulestore_branch_mapping(self):
        self.assertEqual(_get_modulestore_branch_setting(), 'fake_default_branch')

@attr(shard=1)
@override_settings(CMS_BASE=CMS_BASE_TEST)
class MongoCourseImageTestCase(ModuleStoreTestCase):
    """Tests for course image URLs when using a mongo modulestore."""

    def test_get_image_url(self):
        """Test image URL formatting."""
        course = CourseFactory.create(org='edX', course='999')
        self.assertEquals(course_image_url(course), '/c4x/edX/999/asset/{0}'.format(course.course_image))

    def test_non_ascii_image_name(self):
        # Verify that non-ascii image names are cleaned
        course = CourseFactory.create(course_image=u'before_\N{SNOWMAN}_after.jpg')
        self.assertEquals(
            course_image_url(course),
            '/c4x/{org}/{course}/asset/before___after.jpg'.format(
                org=course.location.org,
                course=course.location.course
            )
        )

    def test_spaces_in_image_name(self):
        # Verify that image names with spaces in them are cleaned
        course = CourseFactory.create(course_image=u'before after.jpg')
        self.assertEquals(
            course_image_url(course),
            '/c4x/{org}/{course}/asset/before_after.jpg'.format(
                org=course.location.org,
                course=course.location.course
            )
        )

    def test_static_asset_path_course_image_default(self):
        """
        Test that without course_image being set, but static_asset_path
        being set that we get the right course_image url.
        """