"""
Django module for Course Metadata class -- manages advanced settings and related parameters
"""
from xblock.fields import Scope
from xmodule.modulestore.django import modulestore
from django.utils.translation import ugettext as _
from django.conf import settings

class CourseMetadata(object):
    '''
    For CRUD operations on metadata fields which do not have specific editors
    on the other pages including any user generated ones.
    The objects have no predefined attrs but instead are obj encodings of the
    editable metadata.
    '''
    # The list of fields that wouldn't be shown in Advanced Settings.
    # Should not be used directly. Instead the filtered_list method should
    # be used if the field needs to be filtered depending on the feature flag.
    FILTERED_LIST = [
        'cohort_config',
        'xml_attributes',
        'start',
        'end',
        'enrollment_start',
        'enrollment_end',
        'tabs',
        'graceperiod',
        'checklists',
        'show_timezone',
        'format',
        'graded',
        'hide_from_toc',
        'pdf_textbooks',
        'user_partitions',
        'name',  # from xblock
        'tags',  # from xblock
        'enable_enrollment_email'
        'visible_to_staff_only',
        'group_access',
        'pre_requisite_courses',
        'entrance_exam_enabled',
        'entrance_exam_minimum_score_pct',
        'entrance_exam_id',
        'is_entrance_exam',
        'in_entrance_exam',
        'language',
        'certificates',
        'minimum_grade_credit',
        'default_time_limit_minutes',
        'is_proctored_enabled',
        'is_time_limited',
        'is_practice_exam',
    ]

    @classmethod
    def filtered_list(cls):
        """
        Filter fields based on feature flag, i.e. enabled, disabled.
        """
        # Copy the filtered list to avoid permanently changing the class attribute.
        filtered_list = list(cls.FILTERED_LIST)

        # Do not show giturl if feature is not enabled.
        if not settings.FEATURES.get('ENABLE_EXPORT_GIT'):
            filtered_list.append('giturl')

        # Do not show edxnotes if the feature is disabled.
        if not settings.FEATURES.get('ENABLE_EDXNOTES'):
            filtered_list.append('edxnotes')

        # Do not show video_upload_pipeline if the feature is disabled.
        if not settings.FEATURES.get('ENABLE_VIDEO_UPLOAD_PIPELINE'):
            filtered_list.append('video_upload_pipeline')

        # Do not show facebook_url if the feature is disabled.
        if not settings.FEATURES.get('ENABLE_MOBILE_SOCIAL_FACEBOOK_FEATURES'):
            filtered_list.append('facebook_url')

        # Do not show social sharing url field if the feature is disabled.
        if (not settings.FEATURES.get('SOCIAL_SHARING_SETTINGS') or
                not settings.FEATURES.get("SOCIAL_SHARING_SETTINGS").get("CUSTOM_COURSE_URLS")):
            filtered_list.append('social_sharing_url')

        # Do not show teams configuration if feature is disabled.
        if not settings.FEATURES.get('ENABLE_TEAMS'):
            filtered_list.append('teams_configuration')

        if not settings.FEATURES.get('ENABLE_VIDEO_BUMPER'):
            filtered_list.append('video_bumper')

        # Do not show enable_ccx if feature is not enabled.
        if not settings.FEATURES.get('CUSTOM_COURSES_EDX'):
            filtered_list.append('enable_ccx')

        return filtered_list

    @classmethod
    def fetch(cls, descriptor):
        """
        Fetch the key:value editable course details for the given course from
        persistence and return a CourseMetadata model.
        """
        result = {}
        metadata = cls.fetch_all(descriptor)
        for key, value in metadata.iteritems():
            if key in cls.filtered_list():
                continue
            result[key] = value
        return result

    @classmethod
    def fetch_all(cls, descriptor):
        """
        Fetches all key:value pairs from persistence and returns a CourseMetadata model.
        """
        result = {}
        for field in descriptor.fields.values():
            if field.scope != Scope.settings:
                continue
            result[field.name] = {
                'value': field.read_json(descriptor),
                'display_name': _(field.display_name),    # pylint: disable=translation-of-non-string
                'help': _(field.help),                    # pylint: disable=translation-of-non-string
                'deprecated': field.runtime_options.get('deprecated', False)
            }
        return result

    @classmethod
    def update_from_json(cls, descriptor, jsondict, user, filter_tabs=True):
        """
        Decode the json into CourseMetadata and save any changed attrs to the db.

        Ensures none of the fields are in the blacklist.
        """
        filtered_list = cls.filtered_list()
        # Don't filter on the tab attribute if filter_tabs is False.
        if not filter_tabs:
            filtered_list.remove("tabs")

        # Validate the values before actually setting them.
        key_values = {}

        for key, model in jsondict.iteritems():
            # should it be an error if one of the filtered list items is in the payload?
            if key in filtered_list:
                continue
            try:
                val = model['value']
                if hasattr(descriptor, key) and getattr(descriptor, key) != val:
                    key_values[key] = descriptor.fields[key].from_json(val)
            except (TypeError, ValueError) as err:
                raise ValueError(_("Incorrect format for field '{name}'. {detailed_message}").format(
                    name=model['display_name'], detailed_message=err.message))

        return cls.update_from_dict(key_values, descriptor, user)

    @classmethod
    def validate_and_update_from_json(cls, descriptor, jsondict, user, filter_tabs=True):
        """
        Validate the values in the json dict (validated by xblock fields from_json method)

        If all fields validate, go ahead and update those values on the object and return it without
        persisting it to the DB.
        If not, return the error objects list.

        Returns:
            did_validate: whether values pass validation or not
            errors: list of error objects
            result: the updated course metadata or None if error
        """
        filtered_list = cls.filtered_list()
        if not filter_tabs:
            filtered_list.remove("tabs")

        filtered_dict = dict((k, v) for k, v in jsondict.iteritems() if k not in filtered_list)
        did_validate = True
        errors = []
        key_values = {}
        updated_data = None

        for key, model in filtered_dict.iteritems():
            try:
                val = model['value']
                if hasattr(descriptor, key) and getattr(descriptor, key) != val:
                    key_values[key] = descriptor.fields[key].from_json(val)
            except (TypeError, ValueError) as err:
                did_validate = False
                errors.append({'message': err.message, 'model': model})

        # If did validate, go ahead and update the metadata
        if did_validate:
            updated_data = cls.update_from_dict(key_values, descriptor, user, save=False)

        return did_validate, errors, updated_data

    @classmethod
    def update_from_dict(cls, key_values, descriptor, user, save=True):
        """
        Update metadata descriptor from key_values. Saves to modulestore if save is true.
        """
        for key, value in key_values.iteritems():
            setattr(descriptor, key, value)


#
# Gramps - a GTK+/GNOME based genealogy program
#
# Copyright (C) 2000-2007  Donald N. Allingham
# Copyright (C) 2009       Brian G. Matherly
# Copyright (C) 2009       Gary Burton
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

# $Id$

"""
Provide the management of databases from CLI. This includes opening, renaming,
creating, and deleting of databases.
"""

#-------------------------------------------------------------------------
#
# Standard python modules
#
#-------------------------------------------------------------------------
from __future__ import print_function

import os
import sys
import time
if sys.version_info[0] < 3:
    from urlparse import urlparse
    from urllib2 import urlopen, url2pathname
else:
    from urllib.parse import urlparse
    from urllib.request import urlopen, url2pathname
import tempfile
#-------------------------------------------------------------------------
#
# set up logging
#
#-------------------------------------------------------------------------
import logging
LOG = logging.getLogger(".clidbman")

#-------------------------------------------------------------------------
#
# gramps modules
#
#-------------------------------------------------------------------------
from gramps.gen.const import GRAMPS_LOCALE as glocale
_ = glocale.get_translation().gettext
from gramps.gen.db import DbBsddb
from gramps.gen.plug import BasePluginManager
from gramps.gen.config import config
from gramps.gen.constfunc import win, conv_to_unicode
#-------------------------------------------------------------------------
#
# constants
#
#-------------------------------------------------------------------------

DEFAULT_TITLE = _("Family Tree")
NAME_FILE     = "name.txt"
META_NAME     = "meta_data.db"

#-------------------------------------------------------------------------
#
# functions
#
#-------------------------------------------------------------------------
def _errordialog(title, errormessage):
    """
    Show the error. A title for the error and an errormessage
    """
    print(_('ERROR: %(title)s \n       %(message)s') % {
                'title': title,
                'message': errormessage})
    sys.exit()

#-------------------------------------------------------------------------
#
# CLIDbManager
#
#-------------------------------------------------------------------------
class CLIDbManager(object):
    """
    Database manager without GTK functionality, allows users to create and
    open databases
    """
    IND_NAME = 0
    IND_PATH = 1
    IND_PATH_NAMEFILE = 2
    IND_TVAL_STR = 3
    IND_TVAL = 4
    IND_USE_ICON_BOOL = 5
    IND_STOCK_ID =6

    ICON_NONE     = 0
    ICON_RECOVERY = 1
    ICON_LOCK     = 2
    ICON_OPEN     = 3

    ICON_MAP = {
                ICON_NONE : None,
                ICON_RECOVERY : None,
                ICON_LOCK : None,
                ICON_OPEN : None,
               }

    ERROR = _errordialog
    def __init__(self, dbstate):
        self.dbstate = dbstate
        self.msg = None

        if dbstate:
            self.active  = dbstate.db.get_save_path()
        else:
            self.active = None

        self.current_names = []
        if dbstate:
            self._populate_cli()

    def empty(self, val):
        """Callback that does nothing
        """
        pass

    def get_dbdir_summary(self, file_name):
        """
        Returns (people_count, version_number) of current DB.
        Returns ("Unknown", "Unknown") if invalid DB or other error.
        """
        if config.get('preferences.use-bsddb3') or sys.version_info[0] >= 3:
            from bsddb3 import dbshelve, db
        else:
            from bsddb import dbshelve, db
        from gramps.gen.db import META, PERSON_TBL
        env = db.DBEnv()
        flags = db.DB_CREATE | db.DB_PRIVATE |\
            db.DB_INIT_MPOOL | db.DB_INIT_LOCK |\
            db.DB_INIT_LOG | db.DB_INIT_TXN | db.DB_THREAD
        try:
            env.open(file_name, flags)
        except:
            return "Unknown", "Unknown"
        dbmap1 = dbshelve.DBShelf(env)
        fname = os.path.join(file_name, META + ".db")
        try:
            dbmap1.open(fname, META, db.DB_HASH, db.DB_RDONLY)
        except:
            env.close()
            return "Unknown", "Unknown"
        version = dbmap1.get('version', default=None)
        dbmap1.close()
        dbmap2 = dbshelve.DBShelf(env)
        fname = os.path.join(file_name, PERSON_TBL + ".db")
        try:
            dbmap2.open(fname, PERSON_TBL, db.DB_HASH, db.DB_RDONLY)
        except:
            env.close()
            return "Unknown", "Unknown"
        count = len(dbmap2)
        dbmap2.close()
        env.close()
        return (count, version)

    def family_tree_summary(self):
        """
        Return a list of dictionaries of the known family trees.
        """
        # make the default directory if it does not exist
        list = []
        for item in self.current_names:
            (name, dirpath, path_name, last,
             tval, enable, stock_id) = item
            count, version = self.get_dbdir_summary(dirpath)
            retval = {}
            retval["Number of people"] = count
            if enable:
                retval["Locked?"] = "yes"
            else:
                retval["Locked?"] = "no"
            retval["DB version"] = version
            if sys.version_info[0] < 3:
                retval["Family tree"] = name.encode(glocale.getfilesystemencoding())
            else:
                retval["Family tree"] = name
            retval["Path"] = dirpath
            retval["Last accessed"] = time.strftime('%x %X',
                                                    time.localtime(tval))
            list.append( retval )
        return list

    def _populate_cli(self):
        """ Get the list of current names in the database dir
        """
        # make the default directory if it does not exist
        dbdir = os.path.expanduser(config.get('behavior.database-path'))
        if sys.version_info[0] < 3:
            dbdir = dbdir.encode(glocale.getfilesystemencoding())
        db_ok = make_dbdir(dbdir)

        self.current_names = []
        if db_ok:
            for dpath in os.listdir(dbdir):
                dirpath = os.path.join(dbdir, dpath)
                path_name = os.path.join(dirpath, NAME_FILE)
                if os.path.isfile(path_name):
                    file = open(path_name)
                    name = file.readline().strip()
                    file.close()

                    (tval, last) = time_val(dirpath)
                    (enable, stock_id) = self.icon_values(dirpath, self.active,
                                                     self.dbstate.db.is_open())

                    if (stock_id == 'gramps-lock'):
                        last = find_locker_name(dirpath)

                    self.current_names.append(
                        (name, os.path.join(dbdir, dpath), path_name,
                         last, tval, enable, stock_id))

        self.current_names.sort()

    def get_family_tree_path(self, name):
        """
        Given a name, return None if name not existing or the path to the
        database if it is a known database name.
        """
        for data in self.current_names:
            if data[0] == name:
                return data[1]
        return None

    def family_tree_list(self):
        """Return a list of name, dirname of the known family trees
        """
        lst = [(x[0], x[1]) for x in self.current_names]
        return lst

    def __start_cursor(self, msg):
        """
        Do needed things to start import visually, eg busy cursor
        """
        print(_('Starting Import, %s') % msg)

    def __end_cursor(self):
        """
        Set end of a busy cursor
        """
        print(_('Import finished...'))

    def create_new_db_cli(self, title=None):
        """
        Create a new database.
        """


# This file is dual licensed under the terms of the Apache License, Version
# 2.0, and the BSD License. See the LICENSE file in the root of this repository
# for complete details.

from __future__ import absolute_import, division, print_function

from enum import Enum

import six

from cryptography import utils
from cryptography.x509.oid import NameOID, ObjectIdentifier

class _ASN1Type(Enum):
    UTF8String = 12
    NumericString = 18
    PrintableString = 19
    T61String = 20
    IA5String = 22
    UTCTime = 23
    GeneralizedTime = 24
    VisibleString = 26
    UniversalString = 28
    BMPString = 30

_ASN1_TYPE_TO_ENUM = dict((i.value, i) for i in _ASN1Type)
_SENTINEL = object()
_NAMEOID_DEFAULT_TYPE = {
    NameOID.COUNTRY_NAME: _ASN1Type.PrintableString,
    NameOID.JURISDICTION_COUNTRY_NAME: _ASN1Type.PrintableString,
    NameOID.SERIAL_NUMBER: _ASN1Type.PrintableString,
    NameOID.DN_QUALIFIER: _ASN1Type.PrintableString,
    NameOID.EMAIL_ADDRESS: _ASN1Type.IA5String,
    NameOID.DOMAIN_COMPONENT: _ASN1Type.IA5String,
}

#: Short attribute names from RFC 4514:
#: https://tools.ietf.org/html/rfc4514#page-7
_NAMEOID_TO_NAME = {
    NameOID.COMMON_NAME: 'CN',
    NameOID.LOCALITY_NAME: 'L',
    NameOID.STATE_OR_PROVINCE_NAME: 'ST',
    NameOID.ORGANIZATION_NAME: 'O',
    NameOID.ORGANIZATIONAL_UNIT_NAME: 'OU',
    NameOID.COUNTRY_NAME: 'C',
    NameOID.STREET_ADDRESS: 'STREET',
    NameOID.DOMAIN_COMPONENT: 'DC',
    NameOID.USER_ID: 'UID',
}

def _escape_dn_value(val):
    """Escape special characters in RFC4514 Distinguished Name value."""

    # See https://tools.ietf.org/html/rfc4514#section-2.4
    val = val.replace('\\', '\\\\')
    val = val.replace('"', '\\"')
    val = val.replace('+', '\\+')
    val = val.replace(',', '\\,')
    val = val.replace(';', '\\;')
    val = val.replace('<', '\\<')
    val = val.replace('>', '\\>')
    val = val.replace('\0', '\\00')

    if val[0] in ('#', ' '):
        val = '\\' + val
    if val[-1] == ' ':
        val = val[:-1] + '\\ '

    return val

class NameAttribute(object):
    def __init__(self, oid, value, _type=_SENTINEL):
        if not isinstance(oid, ObjectIdentifier):
            raise TypeError(
                "oid argument must be an ObjectIdentifier instance."
            )

        if not isinstance(value, six.text_type):
            raise TypeError(
                "value argument must be a text type."
            )

        if (
            oid == NameOID.COUNTRY_NAME or
            oid == NameOID.JURISDICTION_COUNTRY_NAME
        ):
            if len(value.encode("utf8")) != 2:
                raise ValueError(
                    "Country name must be a 2 character country code"
                )

        if len(value) == 0:
            raise ValueError("Value cannot be an empty string")

        # The appropriate ASN1 string type varies by OID and is defined across
        # multiple RFCs including 2459, 3280, and 5280. In general UTF8String
        # is preferred (2459), but 3280 and 5280 specify several OIDs with
        # alternate types. This means when we see the sentinel value we need
        # to look up whether the OID has a non-UTF8 type. If it does, set it
        # to that. Otherwise, UTF8!
        if _type == _SENTINEL:
            _type = _NAMEOID_DEFAULT_TYPE.get(oid, _ASN1Type.UTF8String)

        if not isinstance(_type, _ASN1Type):
            raise TypeError("_type must be from the _ASN1Type enum")

        self._oid = oid
        self._value = value
        self._type = _type

    oid = utils.read_only_property("_oid")
    value = utils.read_only_property("_value")

    def rfc4514_string(self):
        """
        Format as RFC4514 Distinguished Name string.

        Use short attribute name if available, otherwise fall back to OID
        dotted string.
        """
        key = _NAMEOID_TO_NAME.get(self.oid, self.oid.dotted_string)
        return '%s=%s' % (key, _escape_dn_value(self.value))

    def __eq__(self, other):
        if not isinstance(other, NameAttribute):
            return NotImplemented

        return (
            self.oid == other.oid and
            self.value == other.value
        )

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash((self.oid, self.value))

    def __repr__(self):
        return "<NameAttribute(oid={0.oid}, value={0.value!r})>".format(self)

class RelativeDistinguishedName(object):
    def __init__(self, attributes):
        attributes = list(attributes)
        if not attributes:
            raise ValueError("a relative distinguished name cannot be empty")
        if not all(isinstance(x, NameAttribute) for x in attributes):
            raise TypeError("attributes must be an iterable of NameAttribute")

        # Keep list and frozenset to preserve attribute order where it matters
        self._attributes = attributes
        self._attribute_set = frozenset(attributes)

        if len(self._attribute_set) != len(attributes):
            raise ValueError("duplicate attributes are not allowed")

    def get_attributes_for_oid(self, oid):
        return [i for i in self if i.oid == oid]

    def rfc4514_string(self):
        """
        Format as RFC4514 Distinguished Name string.

        Within each RDN, attributes are joined by '+', although that is rarely
        used in certificates.
        """
        return '+'.join(attr.rfc4514_string() for attr in self._attributes)

    def __eq__(self, other):
        if not isinstance(other, RelativeDistinguishedName):
            return NotImplemented

        return self._attribute_set == other._attribute_set

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._attribute_set)

    def __iter__(self):
        return iter(self._attributes)

    def __len__(self):
        return len(self._attributes)

    def __repr__(self):
        return "<RelativeDistinguishedName({})>".format(self.rfc4514_string())

class Name(object):
    def __init__(self, attributes):
        attributes = list(attributes)
        if all(isinstance(x, NameAttribute) for x in attributes):
            self._attributes = [
                RelativeDistinguishedName([x]) for x in attributes
            ]
        elif all(isinstance(x, RelativeDistinguishedName) for x in attributes):
            self._attributes = attributes
        else:
            raise TypeError(
                "attributes must be a list of NameAttribute"
                " or a list RelativeDistinguishedName"
            )

    def rfc4514_string(self):
        """
        Format as RFC4514 Distinguished Name string.
        For example 'CN=foobar.com,O=Foo Corp,C=US'

        An X.509 name is a two-level structure: a list of sets of attributes.
        Each list element is separated by ',' and within each list element, set
        elements are separated by '+'. The latter is almost never used in
        real world certificates.
        """
        return ','.join(attr.rfc4514_string() for attr in self._attributes)

    def get_attributes_for_oid(self, oid):
        return [i for i in self if i.oid == oid]

    @property
    def rdns(self):
        return self._attributes

    def public_bytes(self, backend):
        return backend.x509_name_bytes(self)

    def __eq__(self, other):


# -*- coding: utf-8 -*-
#
# (c) 2017 Red Hat, Inc.
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

# Make coding more python3-ish
from __future__ import absolute_import, division, print_function
__metaclass__ = type

import pytest

from ansible.module_utils.network.common.utils import to_list, sort_list
from ansible.module_utils.network.common.utils import dict_diff, dict_merge
from ansible.module_utils.network.common.utils import conditional, Template
from ansible.module_utils.network.common.utils import to_masklen, to_netmask, to_subnet, to_ipv6_network
from ansible.module_utils.network.common.utils import is_masklen, is_netmask

def test_to_list():
    for scalar in ('string', 1, True, False, None):
        assert isinstance(to_list(scalar), list)

    for container in ([1, 2, 3], {'one': 1}):
        assert isinstance(to_list(container), list)

    test_list = [1, 2, 3]
    assert id(test_list) != id(to_list(test_list))

def test_sort():
    data = [3, 1, 2]
    assert [1, 2, 3] == sort_list(data)

    string_data = '123'
    assert string_data == sort_list(string_data)

def test_dict_diff():
    base = dict(obj2=dict(), b1=True, b2=False, b3=False,
                one=1, two=2, three=3, obj1=dict(key1=1, key2=2),
                l1=[1, 3], l2=[1, 2, 3], l4=[4],
                nested=dict(n1=dict(n2=2)))

    other = dict(b1=True, b2=False, b3=True, b4=True,
                 one=1, three=4, four=4, obj1=dict(key1=2),
                 l1=[2, 1], l2=[3, 2, 1], l3=[1],
                 nested=dict(n1=dict(n2=2, n3=3)))

    result = dict_diff(base, other)

    # string assertions
    assert 'one' not in result
    assert 'two' not in result
    assert result['three'] == 4
    assert result['four'] == 4

    # dict assertions
    assert 'obj1' in result
    assert 'key1' in result['obj1']
    assert 'key2' not in result['obj1']

    # list assertions
    assert result['l1'] == [2, 1]
    assert 'l2' not in result
    assert result['l3'] == [1]
    assert 'l4' not in result

    # nested assertions
    assert 'obj1' in result
    assert result['obj1']['key1'] == 2
    assert 'key2' not in result['obj1']

    # bool assertions
    assert 'b1' not in result
    assert 'b2' not in result
    assert result['b3']
    assert result['b4']

def test_dict_merge():
    base = dict(obj2=dict(), b1=True, b2=False, b3=False,
                one=1, two=2, three=3, obj1=dict(key1=1, key2=2),
                l1=[1, 3], l2=[1, 2, 3], l4=[4],
                nested=dict(n1=dict(n2=2)))

    other = dict(b1=True, b2=False, b3=True, b4=True,
                 one=1, three=4, four=4, obj1=dict(key1=2),
                 l1=[2, 1], l2=[3, 2, 1], l3=[1],
                 nested=dict(n1=dict(n2=2, n3=3)))

    result = dict_merge(base, other)

    # string assertions
    assert 'one' in result
    assert 'two' in result
    assert result['three'] == 4
    assert result['four'] == 4

    # dict assertions
    assert 'obj1' in result
    assert 'key1' in result['obj1']
    assert 'key2' in result['obj1']

    # list assertions
    assert result['l1'] == [1, 2, 3]
    assert 'l2' in result
    assert result['l3'] == [1]
    assert 'l4' in result

    # nested assertions
    assert 'obj1' in result
    assert result['obj1']['key1'] == 2
    assert 'key2' in result['obj1']

    # bool assertions
    assert 'b1' in result
    assert 'b2' in result
    assert result['b3']
    assert result['b4']

def test_conditional():
    assert conditional(10, 10)
    assert conditional('10', '10')
    assert conditional('foo', 'foo')
    assert conditional(True, True)
    assert conditional(False, False)
    assert conditional(None, None)
    assert conditional("ge(1)", 1)
    assert conditional("gt(1)", 2)
    assert conditional("le(2)", 2)
    assert conditional("lt(3)", 2)
    assert conditional("eq(1)", 1)
    assert conditional("neq(0)", 1)
    assert conditional("min(1)", 1)
    assert conditional("max(1)", 1)
    assert conditional("exactly(1)", 1)

def test_template():
    tmpl = Template()
    assert 'foo' == tmpl('{{ test }}', {'test': 'foo'})

def test_to_masklen():
    assert 24 == to_masklen('255.255.255.0')

def test_to_masklen_invalid():
    with pytest.raises(ValueError):
        to_masklen('255')

def test_to_netmask():
    assert '255.0.0.0' == to_netmask(8)
    assert '255.0.0.0' == to_netmask('8')

def test_to_netmask_invalid():
    with pytest.raises(ValueError):
        to_netmask(128)

def test_to_subnet():
    result = to_subnet('192.168.1.1', 24)
    assert '192.168.1.0/24' == result

    result = to_subnet('192.168.1.1', 24, dotted_notation=True)
    assert '192.168.1.0 255.255.255.0' == result

def test_to_subnet_invalid():
    with pytest.raises(ValueError):
        to_subnet('foo', 'bar')

def test_is_masklen():
    assert is_masklen(32)
    assert not is_masklen(33)
    assert not is_masklen('foo')

def test_is_netmask():
    assert is_netmask('255.255.255.255')
    assert not is_netmask(24)
    assert not is_netmask('foo')

def test_to_ipv6_network():


import ast, astor, json
import sys
import os
import simplejson as json
#x = ast.parse(open("dec4.py").read())
from bs4 import BeautifulSoup
import bs4
class FirstParser(ast.NodeVisitor):
    def __init__(self):
        pass

    def continue_(self, stmt):
        '''Helper: parse a node's children'''
        return super(FirstParser, self).generic_visit(stmt)

    def parse(self, code):
        '''Parse text into a tree and walk the result'''
        tree = ast.parse(code)
        self.visit(tree)
    def visit(self, node):
        """Dispatches the call to visit_$nodetype method."""
        nodetype = node.__class__.__name__
        f = getattr(self, "visit_" + nodetype, self.generic_visit)
        return f(node)
    def visit_ImportFrom(self, stmt_module):
        # retrieve the name from the returned object
        # normally, there is just a single alias
        listofimports = []
        for imp in stmt_module.names:
            listofimports.append(imp.name)

        if len(listofimports) == 1:
            stmt_module.js = ('import '+ ','.join(listofimports) + ' from \''+ str(stmt_module.module).replace('.','/') + '\';')

        else:
            stmt_module.js = ('import { '+ ','.join(listofimports) + '} from \''+ str(stmt_module.module).replace('.','/') + '\';')
        pass
        #self.continue_(stmt_module)

    def visit_ClassDef(self, node):
        scope = {'isclass':True}
        for b in node.body:
            b._scope = scope
        self.continue_(node)
        r = ''
        for call in node.decorator_list:

            r = r + ('@'+call.func.id+'('+','.join([ arg.js for arg  in call.args])+')') + '\n'

        r = r + ('class '+node.name+' { \n')
        r = r + (';\n').join([b.js for b in node.body]) +  '\n'

        r = r+'}'

        node.js = r

    def visit_Dict(self, stmt_dict):
        d = {}
        self.continue_(stmt_dict)
        for k,v in zip(stmt_dict.keys, stmt_dict.values):
            #d[k.s] = v.s
            #print (v)
            self.continue_(v)
        for k,v in zip(stmt_dict.keys, stmt_dict.values):
            #d[k.s] = v.s
            d[k.js] = v.js
        stmt_dict.js = json.dumps(d)
        #stmt_dict.js=stmt_dict.js.replace('\\\"', '')

        res = []
        for k,v in zip(stmt_dict.keys, stmt_dict.values):
            #d[k.s] = v.s
            #print (v)
            res.append(k.js+":" + v.js )
        #stmt_dict.js = json.dumps(res)
        #stmt_dict.js=stmt_dict.js.replace('\\\"', '')
        stmt_dict.js = '{'+','.join(res)+'}'
        return stmt_dict

    def visit_List(self, node):
        d = []
        (self.continue_(node))

        for v in node.elts:
            d.append( v.js)
        node.js = '['+','.join(d)+']'

    def visit_Module(self, stmt_module):
        scope = {'variables':[]}

        for b in stmt_module.body:
            b._scope = scope
            #self.continue_(b)
        (self.continue_(stmt_module))
        for b in stmt_module.body:
            print (b.js)

    def visit_Assign(self, node):

        self.continue_(node)
        node.js = ''
        node.alreadyassigned = []
        lfs = ''
        llist = []
        replace_self_with_this = False
        if hasattr(node, '_scope'):
            if 'isclass' in node._scope:
                replace_self_with_this = node._scope['isclass']
        for t in node.targets:
            if isinstance( t, ast.Attribute) :
                #llist.append(('this' if replace_self_with_this else t.value.id )+'.'+t.attr)
                llist.append(('this' if replace_self_with_this else t.value.js )+'.'+t.attr)
            else:
                if not t.id in node._scope['variables']:
                    lfs = 'var ' + t.id #+ ' : any'

                    node._scope['variables'].append(t.id)
                else:
                    llist.append(t.id)

        leftside = lfs + (','.join(llist))
        rightside = (node.value.js)
        node.js= (str(leftside) + ' = '+ str(rightside) + ' ;')

    def visit_Num(self, node):
        self.continue_(node)
        node.js = (node.n)
    def visit_Str(self, node):

        self.continue_(node)
        node.js = '"'+str(node.s)+'"'
        #node.js = node.s
        node.jsvalue = node.s
    def visit_Name(self, node):
        self.continue_(node)
        node.js = str(node.id)
    def visit_NameConstant(self, node):
        self.continue_(node)
        node.js = str(node.value)
        node.jsvalue = node.value
        if str(node.value) == 'True':
            node.js = 'true'
        if str(node.value) == 'False':
            node.js = 'false'
    def visit_Expr(self, node):
        self.continue_(node)
        node.js = (node.value.js)
    def visit_Attribute(self, node):
        self.continue_(node)
        #node.js = (str(node.value.id) if hasattr(node.value, 'id') else '')+"." +str(node.attr)
        #print (node.attr)
        #node.js = (str(node.value.id) if hasattr(node.value, 'id') else node.attr)+"." +str(node.attr)
        #print (str(node.attr))
        node.js = node.value.js + '.' +  node.attr
        node.jsvalue = node.js
        #print (node.js)
    def visit_Return(self, node):
        self.continue_(node)

        node.js = ''
        if (hasattr(node.value, 'js')):
            rightside = (node.value.js)
            node.js= 'return '+str(rightside) + ';'

    def visit_Call(self, node):
        self.continue_(node)
        funcname = node.func.js
        #JSX plug
        if funcname == 'bb_jsx':
            funcname = ''
            node.js = funcname + ''   +''+','.join([(arg.jsvalue) for arg in node.args])+''
        elif funcname == 'bb_export':
            node.js = 'export ' +','.join([str(arg.jsvalue) for arg in node.args])+';'
        else:
        #node.js = str(node.func.id if type(node.func)!=ast.Attribute else node.func.value.id)   +'('+','.join([arg.js for arg in node.args])+')'
            node.js = funcname + ''   +'('+','.join([str(arg.js) for arg in node.args])+')'

    def visit_FunctionDef(self, node):
        scope = {'variables':[]}
        skip_first_arg = False
        if hasattr(node, '_scope'):
            scope = dict(scope, **node._scope)
            if 'isclass' in node._scope:
                skip_first_arg = node._scope['isclass']
        for a in node.body:
            a._scope = scope
            #self.continue_(a)

        self.continue_(node)
        funcname = ('' if skip_first_arg else "function ") + node.name

        if node.name == '__init__':
            funcname = "constructor"
        node.js = funcname +'('+','.join([arg.arg for arg in node.args.args[1 if skip_first_arg else 0:]])+')  { '
        for a in node.body:

            node.js += '\n   '+(a.js)  + '\n'


import sys
sys.path.append("../")
import unittest
import AcraNetwork.Chapter7 as ch7
import AcraNetwork.SimpleEthernet as eth
import AcraNetwork.Pcap as pcap
import os
import copy
import struct
import logging
from pstats import Stats
import cProfile

THIS_DIR = os.path.dirname(os.path.abspath(__file__))

def buf_generator(count, llp_count=0):
    running_count = 0
    while running_count < count:
        buf_len = os.urandom((running_count+1) * 128)
        running_count += 1
        if running_count <= llp_count:
            low_latency = True
        else:
            low_latency = False
        yield buf_len, low_latency

class TestCaseCh7(unittest.TestCase):

    def test_basic(self):
        ch7_pkt = ch7.PDFR()
        ch7_pkt.length = 132
        ch7_pkt.llp = 0
        remainder = bytes()
        while remainder == bytes():
            ch7_pd = ch7.PTDP()
            ch7_pd.content = ch7.PTDP_CONTENT_MAC
            ch7_pd.fragment = ch7.PTDP_FRAGMENT_COMPLETE
            ch7_pd.payload = os.urandom(58)
            remainder = ch7_pkt.add_payload(ch7_pd.pack())
            #print("Packet added")
        buf = ch7_pkt.pack()
        #print("Len Buf={} Len Rem={}".format(len(buf), len(remainder)))
        # Check the comparsion
        ch7_pkt_copy = copy.deepcopy(ch7_pkt)
        self.assertTrue(ch7_pkt==ch7_pkt_copy)

        ch7_unpack = ch7.PDFR()
        ch7_unpack.length = 132
        self.assertTrue(ch7_unpack.unpack(buf))
        self.assertTrue(ch7_unpack==ch7_pkt)

    def test_partial_fill(self):
        ch7_pkt = ch7.PDFR()
        ch7_pkt.length = 140
        ch7_pkt.llp = 0
        remainder = bytes()
        while remainder == bytes():
            ch7_pd = ch7.PTDP()
            ch7_pd.content = ch7.PTDP_CONTENT_MAC
            ch7_pd.fragment = ch7.PTDP_FRAGMENT_COMPLETE
            ch7_pd.payload = os.urandom(58)
            remainder = ch7_pkt.add_payload(ch7_pd.pack())
            #print("Packet added")
        buf = ch7_pkt.pack()
        ch7_unpack = ch7.PDFR()
        ch7_unpack.length = 140
        self.assertTrue(ch7_unpack.unpack(buf))
        #print(repr(ch7_pkt))
        #print(repr(ch7_unpack))
        self.assertEqual(52, len(remainder))
        self.assertTrue(ch7_unpack == ch7_pkt)

    def test_comparsion(self):
        ch7_pd = ch7.PTDP()
        ch7_pd.content = ch7.PTDP_CONTENT_MAC
        ch7_pd.fragment = ch7.PTDP_FRAGMENT_COMPLETE
        ch7_pd.payload = os.urandom(60)
        ch_pd_copy = copy.deepcopy(ch7_pd)
        self.assertTrue(ch7_pd==ch_pd_copy)

class TestGenerators(unittest.TestCase):

    def test_ptdp_generator(self):
        for ptdp_pkt in ch7.datapkts_to_ptdp(buf_generator(5)):
            #print(repr(ptdp_pkt))
            self.assertIsInstance(ptdp_pkt, ch7.PTDP)

    def test_pdfr_geenerator(self):
        remainder = bytes()
        for pdfr in ch7.datapkts_to_pdfr(buf_generator(5), pdfr_len=200):
            #print(repr(pdfr))
            ch7_pkt = ch7.PDFR()
            ch7_pkt.length = 800
            ch7_pkt.unpack(pdfr.pack())
            for (p, remainder, e) in ch7_pkt.get_aligned_payload(remainder):
                if p is None:
                    continue
                else:
                    #print(repr(p))
                    self.assertEqual(0, p.length % 128)

class TestRealEthernet(unittest.TestCase):

    def setUp(self):
        self.pr = cProfile.Profile()
        self.pr.enable()

    def tearDown(self):
        p = Stats(self.pr)
        p.sort_stats('cumtime')
        #p.print_stats()

    pkts_sent = []
    @staticmethod
    def eth_gen(count, low_latency_pkts=None, size_mult=128):
        """

        :param count:
        :param low_latency_pkts:
        :param size_mult:
        :rtype:  collections.Iterable[bytes, bool]
        """

        pf = pcap.Pcap(THIS_DIR+"/generated_eth.pcap", mode="w")
        r = pcap.PcapRecord()
        for i in range(1, count + 1):
            e = eth.Ethernet()
            e.dstmac = 0xABABABABABAB
            e.srcmac = 0xDCDCDCDCDCDC
            e.type = e.TYPE_IP
            ip = eth.IP()
            ip.srcip = "192.168.28.3"
            ip.dstip = "192.168.28.1"
            u = eth.UDP()
            u.dstport = i
            u.srcport = i
            if size_mult == 0:
                u.payload = struct.pack(">{}H".format(64), *([0xA5A5] * (64)))
            else:
                u.payload = struct.pack(">{}H".format(i * size_mult), *([0xA5A5] * (i * size_mult)))
            ip.payload = u.pack()
            e.payload = ip.pack()

            if low_latency_pkts is None:
                low_latency = False
            elif i in low_latency_pkts:
                low_latency = True
            else:
                low_latency = False
            r.setCurrentTime()
            r.payload = e.pack()
            pf.write(r)
            TestRealEthernet.pkts_sent.append(e.pack())
            yield e.pack(), low_latency
        pf.close()

    def test_eth_in_packets(self):
        pf = pcap.Pcap(THIS_DIR+"/gen.pcap", mode="w")
        r = pcap.PcapRecord()
        remainder = bytes()
        eth_p = bytes()
        pkt_count = 1
        for pdfr in ch7.datapkts_to_pdfr(TestRealEthernet.eth_gen(3), pdfr_len=200):
            ch7_pkt = ch7.PDFR()
            ch7_pkt.length = 200
            ch7_pkt.unpack(pdfr.pack())
            for (p, remainder, e) in ch7_pkt.get_aligned_payload(remainder):
                if p is None:
                    continue
                else:
                    eth_p += p.payload
                    if p.fragment == ch7.PTDP_FRAGMENT_COMPLETE or  p.fragment == ch7.PTDP_FRAGMENT_LAST:
                        r.setCurrentTime()
                        r.payload = eth_p
                        # Verify the size of the packets and it's probably good. I could unpack them here too
                        self.assertEqual(pkt_count*2*128+eth.UDP.UDP_HEADER_SIZE+eth.IP.IP_HEADER_SIZE+eth.Ethernet.HEADERLEN, len(eth_p))
                        pkt_count += 1
                        pf.write(r)
                        eth_p = bytes()

                    #self.assertEqual(0, p.length % 128)
                    #self.assertTrue(p.length <= 1024)
        pf.close()

    def test_eth_in_packets_low_latency(self):
        #logging.basicConfig(level=logging.DEBUG)
        pf = pcap.Pcap(THIS_DIR+"/captured_llp.pcap", mode="w")
        r = pcap.PcapRecord()
        remainder = bytes()
        eth_p = bytes()
        pkt_count = 1
        ptfr_idx = 0
        ptdp_idx = 0
        pkt_size_mult = 16
        for pdfr in ch7.datapkts_to_pdfr(TestRealEthernet.eth_gen(20, low_latency_pkts=[2, 4, 10], size_mult=8), pdfr_len=400):
            ch7_pkt = ch7.PDFR()
            ch7_pkt.length = 400
            ch7_pkt.unpack(pdfr.pack())
            #b = open("gen_llp_{}.bin".format(ptfr_idx), mode="wb")
            ptfr_idx += 1
            #b.write(pdfr.pack())
            #b.close()
            #print(repr(pdfr))
            for (p, remainder, e) in ch7_pkt.get_aligned_payload(remainder):
                if p is None:
                    continue
                else:
                    #print(repr(p))
                    eth_p += p.payload
                    if p.fragment == ch7.PTDP_FRAGMENT_COMPLETE or  p.fragment == ch7.PTDP_FRAGMENT_LAST:
                        if ptdp_idx == 0 :
                            self.assertEqual(p.low_latency, True)
                        r.setCurrentTime()
                        r.payload = eth_p
                        # Verify the size of the packets and it's probably good. I could unpack them here too
                        #self.assertEqual(2*64+eth.UDP.UDP_HEADER_SIZE+eth.IP.IP_HEADER_SIZE+eth.Ethernet.HEADERLEN, len(eth_p))
                        if eth_p not in TestRealEthernet.pkts_sent:
                            self.assertTrue(False)
                        pkt_count += 1
                        pf.write(r)
                        eth_p = bytes()
                    ptdp_idx += 1
        self.assertEqual(pkt_count, 20)

                    #self.assertEqual(0, p.length % 128)
                    #self.assertTrue(p.length <= 1024)
        pf.close()

    def test_llc(self):
        llc_count = 0
        fill_count = 0
        mac_count = 0
        logging.basicConfig(level=logging.WARN)
        remainder = b""
        for i in range(3, 6):
            f = open(THIS_DIR+"/ptfr_{}.bin".format(i), "rb")
            ptfr_data = f.read()
            f.close()

            ch7_pkt = ch7.PDFR()
            ch7_pkt.length = len(ptfr_data)
            eth_p = b""
            ch7_pkt.unpack(ptfr_data)
            #print repr(ch7_pkt)


# -*- coding: utf-8 -*-

# Copyright: (c) 2015, Matt Martz <matt@sivel.net>
# Copyright: (c) 2015, Rackspace US, Inc.
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

import re

from voluptuous import ALLOW_EXTRA, PREVENT_EXTRA, All, Any, Invalid, Length, Required, Schema, Self, ValueInvalid
from ansible.module_utils.six import string_types
from ansible.module_utils.common.collections import is_iterable

list_string_types = list(string_types)
tuple_string_types = tuple(string_types)
any_string_types = Any(*string_types)

# Valid DOCUMENTATION.author lines
# Based on Ansibulbot's extract_github_id()
#   author: First Last (@name) [optional anything]
#     "Ansible Core Team" - Used by the Bot
#     "Michael DeHaan" - nop
#     "Name (!UNKNOWN)" - For the few untraceable authors
author_line = re.compile(r'^\w.*(\(@([\w-]+)\)|!UNKNOWN)(?![\w.])|^Ansible Core Team$|^Michael DeHaan$')

def is_callable(v):
    if not callable(v):
        raise ValueInvalid('not a valid value')
    return v

def sequence_of_sequences(min=None, max=None):
    return All(
        Any(
            None,
            [Any(list, tuple)],
            tuple([Any(list, tuple)]),
        ),
        Any(
            None,
            [Length(min=min, max=max)],
            tuple([Length(min=min, max=max)]),
        ),
    )

seealso_schema = Schema(
    [
        Any(
            {
                Required('module'): Any(*string_types),
                'description': Any(*string_types),
            },
            {
                Required('ref'): Any(*string_types),
                Required('description'): Any(*string_types),
            },
            {
                Required('name'): Any(*string_types),
                Required('link'): Any(*string_types),
                Required('description'): Any(*string_types),
            },
        ),
    ]
)

argument_spec_types = ['bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw',
                       'sid', 'str']

argument_spec_modifiers = {
    'mutually_exclusive': sequence_of_sequences(min=2),
    'required_together': sequence_of_sequences(min=2),
    'required_one_of': sequence_of_sequences(min=2),
    'required_if': sequence_of_sequences(min=3),
    'required_by': Schema({str: Any(list_string_types, tuple_string_types, *string_types)}),
}

def no_required_with_default(v):
    if v.get('default') and v.get('required'):
        raise Invalid('required=True cannot be supplied with a default')
    return v

def elements_with_list(v):
    if v.get('elements') and v.get('type') != 'list':
        raise Invalid('type must be list to use elements')
    return v

def options_with_apply_defaults(v):
    if v.get('apply_defaults') and not v.get('options'):
        raise Invalid('apply_defaults=True requires options to be set')
    return v

def argument_spec_schema():
    any_string_types = Any(*string_types)
    schema = {
        any_string_types: {
            'type': Any(is_callable, *argument_spec_types),
            'elements': Any(*argument_spec_types),
            'default': object,
            'fallback': Any(
                (is_callable, list_string_types),
                [is_callable, list_string_types],
            ),
            'choices': Any([object], (object,)),
            'required': bool,
            'no_log': bool,
            'aliases': Any(list_string_types, tuple(list_string_types)),
            'apply_defaults': bool,
            'removed_in_version': Any(float, *string_types),
            'options': Self,
            'deprecated_aliases': Any([
                {
                    Required('name'): Any(*string_types),
                    Required('version'): Any(float, *string_types),
                },
            ]),
        }
    }
    schema[any_string_types].update(argument_spec_modifiers)
    schemas = All(
        schema,
        Schema({any_string_types: no_required_with_default}),
        Schema({any_string_types: elements_with_list}),
        Schema({any_string_types: options_with_apply_defaults}),
    )
    return Schema(schemas)

def ansible_module_kwargs_schema():
    schema = {
        'argument_spec': argument_spec_schema(),
        'bypass_checks': bool,
        'no_log': bool,
        'check_invalid_arguments': Any(None, bool),
        'add_file_common_args': bool,
        'supports_check_mode': bool,
    }
    schema.update(argument_spec_modifiers)
    return Schema(schema)

suboption_schema = Schema(
    {
        Required('description'): Any(list_string_types, *string_types),
        'required': bool,
        'choices': list,
        'aliases': Any(list_string_types),
        'version_added': Any(float, *string_types),
        'default': Any(None, float, int, bool, list, dict, *string_types),
        # Note: Types are strings, not literal bools, such as True or False
        'type': Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str'),
        # in case of type='list' elements define type of individual item in list
        'elements': Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str'),
        # Recursive suboptions
        'suboptions': Any(None, *list({str_type: Self} for str_type in string_types)),
    },
    extra=PREVENT_EXTRA
)

# This generates list of dicts with keys from string_types and suboption_schema value
# for example in Python 3: {str: suboption_schema}
list_dict_suboption_schema = [{str_type: suboption_schema} for str_type in string_types]

option_schema = Schema(
    {
        Required('description'): Any(list_string_types, *string_types),
        'required': bool,
        'choices': list,
        'aliases': Any(list_string_types),
        'version_added': Any(float, *string_types),
        'default': Any(None, float, int, bool, list, dict, *string_types),
        'suboptions': Any(None, *list_dict_suboption_schema),
        # Note: Types are strings, not literal bools, such as True or False
        'type': Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str'),
        # in case of type='list' elements define type of individual item in list
        'elements': Any(None, 'bits', 'bool', 'bytes', 'dict', 'float', 'int', 'json', 'jsonarg', 'list', 'path', 'raw', 'sid', 'str'),
    },
    extra=PREVENT_EXTRA
)

# This generates list of dicts with keys from string_types and option_schema value
# for example in Python 3: {str: option_schema}
list_dict_option_schema = [{str_type: option_schema} for str_type in string_types]

def return_contains(v):
    schema = Schema(
        {
            Required('contains'): Any(dict, list, *string_types)
        },
        extra=ALLOW_EXTRA
    )
    if v.get('type') == 'complex':
        return schema(v)
    return v

return_contains_schema = Any(
    All(
        Schema(
            {


import networkx as nx
import matplotlib.pyplot as plt
from geopy.distance import vincenty
import random

#minimum number of nodes required for communication
# 1- CC; 1- convoy and 1 - Boat
minNumNodes = 3
minTransRange = 300
minViewRange = 50

# total number of times
tn = 50

def set_edges(graph):

    for eachNode in graph.nodes():

        for nextNode in graph.nodes():

            # not same node in two loops and no conncetion
            if (eachNode != nextNode) and \
             not graph.has_edge(eachNode, nextNode):

                # for Command Center node
                 if(graph.node[eachNode]["type"] == "CC"):
                     if (graph.node[nextNode]["type"] == "CC") or \
                             ( (graph.node[nextNode]["type"] == "Uboat") \
                                   and (graph.node[nextNode]["listeningZone"] == 0) ):
                        graph.add_edge(eachNode,nextNode)
                        graph.edge[eachNode][nextNode]["weight"] = 1

                # for Uboat node
                 elif(graph.node[eachNode]["type"] == "Uboat"):

                     eachNodeLoc= (graph.node[eachNode]["lat"],graph.node[eachNode]["lon"])
                     nextNodeLoc= (graph.node[nextNode]["lat"],graph.node[nextNode]["lon"])

                     if(graph.node[nextNode]["type"] == "Uboat") and \
                        ( vincenty(eachNodeLoc, nextNodeLoc).kilometers <= minTransRange ):
                        graph.add_edge(eachNode,nextNode)
                        graph.edge[eachNode][nextNode]["weight"] = 1

                     elif(graph.node[nextNode]["type"] == "convoy") and \
                          (vincenty(eachNodeLoc, nextNodeLoc).kilometers <= minViewRange ):
                            graph.add_edge(eachNode,nextNode)
                            graph.edge[eachNode][nextNode]["weight"] = 1
    return graph

def set_CC(graph, CCNode):

    # Sanity checks for input
    if(graph.number_of_nodes() < minNumNodes):
        raise ValueError("Graph doesnt contain sufficient number of nodes to continue.")

    if not graph.has_node(CCNode):
        raise ValueError("CC node " + str(CCNode) + " not present")

    #CC Setup
    graph.node[CCNode]["lat"]=47.74
    graph.node[CCNode]["lon"]=-3.36
    graph.node[CCNode]["type"] = "CC"

    return graph

def set_convoy(graph, ConvoyNode, location):

    # Sanity checks for input
    if not graph.has_node(ConvoyNode):
        raise ValueError("Convoy Node " + str(ConvoyNode) + " not present")

    #Convoy setup
    graph.node[ConvoyNode]["lat"] = location[0]
    graph.node[ConvoyNode]["lon"] = location[1]
    graph.node[ConvoyNode]["type"] = "convoy"

    return graph

def set_Uboats(graph, numOfUboats, UboatLocations=()):

    # Sanity checks for input
    if(graph.number_of_nodes() < numOfUboats):
        raise ValueError("Graph doesnt contain sufficient number of Uboat nodes to continue.")

    # check enough location points sent for Uboats
    if (len(UboatLocations) != numOfUboats):
        raise ValueError("Enough Locations not provided for Uboats.")

    #Uboat setup
    UboatCount = 0

    for eachNode in graph.nodes():
        if(graph.node[eachNode]["type"] == ""):
            graph.node[eachNode]["lat"] = UboatLocations[UboatCount][0]
            graph.node[eachNode]["lon"] = UboatLocations[UboatCount][1]
            graph.node[eachNode]["type"] = "Uboat"
            graph.node[eachNode]["listeningZone"] = 1
            UboatCount = UboatCount+1

        if(UboatCount == numOfUboats):
            graph.node[eachNode]["listeningZone"] = 0
            break

    return graph

def initalize(graph):

    for eachNode in graph.nodes():
        graph.node[eachNode]["lat"] = 0.0
        graph.node[eachNode]["lon"] = 0.0
        graph.node[eachNode]["type"] = ""
        graph.node[eachNode]["state"] = "receive"
        graph.node[eachNode]["listeningZone"] = 0

    return graph

def get_graph(numOfNodes=16):
    graph = nx.Graph()

    # Generate a graph and intialize node attributes
    graph.add_nodes_from(range(numOfNodes))
    graph = initalize(graph)

    #Set Nodes as CC location
    graph = set_CC(graph,graph.nodes()[0])

    #set Nodes as Uboats
    UboatLocations = ([52.48, -59.28], # Node -2
                      [52.00, -55.78], # Node -3
                      [54.50, -57.78], # Node -4
                      [53.68, -54.28], # Node -5
                      [53.48, -50.58], # Node -6
                      [53.00, -46.98], # Node -7
                      [52.98, -43.26], # Node -8
                      [51.50, -39.65], # Node -9
                      [52.45, -35.98], # Node -10
                      [52.43, -32.31], # Node -11
                      [51.10, -28.67], # Node -12
                      [54.18, -28.96], # Node -13
                      [52.91, -35.21], # Node -14
                      [52.50, -26.00]) # Node -15

    numOfUboats = 14

    graph = set_Uboats(graph, numOfUboats, UboatLocations)

    #set convoy node
    graph = set_convoy(graph,graph.nodes()[-1],(52.48,-59.98))

    #set edges
    graph = set_edges(graph)

    return graph

def gen_rand_coord():

    lon=random.uniform(-63.44, -5.93)
    lat=random.uniform(60.52,42.09)
    return((lat,lon))

def gen_rand_coord_dist(ptlat, ptlon, distrange):

    flag=0

    while(flag!=1):

        lon=random.uniform(-63.44, -5.93)
        lat=random.uniform(ptlat,42.09)

        dist=vincenty((lat,lon),(ptlat,ptlon)).kilometers

        if(distrange==50 and (40<=dist<=50)):
            flag=1
        elif(distrange==300 and (250<=dist<=300)):
            flag=1
        else:
            flag=0

    return ((lat,lon))

def draw_graph(graph):

    pos = {}

    for eachNode in graph.nodes():
        pos[eachNode]=(graph.node[eachNode]["lon"],graph.node[eachNode]["lat"])

    # get three subsets of nodes: susceptible, infected, removed
    convoy_node = [n for n in graph.nodes() if graph.node[n]["type"] == "convoy"]
    CC_nodes = [n for n in graph.nodes() if graph.node[n]["type"] == "CC"]
    Uboat_lz_nodes = [n for n in graph.nodes() if ( graph.node[n]["type"] == "Uboat" and \
                     graph.node[n]["listeningZone"] == 1 ) ]
    Uboat_nlz_nodes = [n for n in graph.nodes() if ( graph.node[n]["type"] == "Uboat" and \
                     graph.node[n]["listeningZone"] == 0 ) ]
    empty_nodes = [n for n in graph.nodes() if graph.node[n]["type"] == ""]

    # draw edges, then draw each subset in different colour
    nx.draw_networkx_nodes(graph, pos, nodelist=convoy_node, node_color="r")
    nx.draw_networkx_nodes(graph, pos, nodelist=CC_nodes, node_color="c")
    nx.draw_networkx_nodes(graph, pos, nodelist=Uboat_lz_nodes, node_color="y")
    nx.draw_networkx_nodes(graph, pos, nodelist=Uboat_nlz_nodes, node_color="g")
    nx.draw_networkx_nodes(graph, pos, nodelist=empty_nodes, node_color="w")

    nx.draw_networkx_labels(graph, pos)

    nx.draw_networkx_edges(graph, pos)
    nx.draw_networkx_edge_labels(graph, pos)

    plt.savefig("UboatSurvival.png")
    plt.close()

def draw_graphER(graph):

    pos = nx.layout.random_layout(graph)

    # get three subsets of nodes: susceptible, infected, removed
    convoy_node = [n for n in graph.nodes() if graph.node[n]["type"] == "convoy"]
    CC_nodes = [n for n in graph.nodes() if graph.node[n]["type"] == "CC"]
    Uboat_lz_nodes = [n for n in graph.nodes() if ( graph.node[n]["type"] == "Uboat" and \
                     graph.node[n]["listeningZone"] == 1 ) ]
    Uboat_nlz_nodes = [n for n in graph.nodes() if ( graph.node[n]["type"] == "Uboat" and \
                     graph.node[n]["listeningZone"] == 0 ) ]
    empty_nodes = [n for n in graph.nodes() if graph.node[n]["type"] == ""]

    # draw edges, then draw each subset in different colour
    nx.draw_networkx_nodes(graph, pos, nodelist=convoy_node, node_color="r")
    nx.draw_networkx_nodes(graph, pos, nodelist=CC_nodes, node_color="c")
    nx.draw_networkx_nodes(graph, pos, nodelist=Uboat_lz_nodes, node_color="y")
    nx.draw_networkx_nodes(graph, pos, nodelist=Uboat_nlz_nodes, node_color="g")
    nx.draw_networkx_nodes(graph, pos, nodelist=empty_nodes, node_color="w")

    nx.draw_networkx_labels(graph, pos)
    nx.draw_networkx_edges(graph, pos)

    plt.savefig("UboatSurvivalER.png")
    plt.close()

def get_erdos_renyi_graph(numofNodes=16, edgeProb = 0.2):

    oceanGraph = nx.erdos_renyi_graph(numofNodes - minNumNodes, edgeProb)
    oceanGraph = initalize(oceanGraph)

    #Set Land Graph


""" Multiple, deformable parts classifier.
"""
import numpy as np
import cv2
import theano
from itertools import product

from matching import compile_batch_match, best_response_subwin
from dpm import DPM, vectortodpm
from warpclassifier import WarpClassifier
from latent_mlr import LatentMLR
from features import max_energy_subwindow, warped_fmaps_simple, warped_fmaps_dimred, Combine, BGRHist, HoG, compute_pyramids
from classifier import ClassifierMixin
from dataset_transform import random_windows_fmaps

def _init_dpm(warpmap, nbparts, partsize):
    """ Initializes a DPM by greedily taking high energy subwindows
        in a warped feature map.
    """
    initparts = []
    initanchors = []
    initdeforms = [np.array([0,0,0.01,0.01])] * nbparts
    warpcopy = np.array(warpmap, copy=True)

    for i in range(nbparts):
        # Take the highest energy subwindow as part, zero it out
        # in the feature map, and start again.
        (maxsubwin, (ai, aj)) = max_energy_subwindow(
            warpcopy,
            partsize
        )
        initparts.append(warpmap[ai:ai+partsize,aj:aj+partsize])
        initanchors.append(np.array([ai, aj]))
        warpcopy[ai:ai+partsize,aj:aj+partsize] = 0
    return DPM(initparts, initanchors, initdeforms)

# Another hack to get around pickling restrictions.
_batch_match = None

def _best_matches(beta, pyramids, labels, args):
    global _batch_match
    dpm_size = args['size']
    deform_factor = args['df']
    cst_deform = args['cst_deform']
    max_dims = args['max_dims']
    nb_samples = len(pyramids[0])
    assert len(pyramids) == len(max_dims)
    nb_scales = len(max_dims)
    # Compile the batch matching function if it hasn't already
    # been done. One matching function for each scale
    if _batch_match == None:
        _batch_match = []
        for fmaps in pyramids:
            _batch_match.append(compile_batch_match(fmaps))
    nb_features, nb_classes = beta.shape

    # Concatenate all the filters from all the DPMs into one big list
    # for batch cross-correlation.
    dpms = []
    filters = []
    for i in range(nb_classes):
        dpm = vectortodpm(beta[:, i], dpm_size)
        dpms.append(dpm)
        filters += dpm.parts

    # Run batch cross-correlation for each individual scale.
    responses_per_scale = []
    for i in range(nb_scales):
        responses_per_scale.append(_batch_match[i](filters))

    # Compute the corresponding subwindows and displacements for each
    # part of each DPM on each sample using GDT, and store the corresponding
    # latent vectors into the 4D tensor required by latent MLR.
    nb_parts = len(dpms[0].parts)
    latents = np.empty(
        [nb_classes, nb_samples, nb_features],
        dtype=theano.config.floatX
    )
    lat_cst = np.zeros([nb_classes, nb_samples], theano.config.floatX)
    partsize = dpms[0].parts[0].shape[0]
    max_scale = np.max(max_dims)
    scale_choices = np.zeros([nb_scales])

    for i in range(nb_samples):
        for j in range(nb_classes):
            subwins_and_disps = []
            for k in range(nb_parts):
                max_score = None
                max_score_subwin_disp = None
                # Get the subwindows and displacements for the best matching scale.
                for s in range(nb_scales):
                    # Compute new anchors and deformation scaling for the current scale.
                    response = responses_per_scale[s][i,j*nb_parts + k,:,:]
                    scale_factor = float(max_dims[s]) / max_scale
                    anci, ancj = np.floor(scale_factor * dpms[j].anchors[k]).astype(np.int32)
                    frows, fcols = response.shape[0:2]
                    # Clamp the scaled anchor just in case.
                    anchors = np.array([max(0, min(frows-1, anci)),
                                        max(0, min(fcols-1, ancj))])
                    # Compute the subwindow and corresponding displacement.
                    (score, subwin, di, dj) = best_response_subwin(
                        response,
                        pyramids[s][i],
                        anchors,
                        dpms[j].deforms[k] if cst_deform == None else cst_deform,
                        partsize,
                        deform_factor * scale_factor,
                        debug=False
                    )
                    # Displacements need to be scaled back to original scale.
                    scaled_disp = np.round(
                        np.array([di, dj]).astype(np.float64) / scale_factor
                    ).astype(np.int32)
                    # Keep track of the best score.
                    if max_score == None or score > max_score:
                        max_score = score
                        max_score_subwin_disp = (subwin, scaled_disp[0], scaled_disp[1], s)
                subwins_and_disps.append(max_score_subwin_disp[0:3])
                scale_choices[max_score_subwin_disp[3]] += 1
            # Put the computed latent values into a proper latent vector.
            latvec = np.empty([nb_features])
            offset = 0
            subwins = [sad[0] for sad in subwins_and_disps]
            # Introduce the deformation factor into the displacements.
            disps = [(float(sad[1]) * deform_factor, float(sad[2]) * deform_factor)
                     for sad in subwins_and_disps]

            # Flatten subwindows.
            for subwin in subwins:
                flatwin = subwin.flatten('C')
                latvec[offset:offset+flatwin.size] = flatwin
                offset += flatwin.size
            if cst_deform == None:
                # Introduce the part displacements, with deformation factor.
                for disp in disps:
                    di, dj = disp
                    latvec[offset:offset+4] = -np.array([di, dj, di**2, dj**2])
                    offset = offset+4
            else:
                # Compute the deformation cost using the constant deformation
                # coefficients.
                for disp in disps:
                    di, dj = disp
                    dx, dy, dx2, dy2 = cst_deform
                    lat_cst[j,i] -= dy*di + dx*dj + dy2*di**2 + dx2*dj**2
            assert offset == nb_features
            # Put the computed latent vector into the tensor.
            latents[j,i] = latvec

    print "Scales chosen for " + repr(max_dims)
    print (scale_choices / np.sum(scale_choices))

    return (latents, lat_cst)

class BaseDPMClassifier:
    """ Multi-class DPM classifier based on latent multinomial
        logistic regression.
    """
    def __init__(self, C=0.1, feature=Combine(BGRHist((4,4,4),0),HoG(9,1)),
                 max_dims=[5,7,9,11], nbparts=4, deform_factor=1.,
                 nb_gd_iter=50, learning_rate=0.001,
                 inc_rate=1.2, dec_rate=0.5, partsize=5, cst_deform=None, use_pca=None,
                 verbose=False):
        self.C = C
        self.feature = feature
        self.max_dims = max_dims
        self.nbparts = nbparts
        self.deform_factor = deform_factor
        self.inc_rate = inc_rate
        self.dec_rate = dec_rate
        self.partsize = partsize
        self.nb_gd_iter = nb_gd_iter
        self.learning_rate = learning_rate
        self.cst_deform = cst_deform
        self.use_pca = use_pca
        self.verbose = verbose
        self.pca = None

    def train(self, samples, labels):
        """ Training procedure which takes precomputed feature maps as inputs.
            For efficiency purposes in grid search.
        """
        # Make sure we recompile the matching function.
        global _batch_match
        _batch_match = None
        # Initialize the model with a warping classifier, taking
        # high energy subwindows as parts. Scale is the rounded
        # mean scale across all scales of feature pyramids.
        warp_max_dim = int(round(np.mean(self.max_dims)))
        print "Warping max dimension: " + repr(warp_max_dim)
        warp = WarpClassifier(
            self.feature,
            warp_max_dim,
            C=self.C,
            nb_iter=self.nb_gd_iter,
            learning_rate=self.learning_rate,
            inc_rate=self.inc_rate,
            dec_rate=self.dec_rate,
            use_pca=None,
            verbose=self.verbose
        )
        warp.train(samples, labels)
        warpmaps = warp.model_featmaps

        nb_classes = len(warpmaps)
        initdpms = []
        # Choose the part size at half the mean scale, clamped to
        # fit the minimum scale.
        self.partsize = self.partsize
        print "Part size: " + repr(self.partsize)

        for i in range(nb_classes):
            initdpms.append(
                _init_dpm(
                    warpmaps[i],
                    self.nbparts,
                    self.partsize
                )
            )

        # Prepare feature pyramids.
        pyramids = compute_pyramids(samples, self.max_dims, self.feature)


# Natural Language Toolkit: Corpus Readers
#
# Copyright (C) 2001-2016 NLTK Project
# Author: Edward Loper <edloper@gmail.com>
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

# TODO this docstring isn't up-to-date!
"""
NLTK corpus readers.  The modules in this package provide functions
that can be used to read corpus files in a variety of formats.  These
functions can be used to read both the corpus files that are
distributed in the NLTK corpus package, and corpus files that are part
of external corpora.

Available Corpora
=================

Please see http://www.nltk.org/nltk_data/ for a complete list.
Install corpora using nltk.download().

Corpus Reader Functions
=======================
Each corpus module defines one or more "corpus reader functions",
which can be used to read documents from that corpus.  These functions
take an argument, ``item``, which is used to indicate which document
should be read from the corpus:

- If ``item`` is one of the unique identifiers listed in the corpus
  module's ``items`` variable, then the corresponding document will
  be loaded from the NLTK corpus package.
- If ``item`` is a filename, then that file will be read.

Additionally, corpus reader functions can be given lists of item
names; in which case, they will return a concatenation of the
corresponding documents.

Corpus reader functions are named based on the type of information
they return.  Some common examples, and their return types, are:

- words(): list of str
- sents(): list of (list of str)
- paras(): list of (list of (list of str))
- tagged_words(): list of (str,str) tuple
- tagged_sents(): list of (list of (str,str))
- tagged_paras(): list of (list of (list of (str,str)))
- chunked_sents(): list of (Tree w/ (str,str) leaves)
- parsed_sents(): list of (Tree with str leaves)
- parsed_paras(): list of (list of (Tree with str leaves))
- xml(): A single xml ElementTree
- raw(): unprocessed corpus contents

For example, to read a list of the words in the Brown Corpus, use
``nltk.corpus.brown.words()``:

    >>> from nltk.corpus import brown
    >>> print(", ".join(brown.words()))
    The, Fulton, County, Grand, Jury, said, ...

"""

import re

from nltk.tokenize import RegexpTokenizer
from nltk.corpus.util import LazyCorpusLoader
from nltk.corpus.reader import *

abc = LazyCorpusLoader(
    'abc', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding=[
            ('science', 'latin_1'),
            ('rural', 'utf8')])
alpino = LazyCorpusLoader(
    'alpino', AlpinoCorpusReader, tagset='alpino')
brown = LazyCorpusLoader(
    'brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
    cat_file='cats.txt', tagset='brown', encoding="ascii")
cess_cat = LazyCorpusLoader(
    'cess_cat', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tagset='unknown', encoding='ISO-8859-15')
cess_esp = LazyCorpusLoader(
    'cess_esp', BracketParseCorpusReader, r'(?!\.).*\.tbf',
    tagset='unknown', encoding='ISO-8859-15')
cmudict = LazyCorpusLoader(
    'cmudict', CMUDictCorpusReader, ['cmudict'])
comtrans = LazyCorpusLoader(
    'comtrans', AlignedCorpusReader, r'(?!\.).*\.txt')
comparative_sentences = LazyCorpusLoader(
    'comparative_sentences', ComparativeSentencesCorpusReader, r'labeledSentences\.txt',
    encoding='latin-1')
conll2000 = LazyCorpusLoader(
    'conll2000', ConllChunkCorpusReader,
    ['train.txt', 'test.txt'], ('NP','VP','PP'),
    tagset='wsj', encoding='ascii')
conll2002 = LazyCorpusLoader(
    'conll2002', ConllChunkCorpusReader, '.*\.(test|train).*',
    ('LOC', 'PER', 'ORG', 'MISC'), encoding='utf-8')
conll2007 = LazyCorpusLoader(
    'conll2007', DependencyCorpusReader, '.*\.(test|train).*', encoding=[
        ('eus', 'ISO-8859-2'),
        ('esp', 'utf8')])
crubadan = LazyCorpusLoader(
    'crubadan', CrubadanCorpusReader, '.*\.txt')
dependency_treebank = LazyCorpusLoader(
    'dependency_treebank', DependencyCorpusReader, '.*\.dp',
    encoding='ascii')
floresta = LazyCorpusLoader(
    'floresta', BracketParseCorpusReader, r'(?!\.).*\.ptb', '#',
    tagset='unknown', encoding='ISO-8859-15')
framenet = LazyCorpusLoader(
    'framenet_v15', FramenetCorpusReader, ['frRelation.xml','frameIndex.xml','fulltextIndex.xml','luIndex.xml','semTypes.xml'])
gazetteers = LazyCorpusLoader(
    'gazetteers', WordListCorpusReader, r'(?!LICENSE|\.).*\.txt',
    encoding='ISO-8859-2')
genesis = LazyCorpusLoader(
    'genesis', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding=[
        ('finnish|french|german', 'latin_1'),
        ('swedish', 'cp865'),
        ('.*', 'utf_8')])
gutenberg = LazyCorpusLoader(
    'gutenberg', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding='latin1')
# corpus not available with NLTK; these lines caused help(nltk.corpus) to break
#hebrew_treebank = LazyCorpusLoader(
#    'hebrew_treebank', BracketParseCorpusReader, r'.*\.txt')
ieer = LazyCorpusLoader(
    'ieer', IEERCorpusReader, r'(?!README|\.).*')
inaugural = LazyCorpusLoader(
    'inaugural', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding='latin1')
# [XX] This should probably just use TaggedCorpusReader:
indian = LazyCorpusLoader(
    'indian', IndianCorpusReader, r'(?!\.).*\.pos',
    tagset='unknown', encoding='utf8')
ipipan = LazyCorpusLoader(
    'ipipan', IPIPANCorpusReader, r'(?!\.).*morph\.xml')
jeita = LazyCorpusLoader(
    'jeita', ChasenCorpusReader, r'.*\.chasen', encoding='utf-8')
knbc = LazyCorpusLoader(
    'knbc/corpus1', KNBCorpusReader, r'.*/KN.*', encoding='euc-jp')
lin_thesaurus = LazyCorpusLoader(
    'lin_thesaurus', LinThesaurusCorpusReader, r'.*\.lsp')
mac_morpho = LazyCorpusLoader(
    'mac_morpho', MacMorphoCorpusReader, r'(?!\.).*\.txt',
    tagset='unknown', encoding='latin-1')
machado = LazyCorpusLoader(
    'machado', PortugueseCategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'([a-z]*)/.*', encoding='latin-1')
masc_tagged = LazyCorpusLoader(
    'masc_tagged', CategorizedTaggedCorpusReader, r'(spoken|written)/.*\.txt',
    cat_file='categories.txt', tagset='wsj', encoding="utf-8", sep="_")
movie_reviews = LazyCorpusLoader(
    'movie_reviews', CategorizedPlaintextCorpusReader,
    r'(?!\.).*\.txt', cat_pattern=r'(neg|pos)/.*',
    encoding='ascii')
multext_east = LazyCorpusLoader(
    'mte_teip5', MTECorpusReader, r'(oana).*\.xml', encoding="utf-8")
names = LazyCorpusLoader(
    'names', WordListCorpusReader, r'(?!\.).*\.txt', encoding='ascii')
nkjp = LazyCorpusLoader(
    'nkjp', NKJPCorpusReader, r'', encoding='utf8')
nps_chat = LazyCorpusLoader(
    'nps_chat', NPSChatCorpusReader, r'(?!README|\.).*\.xml', tagset='wsj')
opinion_lexicon = LazyCorpusLoader(
    'opinion_lexicon', OpinionLexiconCorpusReader, r'(\w+)\-words\.txt',
    encoding='ISO-8859-2')
panlex_lite = LazyCorpusLoader(
    'panlex_lite', PanLexLiteCorpusReader)
pl196x = LazyCorpusLoader(
    'pl196x', Pl196xCorpusReader, r'[a-z]-.*\.xml',
    cat_file='cats.txt', textid_file='textids.txt', encoding='utf8')
ppattach = LazyCorpusLoader(
    'ppattach', PPAttachmentCorpusReader, ['training', 'test', 'devset'])
product_reviews_1 = LazyCorpusLoader(
    'product_reviews_1', ReviewsCorpusReader, r'^(?!Readme).*\.txt', encoding='utf8')
product_reviews_2 = LazyCorpusLoader(
    'product_reviews_2', ReviewsCorpusReader, r'^(?!Readme).*\.txt', encoding='utf8')
pros_cons = LazyCorpusLoader(
    'pros_cons', ProsConsCorpusReader, r'Integrated(Cons|Pros)\.txt',
    cat_pattern=r'Integrated(Cons|Pros)\.txt', encoding='ISO-8859-2')
ptb = LazyCorpusLoader( # Penn Treebank v3: WSJ and Brown portions
    'ptb', CategorizedBracketParseCorpusReader, r'(WSJ/\d\d/WSJ_\d\d|BROWN/C[A-Z]/C[A-Z])\d\d.MRG',
    cat_file='allcats.txt', tagset='wsj')
qc = LazyCorpusLoader(
    'qc', StringCategoryCorpusReader, ['train.txt', 'test.txt'], encoding='ISO-8859-2')
reuters = LazyCorpusLoader(
    'reuters', CategorizedPlaintextCorpusReader, '(training|test).*',
    cat_file='cats.txt', encoding='ISO-8859-2')
rte = LazyCorpusLoader(
    'rte', RTECorpusReader, r'(?!\.).*\.xml')
senseval = LazyCorpusLoader(
    'senseval', SensevalCorpusReader, r'(?!\.).*\.pos')
sentence_polarity = LazyCorpusLoader(
    'sentence_polarity', CategorizedSentencesCorpusReader, r'rt-polarity\.(neg|pos)',
    cat_pattern=r'rt-polarity\.(neg|pos)', encoding='utf-8')
sentiwordnet = LazyCorpusLoader(
    'sentiwordnet', SentiWordNetCorpusReader, 'SentiWordNet_3.0.0.txt', encoding='utf-8')
shakespeare = LazyCorpusLoader(
    'shakespeare', XMLCorpusReader, r'(?!\.).*\.xml')
sinica_treebank = LazyCorpusLoader(
    'sinica_treebank', SinicaTreebankCorpusReader, ['parsed'],
    tagset='unknown', encoding='utf-8')
state_union = LazyCorpusLoader(
    'state_union', PlaintextCorpusReader, r'(?!\.).*\.txt',
    encoding='ISO-8859-2')
stopwords = LazyCorpusLoader(
    'stopwords', WordListCorpusReader, r'(?!README|\.).*', encoding='utf8')
subjectivity = LazyCorpusLoader(
    'subjectivity', CategorizedSentencesCorpusReader, r'(quote.tok.gt9|plot.tok.gt9)\.5000',
    cat_map={'quote.tok.gt9.5000':['subj'], 'plot.tok.gt9.5000':['obj']}, encoding='latin-1')
swadesh = LazyCorpusLoader(
    'swadesh', SwadeshCorpusReader, r'(?!README|\.).*', encoding='utf8')
swadesh110 = LazyCorpusLoader(
    'panlex_swadesh', SwadeshCorpusReader, r'swadesh110/.*\.txt', encoding='utf8')
swadesh207 = LazyCorpusLoader(
    'panlex_swadesh', SwadeshCorpusReader, r'swadesh207/.*\.txt', encoding='utf8')
switchboard = LazyCorpusLoader(
    'switchboard', SwitchboardCorpusReader, tagset='wsj')
timit = LazyCorpusLoader(
    'timit', TimitCorpusReader)
timit_tagged = LazyCorpusLoader(
    'timit', TimitTaggedCorpusReader, '.+\.tags',
    tagset='wsj', encoding='ascii')
toolbox = LazyCorpusLoader(
    'toolbox', ToolboxCorpusReader, r'(?!.*(README|\.)).*\.(dic|txt)')
treebank = LazyCorpusLoader(
    'treebank/combined', BracketParseCorpusReader, r'wsj_.*\.mrg',
    tagset='wsj', encoding='ascii')
treebank_chunk = LazyCorpusLoader(
    'treebank/tagged', ChunkedCorpusReader, r'wsj_.*\.pos',
    sent_tokenizer=RegexpTokenizer(r'(?<=/\.)\s*(?![^\[]*\])', gaps=True),
    para_block_reader=tagged_treebank_para_block_reader, tagset='wsj', encoding='ascii')
treebank_raw = LazyCorpusLoader(
    'treebank/raw', PlaintextCorpusReader, r'wsj_.*', encoding='ISO-8859-2')
twitter_samples = LazyCorpusLoader(
    'twitter_samples', TwitterCorpusReader, '.*\.json')
udhr = LazyCorpusLoader(
    'udhr', UdhrCorpusReader)
udhr2 = LazyCorpusLoader(
    'udhr2', PlaintextCorpusReader, r'.*\.txt', encoding='utf8')
universal_treebanks = LazyCorpusLoader(
    'universal_treebanks_v20', ConllCorpusReader, r'.*\.conll',
    columntypes = ('ignore', 'words', 'ignore', 'ignore', 'pos',
                   'ignore', 'ignore', 'ignore', 'ignore', 'ignore'))
verbnet = LazyCorpusLoader(
    'verbnet', VerbnetCorpusReader, r'(?!\.).*\.xml')
webtext = LazyCorpusLoader(
    'webtext', PlaintextCorpusReader, r'(?!README|\.).*\.txt', encoding='ISO-8859-2')
wordnet = LazyCorpusLoader(
    'wordnet', WordNetCorpusReader,
    LazyCorpusLoader('omw', CorpusReader, r'.*/wn-data-.*\.tab', encoding='utf8'))
wordnet_ic = LazyCorpusLoader(
    'wordnet_ic', WordNetICCorpusReader, '.*\.dat')
words = LazyCorpusLoader(
    'words', WordListCorpusReader, r'(?!README|\.).*', encoding='ascii')
ycoe = LazyCorpusLoader(
    'ycoe', YCOECorpusReader)
# defined after treebank
propbank = LazyCorpusLoader(
    'propbank', PropbankCorpusReader,
    'prop.txt', 'frames/.*\.xml', 'verbs.txt',


'''Functions to match according to a requirement specification.'''

import logging
import re
import sys
try:
    import ipaddr
    _HAS_IPADDR = True
except ImportError:
    _HAS_IPADDR = False

LOG = logging.getLogger('hardware.matcher')

def _adder(array, index, value):
    'Auxiliary function to add a value to an array.'
    array[index] = value

def _appender(array, index, value):
    'Auxiliary function to append a value to an array.'
    try:
        array[index].append(value)
    except KeyError:
        array[index] = [value, ]

def _gt(left, right):
    'Helper for match_spec.'
    return int(left) > int(right)

def _ge(left, right):
    'Helper for match_spec.'
    return int(left) >= int(right)

def _lt(left, right):
    'Helper for match_spec.'
    return int(left) < int(right)

def _le(left, right):
    'Helper for match_spec.'
    return int(left) <= int(right)

def _not(_, right):
    'Helper for match_spec.'
    return not right

def _and(_, left, right):
    'Helper for match_spec.'
    return left and right

def _or(_, left, right):
    'Helper for match_spec.'
    return left or right

def _network(left, right):
    'Helper for match_spec.'
    if _HAS_IPADDR:
        return ipaddr.IPv4Address(left) in ipaddr.IPv4Network(right)
    else:
        return False

def _regexp(left, right):
    'Helper for match_spec.'
    return re.search(right, left) is not None

def _in(elt, *lst):
    'Helper for match_spec.'
    return elt in lst

_FUNC_REGEXP = re.compile(r'^([^(]+)'          # function name
                          r'\(\s*([^,]+)'      # first argument
                          r'(?:\s*,\s*(.+))?'  # remaining optional arguments
                          r'\)$')              # last parenthesis

def _call_func(func, implicit, res):
    'Helper function for extract_result and match_spec'
    args = [implicit, res.group(2)]
    # split the optional arguments if we have some
    if res.group(3):
        args = args + re.split(r'\s*,\s*', res.group(3))
    # remove strings delimiters
    args = [x.strip('\'"') for x in args]
    # call function
    args = [_extract_result(implicit, x) for x in args]
    return func(*args)

def _extract_result(implicit, expr):
    'Helper function for match_spec'
    res = _FUNC_REGEXP.search(expr)
    if res:
        func_name = '_' + res.group(1)
        if func_name in globals():
            return _call_func(globals()[func_name], implicit, res)
        else:
            return expr
    else:
        return expr

def match_spec(spec, lines, arr, adder=_adder):
    'Match a line according to a spec and store variables in <var>.'
    # match a line without variable
    for idx in range(len(lines)):
        if lines[idx] == spec:
            res = lines[idx]
            del lines[idx]
            return res
    # match a line with a variable, a function or both
    for lidx in range(len(lines)):
        line = lines[lidx]
        varidx = []
        for idx in range(4):
            # try to split the variable and function parts if we have both
            if spec[idx][0] == '$':
                parts = spec[idx].split('=')
                if len(parts) == 2:
                    var, func = parts
                    matched = False
                else:
                    var = func = spec[idx]
            else:
                var = func = spec[idx]
            # Match a function
            if func[-1] == ')':
                res = _FUNC_REGEXP.search(func)
                if res:
                    func_name = '_' + res.group(1)
                    if func_name in globals():
                        if not _call_func(globals()[func_name],
                                          line[idx], res):
                            if var == func:
                                break
                        else:
                            if var == func:
                                continue
                            matched = True
                    else:
                        if var == func:
                            break
            # Match a variable
            if ((var == func) or (var != func and matched)) and var[0] == '$':
                if adder == _adder and var[1:] in arr:
                    if arr[var[1:]] != line[idx]:
                        break
                varidx.append((idx, var[1:]))
            # Match the full string
            elif line[idx] != spec[idx]:
                break
        else:
            for i, var in varidx:
                adder(arr, var, line[i])
            res = lines[lidx]
            del lines[lidx]
            return res
    return False

def match_all(lines, specs, arr, arr2, debug=False, level=0):
    '''Match all lines according to a spec.

Store variables starting with a $ in <arr>. Variables starting with
2 $ like $$vda are stored in arr and arr2.
'''
    # Work on a copy of lines to avoid changing the real lines because
    # match_spec removes the matched line to not match it again on next
    # calls.
    lines = list(lines)
    specs = list(specs)
    copy_arr = dict(arr)
    points = []
    # Prevent infinit loops
    if level == 50:
        return False
    # Match lines using specs
    while len(specs) > 0:
        copy_specs = list(specs)
        spec = specs.pop(0)
        line = match_spec(spec, lines, arr)
        # No match
        if not line:
            # Backtrack on the backtracking points
            while len(points) > 0:
                lines, specs, new_arr = points.pop()
                if match_all(lines, specs, new_arr, arr2, debug, level + 1):
                    # Copy arr back
                    for k in new_arr:
                        arr[k] = new_arr[k]
                    return True
            if level == 0 and debug:
                sys.stderr.write('spec: %s not matched\n' % str(spec))
            return False
        else:
            # Store backtraking points when we find a new variable
            if arr != copy_arr:
                copy_lines = list(lines)
                # Put the matching line at the end of the lines
                copy_lines.append(line)
                points.append((copy_lines, copy_specs, copy_arr))
                copy_arr = arr
    # Manage $$ variables
    for key in arr:
        if key[0] == '$':
            nkey = key[1:]


# -*- coding: utf-8 -*-
# Copyright 2015 OpenSynergy Indonesia
# Copyright 2020 PT. Simetri Sinergi Indonesia
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).
import time
from decimal import Decimal

from openerp.report import report_sxw

class Parser(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):  # pylint: disable=R8110
        super(Parser, self).__init__(cr, uid, name, context)
        self.lines = []
        self.total_debit = 0.0
        self.total_credit = 0.0
        self.total_beginning_debit = 0.0
        self.total_beginning_credit = 0.0
        self.total_ending_debit = 0.0
        self.total_ending_credit = 0.0
        self.total_ending_balance = 0.0
        self.localcontext.update(
            {
                "time": time,
                "get_period": self.get_period,
                "get_company": self.get_company,
                "total_debit": self.get_total_debit,
                "total_credit": self.get_total_credit,
                "total_beginning_debit": self.get_total_beginning_debit,
                "total_beginning_credit": self.get_total_beginning_credit,
                "total_ending_debit": self.get_total_ending_debit,
                "total_ending_credit": self.get_total_ending_credit,
                "total_ending_balance": self.get_total_ending_balance,
                "lines": self.get_trial_balance_line,
            }
        )

    def get_company(self):
        data = self.localcontext["data"]["form"]
        company_name = data["company_id"] and data["company_id"][1] or False

        return company_name

    def get_period(self):
        data = self.localcontext["data"]["form"]
        period_name = data["period_id"] and data["period_id"][1] or False

        return period_name

    def get_balance(self, type_balance, account_id):
        res = {}
        obj_period = self.pool.get("account.period")
        obj_account = self.pool.get("account.account")

        data = self.localcontext["data"]["form"]
        data_period = data["period_id"]
        data_fiscalyear = data["fiscalyear_id"]

        period_id = data_period and data_period[0] or False
        fiscalyear_id = data_fiscalyear and data_fiscalyear[0] or False
        state = data["state"]

        period_from = False
        period_to = False

        if period_id:
            kriteria_opening_balance = [("fiscalyear_id", "=", fiscalyear_id)]
            period_ids = obj_period.search(self.cr, self.uid, kriteria_opening_balance)

            for list_index, opening_period_id in enumerate(period_ids):
                if opening_period_id == period_id:
                    previous_period_id = period_ids[list_index - 1]

            first_period_id = period_ids[0]

            beginning_period_from = first_period_id
            beginning_period_to = previous_period_id

            period_from = period_id
            period_to = period_id

            ending_period_from = first_period_id
            ending_period_to = period_id

        res = {
            "beginning_debit": 0.0,
            "beginning_credit": 0.0,
            "beginning_balance": 0.0,
            "debit": 0.0,
            "credit": 0.0,
            "ending_debit": 0.0,
            "ending_credit": 0.0,
            "ending_balance": 0.0,
        }

        if period_id:
            context_beginning = {
                "period_from": beginning_period_from,
                "period_to": beginning_period_to,
            }

            context_ending = {
                "period_from": ending_period_from,
                "period_to": ending_period_to,
            }

            context_now = {
                "period_from": period_from,
                "period_to": period_to,
            }

            if state != "all":
                context_beginning.update({"state": state})
                context_ending.update({"state": state})
                context_now.update({"state": state})

            account_beginning = obj_account.browse(
                self.cr, self.uid, [account_id], context=context_beginning
            )[0]
            account_ending = obj_account.browse(
                self.cr, self.uid, [account_id], context=context_ending
            )[0]
            account_now = obj_account.browse(
                self.cr, self.uid, [account_id], context=context_now
            )[0]

            if type_balance == "beginning":
                report_type = account_beginning.user_type.report_type
                if report_type in ["income", "liability"]:
                    beginning_balance = (
                        account_beginning.credit - account_beginning.debit
                    )
                else:
                    beginning_balance = (
                        account_beginning.debit - account_beginning.credit
                    )
                res = {
                    "beginning_debit": account_beginning.debit,
                    "beginning_credit": account_beginning.credit,
                    "beginning_balance": beginning_balance,
                }
            if type_balance == "now":
                res = {"debit": account_now.debit, "credit": account_now.credit}
            if type_balance == "ending":
                report_type = account_ending.user_type.report_type
                if report_type in ["income", "liability"]:
                    ending_balance = account_ending.credit - account_ending.debit
                else:
                    ending_balance = account_ending.debit - account_ending.credit
                res = {
                    "ending_debit": account_ending.debit,
                    "ending_credit": account_ending.credit,
                    "ending_balance": ending_balance,
                }

        return res

    def get_total_debit(self):
        return self.total_debit

    def get_total_credit(self):
        return self.total_credit

    def get_total_beginning_debit(self):
        return self.total_beginning_debit

    def get_total_beginning_credit(self):
        return self.total_beginning_credit

    def get_total_ending_debit(self):
        return self.total_ending_debit

    def get_total_ending_credit(self):
        return self.total_ending_credit

    def get_total_ending_balance(self):
        return self.total_ending_balance

    def get_trial_balance_line(self):
        def _process_child(accounts, parent, level):
            account_rec = [acct for acct in accounts if acct["id"] == parent][0]

            if account_rec["type"] not in ["view"]:

                beginning = self.get_balance("beginning", account_rec["id"])
                now = self.get_balance("now", account_rec["id"])
                ending_debit = self.get_balance("ending", account_rec["id"])

                self.total_beginning_debit += beginning["beginning_debit"]
                self.total_beginning_credit += abs(beginning["beginning_credit"])

                self.total_debit += now["debit"]
                self.total_credit += abs(now["credit"])

                self.total_ending_debit += ending_debit["ending_debit"]
                self.total_ending_credit += abs(ending_debit["ending_credit"])
                self.total_ending_balance += (
                    self.total_ending_debit - self.total_ending_credit
                )

                res = {
                    "id": account_rec["id"],
                    "code": account_rec["code"],
                    "name": account_rec["name"],
                    "beginning_debit": Decimal(abs(beginning["beginning_debit"])),
                    "beginning_credit": Decimal(abs(beginning["beginning_credit"])),
                    "beginning_balance": Decimal(beginning["beginning_balance"]),
                    "debit": Decimal(abs(now["debit"])),
                    "credit": Decimal(abs(now["credit"])),
                    "ending_debit": Decimal(abs(ending_debit["ending_debit"])),
                    "ending_credit": Decimal(abs(ending_debit["ending_credit"])),
                    "ending_balance": Decimal(ending_debit["ending_balance"]),
                    "balance": Decimal(abs(account_rec["balance"])),
                    "parent_id": account_rec["parent_id"],
                }

                self.lines.append(res)

        obj_account_account = self.pool.get("account.account")