# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; version 2 of
# the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.

'''
emtools.cluster.playbookinstall

Generate/executes ansible playbook for installs and upgrades.
'''
import os
import shutil
import distutils.dir_util
from emtools.playbookmgr import PlaybookMgr
import emtools.common.logutils as logutils
import emtools.common.utils as utils
import emtools.common as common
from pkgfilenameparser import PkgFileNameParser
from emtools.cluster.configspec import ConfigSpec

Log = logutils.getLogger(__name__)

class PlaybookInstall(object):
    '''
    PlaybookGen can be used to generate and execute ansible
    playbooks to perform binary/package installs or upgrades.
    '''

    def __init__(self, cluster, optype):
        '''
        Constructor
        '''
        self._cluster = cluster
        self._optype  = optype
        if ((self._optype != 'pkginstall') and
            (self._optype != 'pkgupgrade') and
            (self._optype != 'bininstall') and
            (self._optype != 'binupgrade')):
            raise Exception("Invalid cluster ansible playbook installer type: %s" % self._optype)
        self._pkgfilenameparser = PkgFileNameParser()

        self._ansible_file = ""
        self._extra_vars   = ""
        self._inventory_filename = "infinidb"

    def run_cmd(self):
        '''
        Prepare and run the ansible playbook command for
        the operation type specified in the constructor
        '''
        self._rundir  = self._cluster.get_rundir()
        self._pkgdir  = self._cluster.get_pkgdir()
        self._pkgfile = self._cluster.get_pkgfile()
        self._idbuser = self._cluster.config()['idbuser']
        eflag = self._cluster.config()['enterprise']
        if eflag:
            self._entflag = "true"
        else:
            self._entflag = "false"
        self._version = self._pkgfilenameparser.get_pkg_version(self._pkgfile)
        self._hadoop  = self._cluster.config()['hadoop']
        self._hdfsflag = "false"
        if self._hadoop:
            self._hdfsflag = "true"
        self._upgfile    = self._cluster.get_upgfile()
        self._upgversion = None
        if self._upgfile:
            self._upgversion = self._pkgfilenameparser.get_pkg_version(self._upgfile)
        m = self._cluster.machine('pm1')
        self._pm1_ip = m['ip']
        self._postconfig_opts = self._cluster.get_postconfig_opts()

        # Add -em to postconfig flags for version 4.6 and up
        if self._optype == 'pkginstall':
            if ConfigSpec._version_greaterthan(self._version,'4.6.0-0'):
                self._postconfig_opts += " -em"
            (ansible_yml,cmdargs) = self._prepare_playbook_pkginstall()
        elif self._optype == 'pkgupgrade':
            if ConfigSpec._version_greaterthan(self._upgversion,'4.6.0-0'):
                self._postconfig_opts += " -em"
            (ansible_yml,cmdargs) = self._prepare_playbook_pkgupgrade()
        elif self._optype == 'bininstall':
            if ConfigSpec._version_greaterthan(self._version,'4.6.0-0'):
                self._postconfig_opts += " -em"
            (ansible_yml,cmdargs) = self._prepare_playbook_bininstall()
        elif self._optype == 'binupgrade':
            if ConfigSpec._version_greaterthan(self._upgversion,'4.6.0-0'):
                self._postconfig_opts += " -em"
            (ansible_yml,cmdargs) = self._prepare_playbook_binupgrade()
        else:
            raise Exception("Unsupported ansible playbook type to run: %s" % self._optype)

        extra_playdir = self._cluster.get_extra_playbook_dir()
        p = PlaybookMgr( os.path.basename(self._rundir), extra_playdir )

        # create ansible inventory file with list of hosts;
        # should already exist for an EM-triggered install.
        full_inv_file = '%s/%s' % (p.get_rootdir(), self._inventory_filename)
        if not os.path.exists( full_inv_file ):
            machines = self._cluster.machines()
            iplist = []
            infnodelist = []
            for key in machines:
                m = machines[key]
                iplist.append( m['ip'] )

                # if we are using the EM in invm mode we don't want that
                # node to participate in the normal InfiniDB install
                if key != 'em1':
                    #f.write("key: %s.calpont.com; ip: %s\n" % (key,m['ip']))
                    infnodelist.append( m['ip'] )

            ipdict = { 'all' : iplist, 'infinidb_nodes' : infnodelist }
            p.write_inventory( self._inventory_filename, ipdict )

        # create ansible.cfg file;
        # should already exist for an EM-triggered install.
        full_ans_file = '%s/%s' % (p.get_rootdir(), 'ansible.cfg')
        if not os.path.exists( full_ans_file ):
            keytext = self._cluster.get_sshkey_text()
            p.config_ssh( self._idbuser, keytext )

        # execute playbook thru PlaybookMgr
        self._ansible_file = ansible_yml
        self._extra_vars   = cmdargs
        rc, results, out, err = p.run_playbook(
            ansible_yml,
            self._inventory_filename,
            playbook_args=cmdargs)

        return rc, results, out, err

    def get_playbook_filename(self):
        """
        Return main ansible playbook file
        """
        return self._ansible_file

    def get_extra_vars(self):
        """
        Return extra variable list passed to ansible
        """
        return self._extra_vars

    def get_inventory_filename(self):
        """
        Return inventory file name that was created
        """
        return self._inventory_filename

    def _prepare_playbook_pkginstall(self):
        """
        Prepare pkg install using ansible playbook.
        """
        playbook = ('%s/pkginstall.yml' % self._rundir)
        cmdargs = ("\"pkgbasever=%s "
            "pkgver=%s "
            "pkgfile=%s "
            "pkgfilebase=%s "
            "enterprise=%s "
            "pm1_host=%s "
            "postconfig_opts='%s' "
            "hdfs=%s "
            "rundir=%s "
            "pkgdir=%s "
            "\"" % (self._version[0], self._version, self._pkgfile,
                os.path.basename(self._pkgfile), self._entflag, self._pm1_ip,
                self._postconfig_opts, self._hdfsflag, self._rundir,
                self._pkgdir))

        if self._hadoop:
            Log.info("Running pkg install playbook with Hadoop; --extra-vars=%s" % cmdargs)
        else:
            Log.info("Running pkg install playbook; --extra-vars=%s" % cmdargs)

        return (playbook,cmdargs)

    def _prepare_playbook_pkgupgrade(self):
        """
        Prepare pkg upgrade using ansible playbook.
        """
        playbook = ("%s/pkgupgrade.yml" % self._rundir)
        cmdargs = ("\"pm1_host=%s "
            "pkgdir=%s "
            "pkgbasever=%s "
            "upgbasever=%s "
            "upgver=%s "
            "upgfile=%s "
            "upgfilebase=%s "
            "enterprise=%s "
            "postconfig_opts='%s' "
            "\"" % (self._pm1_ip, self._pkgdir, self._version[0], self._upgversion[0],
                self._upgversion, self._upgfile, os.path.basename(self._upgfile),
                self._entflag, self._postconfig_opts))

        Log.info("Running pkg upgrade playbook; --extra-vars=%s" % cmdargs)

        return (playbook,cmdargs)

    def _prepare_playbook_bininstall(self):
        """
        Prepare binary install using ansible playbook.
        """
        playbook = ('%s/bininstall.yml' % self._rundir)


# -*- encoding: utf-8 -*-
#
# Copyright 2009-2013 Lev Shamardin.
#
# This file is part of gridproxy library.
#
# Gridproxy library is free software: you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Gridproxy library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with gridproxy library. If not, see <http://www.gnu.org/licenses/>.
#

"""Grid proxy certificate functions."""

import os
import re
import struct
import tempfile
import time
from M2Crypto import X509, EVP, RSA, ASN1, m2, BIO

CLOCK_SKEW = 5*60

def get_proxy_filename():
    """
    Return a proxy certificate filename for the calling user.
    """
    if 'X509_USER_PROXY' in os.environ:
        return os.environ['X509_USER_PROXY']

    if os.name == 'posix':
        filename = 'x509up_u%d' % os.getuid()
    else:
        raise RuntimeError("X509_USER_PROXY must be set for %s platform. Can't determine proxy file location." % os.name)

    if os.uname()[0] == 'Darwin':
        proxydir = '/tmp'
    else:
        proxydir = tempfile.gettempdir()

    return os.path.join(proxydir, filename)

def generate_keypair(bits=512, exponent=65537):
    """Generate a new RSA key pair. Returns a tuple of (RSA, PKey)"""
    rsa = RSA.gen_key(bits, exponent, lambda x: None)
    pkey = EVP.PKey()
    pkey.assign_rsa(rsa)
    return rsa, pkey

# pylint: disable-msg=R0914,R0915,R0912
def generate_proxycert(new_pkey, cert, key, **kwargs):
    """Generate a proxy certificate.

     * new_pkey: EVP.PKey for the proxy certificate
     * cert: Issuer certificate
     * key: Issuer private key

    Optional arguments:

     * lifetime: proxy certificate lifetime, in seconds. Default: is 12 hours.
     * full: proxy type, full or limited. Default is False (limited proxy).
     * hash_algorithm: hash algorithm to use for certificate signing. Default: sha1.
     * globus_bug: do not use basicConstraints extension and do not allow Key Agreement. Default: true
    """
    args = kwargs.copy()
    if 'args' in args:
        args.update(args.pop('args'))

    legacy = args.get('legacy', is_legacy_proxy(cert))
    full = args.get('full', False)
    globus_bug = args.get('globus_bug', True)

    proxy = X509.X509()
    proxy.set_pubkey(new_pkey)
    proxy.set_version(2)

    now = int(time.time())
    not_before = ASN1.ASN1_UTCTIME()
    not_before.set_time(now  - CLOCK_SKEW)
    proxy.set_not_before(not_before)
    not_after = ASN1.ASN1_UTCTIME()
    not_after.set_time(now + args.get('lifetime', 12*60*60) + CLOCK_SKEW)
    proxy.set_not_after(not_after)

    proxy.set_issuer_name(cert.get_subject())
    digest = EVP.MessageDigest('sha1')
    digest.update(new_pkey.as_der())
    serial = struct.unpack("<L", digest.final()[:4])[0]
    proxy.set_serial_number(int(serial & 0x7fffffff))

    # It is not completely clear what happens with memory allocation
    # within the next calls, so after building the whole thing we are
    # going to reload it through der encoding/decoding.
    proxy_subject = X509.X509_Name()
    subject = cert.get_subject()
    for idx in xrange(subject.entry_count()):
        entry = subject[idx].x509_name_entry
        m2.x509_name_add_entry(proxy_subject._ptr(), entry, -1, 0)
    if legacy:
        if full:
            proxy_subject.add_entry_by_txt('CN',
                                           ASN1.MBSTRING_ASC,
                                           "proxy", -1, -1, 0)
        else:
            proxy_subject.add_entry_by_txt('CN',
                                           ASN1.MBSTRING_ASC,
                                           "limited proxy", -1, -1, 0)
    else:
        proxy_subject.add_entry_by_txt('CN',
                                       ASN1.MBSTRING_ASC,
                                       str(serial), -1, -1, 0)
    proxy.set_subject(proxy_subject)
    if legacy:
        if globus_bug:
            proxy.add_ext(X509.new_extension(
                "keyUsage",
                "Digital Signature, Key Encipherment, Data Encipherment", 1))
        else:
            proxy.add_ext(X509.new_extension(
                "keyUsage",
                "Digital Signature, Key Encipherment, Data Encipherment, Key Agreement", 1))
            proxy.add_ext(X509.new_extension("basicConstraints", "CA:FALSE", 1))

            # does not work (?) seems like need to add authorityCertIssuer
            # and authorityCertSerialNumber somehow, see rfc 3280 for more
            # details

            # try:
            #     subjkey = cert.get_ext('subjectKeyIdentifier')
            #     keyid = "keyid:%s" % subjkey.get_value()
            #     ext = X509.new_extension("authorityKeyIdentifier", keyid, 0)
            #     proxy.add_ext(ext)
            # except LookupError:
            #     pass
    else:
        if globus_bug:
            proxy.add_ext(X509.new_extension("keyUsage", "Digital Signature, Key Encipherment, Data Encipherment", 1))
        else:
            proxy.add_ext(X509.new_extension("basicConstraints", "CA:FALSE", 1))
            proxy.add_ext(X509.new_extension("keyUsage", "Digital Signature, Key Encipherment, Data Encipherment, Key Agreement", 1))
    if not legacy:
        if full:
            proxy.add_ext(X509.new_extension("proxyCertInfo",
                                             "critical, language:Inherit all",
                                             1))
        else:
            proxy.add_ext(X509.new_extension("proxyCertInfo",
                                             "critical, language:1.3.6.1.4.1.3536.1.1.1.9",
                                             1))

    sign_pkey = EVP.PKey()
    sign_pkey.assign_rsa(key, 0)
    proxy.sign(sign_pkey, args.get('hash_algorithm', 'sha1'))

    return X509.load_cert_string(proxy.as_pem())

def split_proxy(proxy):
    u"""
    Split the proxy string into two lists:
    * certs - certificates
    * keys - keys

    Returns (certs, keys)
    """
    lines = proxy.split("\n")
    result = ([], [])
    state = 3
    endtag = None
    buf = []
    while len(lines) > 0:
        line = lines[0]
        lines = lines[1:]

        if state != 2:
            buf.append(line)

        if line == endtag:
            if state < 2:
                result[state].append("\n".join(buf))
            state = 3
            endtag = None
            buf = []
            continue

        if state == 3:
            what = re.findall("^-----BEGIN (.*)-----$", line)
            if len(what) == 1:
                endtag = "-----END %s-----" % what[0]
                if what[0] == "CERTIFICATE":
                    state = 0
                elif what[0].endswith(" KEY"):
                    state = 1
                else:
                    state = 2
    return result

def load_proxy(proxy):
    """Load a proxy certificate from a string."""
    certs, keys = split_proxy(proxy)
    if len(keys) != 0:
        key = RSA.load_key_string(keys[0])
    else:
        key = None


import json
import logging
import sys
from functools import wraps

from django.conf import settings
from django.core.validators import ValidationError, validate_email
from django.views.decorators.csrf import requires_csrf_token
from django.views.defaults import server_error
from django.http import (Http404, HttpResponse, HttpResponseNotAllowed,
                         HttpResponseServerError)
import dogstats_wrapper as dog_stats_api
from edxmako.shortcuts import render_to_response
import zendesk
from microsite_configuration import microsite

import calc
import track.views

from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey

log = logging.getLogger(__name__)

def ensure_valid_course_key(view_func):
    """
    This decorator should only be used with views which have argument course_key_string (studio) or course_id (lms).
    If course_key_string (studio) or course_id (lms) is not valid raise 404.
    """
    @wraps(view_func)
    def inner(request, *args, **kwargs):
        course_key = kwargs.get('course_key_string') or kwargs.get('course_id')
        if course_key is not None:
            try:
                CourseKey.from_string(course_key)
            except InvalidKeyError:
                raise Http404

        response = view_func(request, *args, **kwargs)
        return response

    return inner

@requires_csrf_token
def jsonable_server_error(request, template_name='500.html'):
    """
    500 error handler that serves JSON on an AJAX request, and proxies
    to the Django default `server_error` view otherwise.
    """
    if request.is_ajax():
        msg = {"error": "The edX servers encountered an error"}
        return HttpResponseServerError(json.dumps(msg))
    else:
        return server_error(request, template_name=template_name)

def handle_500(template_path, context=None, test_func=None):
    """
    Decorator for view specific 500 error handling.
    Custom handling will be skipped only if test_func is passed and it returns False

    Usage:

        @handle_500(
            template_path='certificates/server-error.html',
            context={'error-info': 'Internal Server Error'},
            test_func=lambda request: request.GET.get('preview', None)
        )
        def my_view(request):
            # Any unhandled exception in this view would be handled by the handle_500 decorator
            # ...

    """
    def decorator(func):
        """
        Decorator to render custom html template in case of uncaught exception in wrapped function
        """
        @wraps(func)
        def inner(request, *args, **kwargs):
            """
            Execute the function in try..except block and return custom server-error page in case of unhandled exception
            """
            try:
                return func(request, *args, **kwargs)
            except Exception:  # pylint: disable=broad-except
                if settings.DEBUG:
                    # In debug mode let django process the 500 errors and display debug info for the developer
                    raise
                elif test_func is None or test_func(request):
                    # Display custom 500 page if either
                    #   1. test_func is None (meaning nothing to test)
                    #   2. or test_func(request) returns True
                    log.exception("Error in django view.")
                    return render_to_response(template_path, context)
                else:
                    # Do not show custom 500 error when test fails
                    raise
        return inner
    return decorator

def calculate(request):
    ''' Calculator in footer of every page. '''
    equation = request.GET['equation']
    try:
        result = calc.evaluator({}, {}, equation)
    except:
        event = {'error': map(str, sys.exc_info()),
                 'equation': equation}
        track.views.server_track(request, 'error:calc', event, page='calc')
        return HttpResponse(json.dumps({'result': 'Invalid syntax'}))
    return HttpResponse(json.dumps({'result': str(result)}))

class _ZendeskApi(object):
    def __init__(self):
        """
        Instantiate the Zendesk API.

        All of `ZENDESK_URL`, `ZENDESK_USER`, and `ZENDESK_API_KEY` must be set
        in `django.conf.settings`.
        """
        self._zendesk_instance = zendesk.Zendesk(
            settings.ZENDESK_URL,
            settings.ZENDESK_USER,
            settings.ZENDESK_API_KEY,
            use_api_token=True,
            api_version=2,
            # As of 2012-05-08, Zendesk is using a CA that is not
            # installed on our servers
            client_args={"disable_ssl_certificate_validation": True}
        )

    def create_ticket(self, ticket):
        """
        Create the given `ticket` in Zendesk.

        The ticket should have the format specified by the zendesk package.
        """
        ticket_url = self._zendesk_instance.create_ticket(data=ticket)
        return zendesk.get_id_from_url(ticket_url)

    def update_ticket(self, ticket_id, update):
        """
        Update the Zendesk ticket with id `ticket_id` using the given `update`.

        The update should have the format specified by the zendesk package.
        """
        self._zendesk_instance.update_ticket(ticket_id=ticket_id, data=update)

def _record_feedback_in_zendesk(realname, email, subject, details, tags, additional_info):
    """
    Create a new user-requested Zendesk ticket.

    Once created, the ticket will be updated with a private comment containing
    additional information from the browser and server, such as HTTP headers
    and user state. Returns a boolean value indicating whether ticket creation
    was successful, regardless of whether the private comment update succeeded.
    """
    zendesk_api = _ZendeskApi()

    additional_info_string = (
        "Additional information:\n\n" +
        "\n".join("%s: %s" % (key, value) for (key, value) in additional_info.items() if value is not None)
    )

    # Tag all issues with LMS to distinguish channel in Zendesk; requested by student support team
    zendesk_tags = list(tags.values()) + ["LMS"]

    # Per edX support, we would like to be able to route white label feedback items
    # via tagging
    white_label_org = microsite.get_value('course_org_filter')
    if white_label_org:
        zendesk_tags = zendesk_tags + ["whitelabel_{org}".format(org=white_label_org)]

    new_ticket = {
        "ticket": {
            "requester": {"name": realname, "email": email},
            "subject": subject,
            "comment": {"body": details},
            "tags": zendesk_tags
        }
    }
    try:
        ticket_id = zendesk_api.create_ticket(new_ticket)
    except zendesk.ZendeskError:
        log.exception("Error creating Zendesk ticket")
        return False

    # Additional information is provided as a private update so the information
    # is not visible to the user.
    ticket_update = {"ticket": {"comment": {"public": False, "body": additional_info_string}}}
    try:
        zendesk_api.update_ticket(ticket_id, ticket_update)
    except zendesk.ZendeskError:
        log.exception("Error updating Zendesk ticket")
        # The update is not strictly necessary, so do not indicate failure to the user
        pass

    return True

DATADOG_FEEDBACK_METRIC = "lms_feedback_submissions"

def _record_feedback_in_datadog(tags):
    datadog_tags = [u"{k}:{v}".format(k=k, v=v) for k, v in tags.items()]
    dog_stats_api.increment(DATADOG_FEEDBACK_METRIC, tags=datadog_tags)

def submit_feedback(request):
    """
    Create a new user-requested ticket, currently implemented with Zendesk.

    If feedback submission is not enabled, any request will raise `Http404`.
    If any configuration parameter (`ZENDESK_URL`, `ZENDESK_USER`, or
    `ZENDESK_API_KEY`) is missing, any request will raise an `Exception`.
    The request must be a POST request specifying `subject` and `details`.
    If the user is not authenticated, the request must also specify `name` and
    `email`. If the user is authenticated, the `name` and `email` will be
    populated from the user's information. If any required parameter is
    missing, a 400 error will be returned indicating which field is missing and
    providing an error message. If Zendesk ticket creation fails, 500 error
    will be returned with no body; if ticket creation succeeds, an empty
    successful response (200) will be returned.
    """
    if not settings.FEATURES.get('ENABLE_FEEDBACK_SUBMISSION', False):
        raise Http404()
    if request.method != "POST":
        return HttpResponseNotAllowed(["POST"])
    if (
        not settings.ZENDESK_URL or
        not settings.ZENDESK_USER or
        not settings.ZENDESK_API_KEY
    ):
        raise Exception("Zendesk enabled but not configured")

    def build_error_response(status_code, field, err_msg):
        return HttpResponse(json.dumps({"field": field, "error": err_msg}), status=status_code)

    additional_info = {}

    required_fields = ["subject", "details"]
    if not request.user.is_authenticated():
        required_fields += ["name", "email"]
    required_field_errs = {
        "subject": "Please provide a subject.",
        "details": "Please provide details.",
        "name": "Please provide your name.",
        "email": "Please provide a valid e-mail.",
    }

    for field in required_fields:
        if field not in request.POST or not request.POST[field]:
            return build_error_response(400, field, required_field_errs[field])

    subject = request.POST["subject"]
    details = request.POST["details"]
    tags = dict(
        [(tag, request.POST[tag]) for tag in ["issue_type", "course_id"] if tag in request.POST]
    )

    if request.user.is_authenticated():
        realname = request.user.profile.name
        email = request.user.email
        additional_info["username"] = request.user.username
    else:
        realname = request.POST["name"]
        email = request.POST["email"]
        try:
            validate_email(email)
        except ValidationError:
            return build_error_response(400, "email", required_field_errs["email"])

    for header, pretty in [
        ("HTTP_REFERER", "Page"),
        ("HTTP_USER_AGENT", "Browser"),
        ("REMOTE_ADDR", "Client IP"),
        ("SERVER_NAME", "Host")
    ]:
        additional_info[pretty] = request.META.get(header)

    success = _record_feedback_in_zendesk(realname, email, subject, details, tags, additional_info)
    _record_feedback_in_datadog(tags)

    return HttpResponse(status=(200 if success else 500))

def info(request):
    ''' Info page (link from main header) '''
    return render_to_response("info.html", {})

# From http://djangosnippets.org/snippets/1042/
def parse_accept_header(accept):
    """Parse the Accept header *accept*, returning a list with pairs of
    (media_type, q_value), ordered by q values.
    """
    result = []
    for media_range in accept.split(","):
        parts = media_range.split(";")
        media_type = parts.pop(0)
        media_params = []
        q = 1.0
        for part in parts:
            (key, value) = part.lstrip().split("=", 1)


# -*- coding: utf-8 -*-
# Maestro Music Manager  -  https://github.com/maestromusic/maestro
# Copyright (C) 2009-2015 Martin Altmayer, Michael Helmling
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import re, os, collections, copy

_types = {} # types of config files. Maps name to the corresponding class (see registerConfigType)
_files = [] # registered config files (see addFile)

class ConfigError(Exception):
    """A ConfigError is for example raised if the configuration file contains invalid syntax."""

def registerConfigType(name, theClass):
    """Register a new type of configuration file. *name* is the identifier that must be passed to 'addFile'
    to create such a file. *theClass* is the class responsible for handling such files.
    """
    _types[name] = theClass

def unregisterConfigType(name):
    """Unregister a configuration file type."""
    del _types[name]

def defaultConfigDirectory(name):
    """Return a reasonable directory for configuration files. *name* should be the name of your application.
    """
    if 'XDG_CONFIG_HOME' in os.environ:
        return os.path.join(os.environ['XDG_CONFIG_HOME'], name)
    else: return os.path.join(os.path.expanduser("~"), ".config", name)

def addFile(type, *args, **kwargs):
    """Add a configuration file to the application and return a File-object. *type* is the file type
    (one of 'config', 'json', 'pickle' or a name registered with 'registerConfigType'). All other arguments
    are passed to the corresponding class.

    Note that the file on disk is not read until either 'read' or 'getAccess' is called.
    """
    if type in _types:
        file = _types[type](*args, **kwargs)
        _files.append(file)
        return file
    else: raise ValueError("'{}' is not a valid argument for config.addFile.".format(type))

def removeFile(file):
    """Remove a configuration file from the application without deleting the file from disk."""
    _files.remove(file)

def deleteFile(file):
    """Remove a configuration file from the application and delete it from disk."""
    if os.path.exists(file.path):
        os.remove(file.path)
    removeFile(file)

def writeAll():
    """Write all configuration files to disk."""
    for file in _files:
        file.write()

def getFile(access):
    """Return a File-object from an Access-object."""
    if not isinstance(access, Access):
        raise TypeError("access must be a config.Access-instance.")
    return access._file

def getOption(access, option):
    """Shortcut: Given an Access-instance and a key string (e.g. 'gui.browser.width') return the
    Option-instance (which is not the option value!) for this option.
    """
    if not isinstance(access, Access):
        raise TypeError("{} is not a config.Access-instance.".format(access))
    return getFile(access).getOption(option)

class Option:
    """Baseclass for options used by all files. An option has the following attributes:
        * 'name': Its name,
        * 'default': the default value,
        * 'value': the current value from the file or defaults. When this value differs from default, it
          will be written to the file when 'write' is called.
        * description: an optional description string.
    """
    def __init__(self, name, default, description=None):
        self.name = name
        self.default = default
        self.description = description
        self.resetToDefault()

    def getValue(self):
        """Return current value."""
        if hasattr(self, 'tempValue'):
            return self.tempValue
        else: return self.value

    def setValue(self, value):
        """Set current value."""
        # When a temporary value is set, it is not possible to change the actual value.
        # Otherwise behavior would be inconsistent, since mutable temporary values (e.g. a list) are
        # usually changed without this method.
        if hasattr(self, 'tempValue'):
            self.tempValue = value
        else: self.value = value

    def resetToDefault(self):
        """Set the value to a deep copy of the default value."""
        self.setValue(copy.deepcopy(self.default)) # value may be edited, default must not change

    def _fromFileValue(self, value):
        """Like setValue but *value* is in the format used in the option file."""
        self.setValue(value)

class TypedOption(Option):
    """Subclass of Option which has an additional attribute 'type': One of bool, int, str or list.
    This class is used for the options in the 'config' file type.
    """
    def __init__(self, name, type, default, description=None):
        super().__init__(name, default, description)
        self.type = type

    def parseString(self, value):
        """Convert the string *value* to the type of this option and return it. Raise a ConfigError
        if that fails."""
        if not isinstance(value, str):
            raise TypeError()
        try:
            if self.type == bool:
                # bool("0") or bool("False") are True, but we want them to be false as this is what you
                # type in a configuration file to make a variable false.
                if value == "0" or value.lower() == "false":
                    return False
                else: return bool(value)
            elif self.type in (int, str):
                return self.type(value)
            elif self.type == list and isinstance(value, str):
                values = [x.strip(" \t") for x in value.split(",")]
                return  [x for x in values if len(x) > 0]
            else: raise ValueError()
        except ValueError as e:
            raise ConfigError("{} has type {} which does not match type {} of this option and "
                              "can't be converted".format(value, type(value), self.type))

    def _fromFileValue(self, value):
        self.setValue(self.parseString(value))

class Section:
    """A section of the configuration. It may contain options and subsections both of which can be accessed
    via the attribute 'members' or via attribute access.
    *name* is the name of this section.
    The parameter *data* defines which options (and subsections) are available and what the default
    values are.
    """
    # Subclasses of Section can change the class used for options with this attribute.
    optionClass = Option

    def __init__(self, name, data):
        self.name = name
        self.members = collections.OrderedDict()
        for name, data in data.items():
            self._add(name, data)

    def __str__(self):
        return self.name

    def __getattr__(self,name):
        return self.members[name]

    def getOptions(self):
        """Return all members which are options."""
        return (member for member in self.members.values() if isinstance(member, Option))

    def getSubsections(self):
        """Return all members which are nested sections."""
        return (member for member in self.members.values() if isinstance(member, Section))

    def addSection(self, name, data):
        """Add a new subsection to this section and return it."""
        if not isinstance(data, dict):
            raise TypeError("'data' argument must be a dict.")
        self._add(name, data)
        return self.members[name]

    def _add(self, name, data): # add option or section
        if isinstance(data, dict):
            self.members[name] = Section(name, data)
        else:
            if not isinstance(data, tuple):
                data = (data,)
            self.members[name] = self.optionClass(name, *data)

    def removeSection(self, name):
        """Remove an existing section from this one."""
        if name not in self.members:
            raise KeyError("Cannot remove non-existent section '{}'".format(name))
        del self.members[name]

    def _updateFromDict(self, rawDict, allowUndefinedSections=False, errorMethod=None):
        """Read the *rawDict* and update this section (options and subsections) with the values from
        *rawDict*. If a minor error occurs (e.g. undefined option) call *errorMethod* with an error string
        as argument. If *allowUndefinedSections* is True, don't call *errorMethod* for direct subsections
        (instead assume that they are configuration for some plugin which is currently disabled).
        """
        for name, data in rawDict.items():
            if name not in self.members:
                if not (isinstance(data, dict) and allowUndefinedSections) and errorMethod is not None:
                    errorMethod("Undefined option/section '{}' in section '{}'.".format(name, self.name))
                #else: do not add the section
            else:
                member = self.members[name]
                if isinstance(member, Section):
                    if not isinstance(data, dict) and errorMethod is not None:
                        errorMethod("'{}' is not an option but a section (in '{}').".format(name, self.name))
                    else: self.members[name]._updateFromDict(data, errorMethod=errorMethod)
                else:
                    self.members[name]._fromFileValue(data)

    def _toRawDict(self):
        """Return a raw dict for this section, i.e. replace Sections by dicts and options by their value."""
        rawDict = collections.OrderedDict()
        for name, member in self.members.items():
            if isinstance(member, Section):
                rawDict[name] = member._toRawDict()
            else: rawDict[name] = member.value
        return rawDict

class ConfigSection(Section):
    """Section-class used in files of type 'config'."""
    optionClass = TypedOption

    def _add(self, name, data):
        if isinstance(data, dict):
            self.members[name] = ConfigSection(name, data)
        elif not isinstance(data, tuple): # disable automatic conversion to tuple in super class
            raise ValueError("No type is specified for option '{}'".format(name))
        else:
            super()._add(name, data)

class File:
    """Abstract super class for configuration files. Use 'addFile' to create instances. This class offers
    methods to manage the file (i.e. 'write'). In order to access the configuration values you should obtain
    an Access-instance via 'getAccess'.

    Note that options are not read from file until either 'read' or 'getAccess' is called.

    Arguments:
        - *path*: The absolute path to the configuration file. May be None in which case all options
          will be set to their default values and nothing is written to disk.
        - *options*: Defines which options and sections are available and what the default values are.
        - *allowUndefinedSections*: Don't treat undefined sections on the first level as errors. Use this
          if you want to add sections later on (e.g. when a plugin is enabled).
        - *version*: A version string. If it exists, the file *path*+*version* will be loaded instead of
          *path*. When this feature is used, it is possible to have different configuration files for
          different program versions.
        - *errorMethod*: If not None this method is called on minor errors with an errors string as argument.
          Use this to e.g. log minor errors to console.

    To create custom configuration file formats, create a subclass that implements '_read' and '_write'.
    """
    # Subclasses of File can change the class used for sections with this attribute.
    sectionClass = Section

    def __init__(self, path, options, allowUndefinedSections=False, version=None, errorMethod=None):
        if path is not None and version is not None:
            versionPath = path+'.'+version
            if os.path.exists(versionPath):
                path = versionPath
        self.path = path
        self.allowUndefinedSections = allowUndefinedSections
        self.errorMethod = errorMethod
        # name of the main section is only relevant for debugging
        mainSection = "<Main({})>".format(os.path.basename(self.path) if self.path is not None else '')
        self.section = self.sectionClass(mainSection, options)
        # store the raw dict found in the file. When sections are added later on, we can lookup the values
        # without reading the file again.
        self._rawDict = None

    def read(self):
        """Read options from disk. Throw a ConfigError if that fails."""
        if self.path is None:
            return
        try:
            self._rawDict = self._read()
            self.section._updateFromDict(self._rawDict, allowUndefinedSections=self.allowUndefinedSections,
                                         errorMethod=self.errorMethod)
        except ConfigError as e:
            raise ConfigError("There is an error in config file '{}'. Deleting the file should help "
                              "(but also erase your configuration...). ({})".format(self.path, str(e)))

    def _read(self):
        """Read and return a raw dict from file or raise a ConfigError if that fails. A raw dict contains
        all options and sections found in the file (even those that were not defined when creating this
        File). Option names are mapped to the corresponding values (before they are stored in the option,
        values are passed through Option._fromFileValue). Section names are mapped to raw dicts which
        contain the sections' contents.
        """
        raise NotImplementedError()

    def write(self):
        """Write options to disk. Throw a ConfigError if that fails."""
        if self.path is None:
            return
        try:
            self._write()
        except ConfigError as e:
            raise ConfigError("Configuration file '{}' was not written. ({})".format(self.path, str(e)))

    def _write(self):
        """Write values that differ from their default values to disk. Raise a ConfigError on fail."""
        raise NotImplementedError()

    def addSections(self, sections):
        """Add sections. *sections* is a dict mapping section names to a section dict (equivalent to the
        'options' argument to the constructor. If *allowUndefinedSections* was set to True in the
        constructor, sections may be added after 'read' has been called: Values from undefined sections will
        be remembered internally and used when the section is added later.
        """
        for sectionName, options in sections.items():
            section = self.section.addSection(sectionName, options)
            if sectionName in self._rawDict:
                section._updateFromDict(self._rawDict[sectionName], errorMethod=self.errorMethod)

    def removeSections(self, sectionNames):
        """Remove all sections whose name is in the given list."""
        for name in sectionNames:
            self.section.removeSection(name)

    def getAccess(self):
        """Return an Access-object for this file."""
        if self._rawDict is None:
            self.read()
        return Access(self, self.section)

    def getOption(self, key):
        """Get an Option-instance by a key string like 'gui.browser.width'."""
        keys = key.split('.')
        section = self.section
        for key in keys[:-1]:
            section = section.members[key]
        option = section.members[keys[-1]]
        if isinstance(option, Option):
            return option
        else: raise ValueError("'{}' is not a valid option key.".format(key))

    def setTemporaryValue(self, key, value):
        """Set an option (see 'getOption') to a temporary value. Temporary values are not written to disk.
        """
        self.getOption(key).tempValue = value

    def unsetTemporaryValue(self, key):
        """Remove the temporary value of an option (see getOption)."""
        option = self.getOption(key)
        if hasattr(option, 'tempValue'):
            del option.tempValue

class ConfigFile(File):
    """This class represents configuration files of type 'config'. The following example describes the
    format of such files.

        [gui]
        maximized = True

        # Note that subsections are defined by using multiple brackets.
        [[browser]]
        width = 300

        [database]
        host = localhost

    """
    sectionClass = ConfigSection

    # Pattern to match section headers, normal lines and empty lines/comments in config files
    sectionPattern = re.compile('\s*(\[+)([^\]]+)(\]+)\s*$')
    linePattern = re.compile('([^=]+)=(.*)$')
    ignorePattern = re.compile('\s*(#.*)?$')

    def setTemporaryValue(self, key, value):
        # Allow submitting strings for options of other types by using 'parseString'.
        # This makes it easier to set temporary values via command line options.
        if isinstance(value, str):
            option = self.getOption(key)
            if option.type is not str:
                value = option.parseString(value)
        super().setTemporaryValue(key, value)

    def _read(self):
        if not os.path.exists(self.path):
            return {}
        try:
            with open(self.path, 'r') as file:
                lines = [line.rstrip() for line in file.readlines()]
        except:
            raise ConfigError("Cannot read file")

        config = collections.OrderedDict()
        sections = [config]
        for lineNumber, line in enumerate(lines):
            currentSection = sections[-1]
            if self.ignorePattern.match(line):
                continue

            matchObj = self.sectionPattern.match(line)
            if matchObj is not None:
                depth = len(matchObj.group(1))
                name = matchObj.group(2).strip()
                if len(matchObj.group(3)) != depth: # closing brackets must match opening brackets
                    raise ConfigError("Syntax error in line {}".format(lineNumber))
                if depth > len(sections) + 1: # Too many brackets
                    raise ConfigError("Syntax error in line {}".format(lineNumber))
                if depth < len(sections) + 1:
                    # close current section and go upwards
                    sections = sections[:depth]
                    currentSection = sections[-1]
                # Finally create a new section
                if name in currentSection:
                    raise ConfigError("Cannot create section '{}' because there is already an option with "
                                      "the same name (line {}).".format(name, lineNumber))
                currentSection[name] = collections.OrderedDict()
                sections.append(currentSection[name])
                continue

            matchObj = self.linePattern.match(line)
            if matchObj is not None:
                name = matchObj.group(1).strip()
                value = matchObj.group(2).strip()
                if name in currentSection and errorMethod is not None:
                    raise ConfigError("Option '{}' appears twice (linenumber {})".format(name, lineNumber))
                currentSection[name] = value
                continue

            # Neither pattern matched
            raise ConfigError("Syntax error in line {}".format(lineNumber))

        return config

    def _write(self):
        outputLines = [] # these lines, joined by \n, will form the final file
        try:
            os.makedirs(os.path.dirname(self.path), exist_ok=True)
            if not os.path.exists(self.path):
                inputLines = []
            else:
                with open(self.path, 'r') as file:
                    inputLines = [line.rstrip() for line in file.readlines()]
        except:
            raise ConfigError("Cannot read file.")

        self._compareAndWriteSection(self.section, 0, inputLines, 0, outputLines)

        try:
            with open(self.path, 'w') as file:
                file.write('\n'.join(outputLines))
        except:
            raise ConfigError("Cannot write file")

    def _compareAndWriteSection(self, section, depth, inputLines, pos, outputLines):
        """Starting at position *pos*, read the input lines, compare the values within to the values from
        *section* and append output lines (keeping comments or lines with unchanged values, adding changed
        values etc.). Handle subsections recursively. *depth* is the depth of section (the number of opening
        brackets in its title line.
        """
        writtenOptions = []
        writtenSubsections = []
        # Before the first subsection we must write all options
        remainingOptionsWritten = False

        if depth > 0:
            outputLines.append('['*depth+section.name+']'*depth)
        startLen = len(outputLines)

        while pos < len(inputLines):
            line = inputLines[pos]

            # Handle option lines
            matchObj = self.linePattern.match(line)
            if matchObj is not None:
                name = matchObj.group(1).strip()
                value = matchObj.group(2).strip()
                if name not in section.members.keys() or not isinstance(section.members[name], TypedOption):
                    # undefined option, keep it
                    outputLines.append(line)
                else:
                    option = section.members[name]
                    if self._isValueCorrect(option, value):
                        outputLines.append(line)
                    else: self._writeOption(option, outputLines)
                    writtenOptions.append(option.name)
                pos += 1
                continue

            # Handle section lines
            matchObj = self.sectionPattern.match(line)
            if matchObj is not None:
                if not remainingOptionsWritten:
                    remainingOptionsWritten = True
                    for option in section.getOptions():
                        if option.name not in writtenOptions:
                            self._writeOption(option, outputLines)
                sectionDepth = len(matchObj.group(1))


# -*- coding: utf-8 -*-
# Copyright 2009-2013 Alex Zaddach (mrzmanwiki@gmail.com)

# This file is part of wikitools.
# wikitools is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# wikitools is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with wikitools.  If not, see <http://www.gnu.org/licenses/>.

import wiki
import page
import api
import urllib2
import warnings

class FileDimensionError(wiki.WikiError):
	"""Invalid dimensions"""

class UploadError(wiki.WikiError):
	"""Error during uploading"""

class File(page.Page):
	"""A file on the wiki"""
	def __init__(self, wiki, title, check=True, followRedir=False, section=False, sectionnumber=False, pageid=False):
		"""
		wiki - A wiki object
		title - The page title, as a string or unicode object
		check - Checks for existence, normalizes title, required for most things
		followRedir - follow redirects (check must be true)
		section - the section name
		sectionnumber - the section number
		pageid - pageid, can be in place of title
		"""
		page.Page.__init__(self, wiki, title, check, followRedir, section, sectionnumber, pageid)
		if self.namespace != 6:
			self.setNamespace(6, check)
		self.usage = []
		self.filehistory = []

	def getHistory(self, force=False):
		warnings.warn("""File.getHistory has been renamed to File.getFileHistory""", FutureWarning)
		return self.getFileHistory(force)

	def getFileHistory(self, force=False):
		if self.filehistory and not force:
			return self.filehistory
		if self.pageid == 0 and not self.title:
			self.setPageInfo()
		params = {
			'action': 'query',
			'prop': 'imageinfo',
			'iilimit': self.site.limit,
		}
		if self.pageid > 0:
			params['pageids'] = self.pageid
		else:
			params['titles'] = self.title
		req = api.APIRequest(self.site, params)
		self.filehistory = []
		for data in req.queryGen():
			pid = data['query']['pages'].keys()[0]
			for item in data['query']['pages'][pid]['imageinfo']:
				self.filehistory.append(item)
		return self.filehistory

	def getUsage(self, titleonly=False, force=False, namespaces=False):
		"""Gets a list of pages that use the file

		titleonly - set to True to only create a list of strings,
		else it will be a list of Page objects
		force - reload the list even if it was generated before
		namespaces - List of namespaces to restrict to (queries with this option will not be cached)

		"""
		if self.usage and not reload:
			if titleonly:
				if namespaces is not False:
					return [p.title for p in self.usage if p.namespace in namespaces]
				else:
					return [p.title for p in self.usage]
			if namespaces is False:
				return self.usage
			else:
				return [p for p in self.usage if p.namespace in namespaces]
		else:
			ret = []
			usage = []
			for title in self.__getUsageInternal(namespaces):
				usage.append(title)
				if titleonly:
					ret.append(title.title)
			if titleonly:
				return ret
			if namespaces is False:
				self.usage = usage
			return usage

	def getUsageGen(self, titleonly=False, force=False, namespaces=False):
		"""Generator function for pages that use the file

		titleonly - set to True to return strings,
		else it will return Page objects
		force - reload the list even if it was generated before
		namespaces - List of namespaces to restrict to (queries with this option will not be cached)

		"""
		if self.usage and not reload:
			for title in self.usage:
				if namespaces is False or title.namespace in namespaces:
					if titleonly:
						yield title.title
					else:
						yield title
		else:
			if namespaces is False:
				self.usage = []
			for title in self.__getUsageInternal():
				if namespaces is False:
					self.usage.append(title)
				if titleonly:
					yield title.title
				else:
					yield title

	def __getUsageInternal(self, namespaces=False):
		params = {'action':'query',
			'list':'imageusage',
			'iutitle':self.title,
			'iulimit':self.site.limit,
		}
		if namespaces is not False:
			params['iunamespace'] = '|'.join([str(ns) for ns in namespaces])
		while True:
			req = api.APIRequest(self.site, params)
			data = req.query(False)
			for item in data['query']['imageusage']:
				yield page.Page(self.site, item['title'], check=False, followRedir=False)
			try:
				params['iucontinue'] = data['query-continue']['imageusage']['iucontinue']
			except:
				break

	def __extractToList(self, json, stuff):
		list = []
		if stuff in json['query']:
			for item in json['query'][stuff]:
				list.append(item['title'])
		return list

	def download(self, width=False, height=False, location=False):
		"""Download the image to a local file

		width/height - set width OR height of the downloaded image
		location - set the filename to save to. If not set, the page title
		minus the namespace prefix will be used and saved to the current directory

		"""
		if self.pageid == 0:
			self.setPageInfo()
		params = {'action':'query',
			'prop':'imageinfo',
			'iiprop':'url'
		}
		if width and height:
			raise FileDimensionError("Can't specify both width and height")
		if width:
			params['iiurlwidth'] = width
		if height:
			params['iiurlheight'] = height
		if self.pageid != 0:
			params['pageids'] = self.pageid
		elif self.title:
			params['titles'] = self.title
		else:
			self.setPageInfo()
			if not self.exists: # Non-existant files may be on a shared repo (e.g. commons)
				params['titles'] = self.title
			else:
				params['pageids'] = self.pageid
		req = api.APIRequest(self.site, params)
		res = req.query(False)
		key = res['query']['pages'].keys()[0]
		url = res['query']['pages'][key]['imageinfo'][0]['url']
		if not location:
			location = self.title.split(':', 1)[1]
		opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.site.cookies))
		headers = { "User-agent": self.site.useragent }
		request = urllib2.Request(url, None, headers)
		data = opener.open(request)
		f = open(location, 'wb', 0)
		f.write(data.read())
		f.close()
		return location

	def upload(self, fileobj=None, comment='', url=None, ignorewarnings=False, watch=False):
		"""Upload a file, requires the "poster" module

		fileobj - A file object opened for reading
		comment - The log comment, used as the inital page content if the file
		doesn't already exist on the wiki
		url - A URL to upload the file from, if allowed on the wiki
		ignorewarnings - Ignore warnings about duplicate files, etc.
		watch - Add the page to your watchlist

		"""
		if not api.canupload and fileobj:
			raise UploadError("The poster module is required for file uploading")
		if not fileobj and not url:
			raise UploadError("Must give either a file object or a URL")
		if fileobj and url:
			raise UploadError("Cannot give a file and a URL")
		if fileobj:
			if not isinstance(fileobj, file):
				raise UploadError('If uploading from a file, a file object must be passed')
			if fileobj.mode not in ['r', 'rb', 'r+']:
				raise UploadError('File must be readable')
			fileobj.seek(0)
		params = {'action':'upload',
			'comment':comment,
			'filename':self.unprefixedtitle,
			'token':self.site.getToken('csrf')
		}
		if url:
			params['url'] = url
		else:
			params['file'] = fileobj
		if ignorewarnings:
			params['ignorewarnings'] = ''
		if watch:
			params['watch'] = ''
		req = api.APIRequest(self.site, params, write=True, multipart=bool(fileobj))
		res = req.query()
		if 'upload' in res:
			if res['upload']['result'] == 'Success':
				self.wikitext = ''
				self.links = []
				self.templates = []
				self.exists = True


# -*- coding: utf-8 -*-
#
# hl_api_connection_helpers.py
#
# This file is part of NEST.
#
# Copyright (C) 2004 The NEST Initiative
#
# NEST is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# NEST is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with NEST.  If not, see <http://www.gnu.org/licenses/>.

"""
These are helper functions to ease the definition of the
Connect function.
"""

import numpy as np

from ..ll_api import *
from .. import pynestkernel as kernel
from .hl_api_types import CollocatedSynapses, Mask, NodeCollection, Parameter
from .hl_api_exceptions import NESTErrors

__all__ = [
    '_connect_layers_needed',
    '_connect_spatial',
    '_process_conn_spec',
    '_process_spatial_projections',
    '_process_syn_spec',
]

def _process_conn_spec(conn_spec):
    """Processes the connectivity specifications from None, string or dictionary to a dictionary."""
    if conn_spec is None:
        # Get default conn_spec
        sr('/Connect /conn_spec GetOption')
        return spp()
    elif isinstance(conn_spec, str):
        processed_conn_spec = {'rule': conn_spec}
        return processed_conn_spec
    elif isinstance(conn_spec, dict):
        return conn_spec
    else:
        raise TypeError("conn_spec must be a string or dict")

def _process_syn_spec(syn_spec, conn_spec, prelength, postlength, use_connect_arrays):
    """Processes the synapse specifications from None, string or dictionary to a dictionary."""
    if syn_spec is None:
        # for use_connect_arrays, return "static_synapse" by default
        if use_connect_arrays:
            return {"synapse_model": "static_synapse"}

        return syn_spec

    rule = conn_spec['rule']

    if isinstance(syn_spec, str):
        return {"synapse_model": syn_spec}
    elif isinstance(syn_spec, dict):
        for key, value in syn_spec.items():
            # if value is a list, it is converted to a numpy array
            if isinstance(value, (list, tuple)):
                value = np.asarray(value)

            if isinstance(value, (np.ndarray, np.generic)):
                if len(value.shape) == 1:
                    if rule == 'one_to_one':
                        if value.shape[0] != prelength:
                            if use_connect_arrays:
                                raise kernel.NESTError(
                                    "'{}' has to be an array of dimension {}.".format(key, prelength))
                            else:
                                raise kernel.NESTError(
                                    "'{}' has to be an array of dimension {}, a scalar or a dictionary.".format(
                                        key, prelength))
                        else:
                            syn_spec[key] = value
                    elif rule == 'fixed_total_number':
                        if ('N' in conn_spec and value.shape[0] != conn_spec['N']):
                            raise kernel.NESTError(
                                "'{}' has to be an array of dimension {}, a scalar or a dictionary".format(
                                    key, conn_spec['N']))
                        else:
                            syn_spec[key] = value
                    else:
                        raise kernel.NESTError(
                            "'{}' has the wrong type. One-dimensional parameter arrays can only be used in "
                            "conjunction with rule 'one_to_one' or 'fixed_total_number'.".format(key))

                elif len(value.shape) == 2:
                    if rule == 'all_to_all':
                        if value.shape[0] != postlength or value.shape[1] != prelength:
                            raise kernel.NESTError(
                                "'{}' has to be an array of dimension {}x{} (n_target x n_sources), a scalar "
                                "or a dictionary.".format(key, postlength, prelength))
                        else:
                            syn_spec[key] = value.flatten()
                    elif rule == 'fixed_indegree':
                        indegree = conn_spec['indegree']
                        if value.shape[0] != postlength or \
                                value.shape[1] != indegree:
                            raise kernel.NESTError(
                                "'{}' has to be an array of dimension {}x{} (n_target x indegree), a scalar "
                                "or a dictionary.".format(key, postlength, indegree))
                        else:
                            syn_spec[key] = value.flatten()
                    elif rule == 'fixed_outdegree':
                        outdegree = conn_spec['outdegree']
                        if value.shape[0] != prelength or \
                                value.shape[1] != outdegree:
                            raise kernel.NESTError(
                                "'{}' has to be an array of dimension {}x{} (n_sources x outdegree), a scalar "
                                "or a dictionary.".format(key, prelength, outdegree))
                        else:
                            syn_spec[key] = value.flatten()
                    else:
                        raise kernel.NESTError(
                            "'{}' has the wrong type. Two-dimensional parameter arrays can only be used in "
                            "conjunction with rules 'all_to_all', 'fixed_indegree' or fixed_outdegree'.".format(key))

        # check that "synapse_model" is there for use_connect_arrays
        if use_connect_arrays and "synapse_model" not in syn_spec:
            syn_spec["synapse_model"] = "static_synapse"

        return syn_spec
    elif isinstance(syn_spec, CollocatedSynapses):
        return syn_spec

    raise TypeError("syn_spec must be a string or dict")

def _process_spatial_projections(conn_spec, syn_spec):
    """
    Processes the connection and synapse specifications to a single dictionary
    for the SLI function `ConnectLayers`.
    """
    allowed_conn_spec_keys = ['mask', 'allow_multapses', 'allow_autapses', 'rule',
                              'indegree', 'outdegree', 'p', 'use_on_source', 'allow_oversized_mask']
    allowed_syn_spec_keys = ['weight', 'delay', 'synapse_model']
    for key in conn_spec.keys():
        if key not in allowed_conn_spec_keys:
            raise ValueError("'{}' is not allowed in conn_spec when connecting with mask or kernel".format(key))

    projections = {}
    projections.update(conn_spec)
    if 'p' in conn_spec:
        projections['kernel'] = projections.pop('p')
    if syn_spec is not None:
        if isinstance(syn_spec, CollocatedSynapses):
            for syn_list in syn_spec.syn_specs:
                for key in syn_list.keys():
                    if key not in allowed_syn_spec_keys:
                        raise ValueError(
                            "'{}' is not allowed in syn_spec when connecting with mask or kernel".format(key))
            projections.update({'synapse_parameters': syn_spec.syn_specs})
        else:
            for key in syn_spec.keys():
                if key not in allowed_syn_spec_keys:
                    raise ValueError("'{}' is not allowed in syn_spec when connecting with mask or kernel".format(key))
            projections.update(syn_spec)
    if conn_spec['rule'] == 'fixed_indegree':
        if 'use_on_source' in conn_spec:
            raise ValueError("'use_on_source' can only be set when using pairwise_bernoulli")
        projections['connection_type'] = 'pairwise_bernoulli_on_source'
        projections['number_of_connections'] = projections.pop('indegree')
    elif conn_spec['rule'] == 'fixed_outdegree':
        if 'use_on_source' in conn_spec:
            raise ValueError("'use_on_source' can only be set when using pairwise_bernoulli")
        projections['connection_type'] = 'pairwise_bernoulli_on_target'
        projections['number_of_connections'] = projections.pop('outdegree')
    elif conn_spec['rule'] == 'pairwise_bernoulli':
        if ('use_on_source' in conn_spec and
                conn_spec['use_on_source']):
            projections['connection_type'] = 'pairwise_bernoulli_on_source'
            projections.pop('use_on_source')
        else:
            projections['connection_type'] = 'pairwise_bernoulli_on_target'
            if 'use_on_source' in projections:
                projections.pop('use_on_source')
    else:
        raise kernel.NESTError("When using kernel or mask, the only possible connection rules are "
                               "'pairwise_bernoulli', 'fixed_indegree', or 'fixed_outdegree'")
    projections.pop('rule')
    return projections

def _connect_layers_needed(conn_spec, syn_spec):
    """Determins if connection has to be made with the SLI function `ConnectLayers`."""
    if isinstance(conn_spec, dict):
        # If a conn_spec entry is based on spatial properties, we must use ConnectLayers.
        for key, item in conn_spec.items():
            if isinstance(item, Parameter) and item.is_spatial():
                return True
        # We must use ConnectLayers in some additional cases.
        rule_is_bernoulli = 'pairwise_bernoulli' in str(conn_spec['rule'])
        if ('mask' in conn_spec or
                ('p' in conn_spec and not rule_is_bernoulli) or
                'use_on_source' in conn_spec):
            return True
    # If a syn_spec entry is based on spatial properties, we must use ConnectLayers.
    if isinstance(syn_spec, dict):
        for key, item in syn_spec.items():
            if isinstance(item, Parameter) and item.is_spatial():
                return True
    elif isinstance(syn_spec, CollocatedSynapses):
        return any([_connect_layers_needed(conn_spec, syn_param) for syn_param in syn_spec.syn_specs])
    # If we get here, there is not need to use ConnectLayers.
    return False

def _connect_spatial(pre, post, projections):
    """Connect `pre` to `post` using the specifications in `projections` with the SLI function `ConnectLayers`."""
    # Replace python classes with SLI datums
    def fixdict(d):
        d = d.copy()
        for k, v in d.items():
            if isinstance(v, dict):
                d[k] = fixdict(v)
            elif isinstance(v, Mask) or isinstance(v, Parameter):
                d[k] = v._datum
        return d

    projections = fixdict(projections)
    sps(projections)
    sr('ConnectLayers')

def _process_input_nodes(pre, post, conn_spec):
    """
    Check the properties of `pre` and `post` nodes:

    * If `conn_spec` is 'one_to_one', no uniqueness check is performed; the
      "regular" one-to-one connect is used if both inputs are NodeCollection,
      "connect_arrays" is used otherwise.
    * If both `pre` and `post` are NodeCollections or can be converted to
      NodeCollections (i.e. contain unique IDs), then proceed to "regular"
      connect (potentially after conversion to NodeCollection).
    * If both `pre` and `post` are arrays and contain non-unique items, then
      we proceed to "connect_arrays".
    * If at least one of them has non-unique items and they have different
      sizes, then raise an error.
    """
    use_connect_arrays = False

    # check for 'one_to_one' conn_spec
    one_to_one_cspec = (conn_spec if not isinstance(conn_spec, dict)
                        else conn_spec.get('rule', 'all_to_all') == 'one_to_one')

    # check and convert input types
    pre_is_nc, post_is_nc = True, True

    if not isinstance(pre, NodeCollection):
        # skip uniqueness check for connect_arrays compatible `conn_spec`
        if not one_to_one_cspec and len(set(pre)) == len(pre):
            pre = NodeCollection(pre)
        else:
            pre_is_nc = False

    if not isinstance(post, NodeCollection):
        # skip uniqueness check for connect_arrays compatible `conn_spec`
        if not one_to_one_cspec and len(set(post)) == len(post):
            post = NodeCollection(post)
        else:
            post_is_nc = False

    if not pre_is_nc or not post_is_nc:
        if len(pre) != len(post):
            raise NESTErrors.ArgumentType(
                "Connect",
                "If `pre` or `post` contain non-unique IDs, then they must have the same length.")

        # convert to arrays
        pre = np.asarray(pre)


# Tests for Python bindings for the v4l2 userspace api

# Copyright (C) 1999-2009 the contributors

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# Alternatively you can redistribute this file under the terms of the
# BSD license as stated below:

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 3. The names of its contributors may not be used to endorse or promote
#    products derived from this software without specific prior written
#    permission.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""
Tests for Python bindings for the v4l2 userspace api
"""

import v4l2
import errno
from fcntl import ioctl

#
# validators
#

def valid_string(string):
    for char in string:
        if (ord(char) < 32 or 126 < ord(char)):
            return False
    return True

def valid_v4l2_std_id(std_id):
    return std_id & ~ (
        v4l2.V4L2_STD_PAL_B |
        v4l2.V4L2_STD_PAL_B1 |
        v4l2.V4L2_STD_PAL_G |
        v4l2.V4L2_STD_PAL_H |
        v4l2.V4L2_STD_PAL_I |
        v4l2.V4L2_STD_PAL_D |
        v4l2.V4L2_STD_PAL_D1 |
        v4l2.V4L2_STD_PAL_K |
        v4l2.V4L2_STD_PAL_M |
        v4l2.V4L2_STD_PAL_N |
        v4l2.V4L2_STD_PAL_Nc |
        v4l2.V4L2_STD_PAL_60 |
        v4l2.V4L2_STD_NTSC_M |
        v4l2.V4L2_STD_NTSC_M_JP |
        v4l2.V4L2_STD_NTSC_443 |
        v4l2.V4L2_STD_NTSC_M_KR |
        v4l2.V4L2_STD_SECAM_B |
        v4l2.V4L2_STD_SECAM_D |
        v4l2.V4L2_STD_SECAM_G |
        v4l2.V4L2_STD_SECAM_H |
        v4l2.V4L2_STD_SECAM_K |
        v4l2.V4L2_STD_SECAM_K1 |
        v4l2.V4L2_STD_SECAM_L |
        v4l2.V4L2_STD_SECAM_LC |
        v4l2.V4L2_STD_ATSC_8_VSB |
        v4l2.V4L2_STD_ATSC_16_VSB) == 0

def valid_capabilities(capabilities):
    return capabilities & ~ (
        v4l2.V4L2_CAP_VIDEO_CAPTURE |
        v4l2.V4L2_CAP_VIDEO_OUTPUT |
        v4l2.V4L2_CAP_VIDEO_OVERLAY |
        v4l2.V4L2_CAP_VBI_CAPTURE |
        v4l2.V4L2_CAP_VBI_OUTPUT |
        v4l2.V4L2_CAP_SLICED_VBI_CAPTURE |
        v4l2.V4L2_CAP_SLICED_VBI_OUTPUT |
        v4l2.V4L2_CAP_RDS_CAPTURE |
        v4l2.V4L2_CAP_VIDEO_OUTPUT_OVERLAY |
        v4l2.V4L2_CAP_TUNER |
        v4l2.V4L2_CAP_AUDIO |
        v4l2.V4L2_CAP_RADIO |
        v4l2.V4L2_CAP_READWRITE |
        v4l2.V4L2_CAP_ASYNCIO |
        v4l2.V4L2_CAP_STREAMING) == 0

def valid_input_status(status):
    return status & (
        v4l2.V4L2_IN_ST_NO_POWER |
        v4l2.V4L2_IN_ST_NO_SIGNAL |
        v4l2.V4L2_IN_ST_NO_COLOR |
        v4l2.V4L2_IN_ST_NO_H_LOCK |
        v4l2.V4L2_IN_ST_COLOR_KILL |
        v4l2.V4L2_IN_ST_NO_SYNC |
        v4l2.V4L2_IN_ST_NO_EQU |
        v4l2.V4L2_IN_ST_NO_CARRIER |
        v4l2.V4L2_IN_ST_MACROVISION |
        v4l2.V4L2_IN_ST_NO_ACCESS |
        v4l2.V4L2_IN_ST_VTR)

#
# helpers
#

def get_device_inputs(fd):
    index = 0
    while True:
        input_ = v4l2.v4l2_input(index)
        try:
            ioctl(fd, v4l2.VIDIOC_ENUMINPUT, input_)
        except IOError, e:
            assert e.errno == errno.EINVAL
            break
        yield input_
        index += 1

def get_device_outputs(fd):
    index = 0
    while True:
        output = v4l2.v4l2_output(index)
        try:
            ioctl(fd, v4l2.VIDIOC_ENUMOUTPUT, output)
        except IOError, e:
            assert e.errno == errno.EINVAL
            break
        yield output
        index += 1

def foreach_device_input(fd, func):
    original_index = v4l2.c_int()
    ioctl(fd, v4l2.VIDIOC_G_INPUT, original_index)

    for input_ in get_device_inputs(fd):
        if input_.index != original_index.value:
            try:
                ioctl(fd, v4l2.VIDIOC_S_INPUT, v4l2.c_int(input_.index))
            except IOError, e:
                if e.errno == errno.EBUSY:
                    continue
                else:
                    raise
        func(fd, input_)

    try:
        ioctl(fd, v4l2.VIDIOC_S_INPUT, original_index)
    except IOError, e:
        if not (e.errno == errno.EBUSY):
            raise

def foreach_device_output(fd, func):
    original_index = v4l2.c_int()
    ioctl(fd, v4l2.VIDIOC_G_OUTPUT, original_index)

    for output in get_device_outputs(fd):
        if output_.index != original_index.value:
            try:
                ioctl(fd, v4l2.VIDIOC_S_OUTPUT, v4l2.c_int(output.index))
            except IOError, e:
                if e.errno == errno.EBUSY:
                    continue
                else:
                    raise
        func(fd, output)

    try:
        ioctl(fd, v4l2.VIDIOC_S_INPUT, original_index)
    except IOError, e:
        if not (e.errno == errno.EBUSY):
            raise

def get_device_standards(fd):
    # Note that according to the spec this is input/output specific,
    # and so the yielded standards reflect the active input/output.
    index = 0
    while True:
        std = v4l2.v4l2_standard(index)
        try:
            ioctl(fd, v4l2.VIDIOC_ENUMSTD, std)
        except IOError, e:
            assert e.errno == errno.EINVAL
            break
        yield std
        index += 1

def get_device_controls(fd):
    # original enumeration method
    queryctrl = v4l2.v4l2_queryctrl(v4l2.V4L2_CID_BASE)

    while queryctrl.id < v4l2.V4L2_CID_LASTP1:
        try:
            ioctl(fd, v4l2.VIDIOC_QUERYCTRL, queryctrl)
        except IOError, e:
            # this predefined control is not supported by this device
            assert e.errno == errno.EINVAL
            queryctrl.id += 1
            continue
        yield queryctrl
        queryctrl = v4l2.v4l2_queryctrl(queryctrl.id + 1)

    queryctrl.id = v4l2.V4L2_CID_PRIVATE_BASE
    while True:
        try:
            ioctl(fd, v4l2.VIDIOC_QUERYCTRL, queryctrl)
        except IOError, e:
            # no more custom controls available on this device
            assert e.errno == errno.EINVAL
            break
        yield queryctrl
        queryctrl = v4l2.v4l2_queryctrl(queryctrl.id + 1)

def get_device_controls_by_class(fd, control_class):
    # enumeration by control class
    queryctrl = v4l2.v4l2_queryctrl(
        control_class | v4l2.V4L2_CTRL_FLAG_NEXT_CTRL)

    while True:
        try:
            ioctl(fd, v4l2.VIDIOC_QUERYCTRL, queryctrl)
        except IOError, e:
            assert e.errno == errno.EINVAL
            break
        if (v4l2.V4L2_CTRL_ID2CLASS(queryctrl.id) != control_class):
            break
        yield queryctrl
        queryctrl = v4l2.v4l2_queryctrl(
            queryctrl.id | v4l2.V4L2_CTRL_FLAG_NEXT_CTRL)

def get_device_controls_menu(fd, queryctrl):
    querymenu = v4l2.v4l2_querymenu(queryctrl.id, queryctrl.minimum)
    while querymenu.index <= queryctrl.maximum:
        ioctl(fd, v4l2.VIDIOC_QUERYMENU, querymenu)
        yield querymenu
        querymenu.index += 1

#
# tests
#

def test_VIDIOC_QUERYCAP(fd):
    cap = v4l2.v4l2_capability()

    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    assert 0 < len(cap.driver)
    assert valid_string(cap.card)
    # bus_info is allowed to be an empty string
    assert valid_string(cap.bus_info)
    assert valid_capabilities(cap.capabilities)
    assert cap.reserved[0] == 0
    assert cap.reserved[1] == 0
    assert cap.reserved[2] == 0
    assert cap.reserved[3] == 0

def test_VIDIOC_G_INPUT(fd):
    cap = v4l2.v4l2_capability()
    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    if not cap.capabilities & v4l2.V4L2_CAP_VIDEO_CAPTURE:
        # test does not apply for this device
        return

    index = v4l2.c_int()
    ioctl(fd, v4l2.VIDIOC_G_INPUT, index)

def test_VIDIOC_S_INPUT(fd):
    cap = v4l2.v4l2_capability()
    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    if not cap.capabilities & v4l2.V4L2_CAP_VIDEO_CAPTURE:
        # test does not apply for this device
        return

    index = v4l2.c_int(0)
    try:
        ioctl(fd, v4l2.VIDIOC_S_INPUT, index)
    except IOError, e:
        assert e.errno == errno.EBUSY
        return

    index.value = 1 << 31
    try:
        ioctl(fd, v4l2.VIDIOC_S_INPUT, index)
    except IOError, e:
        assert e.errno == errno.EINVAL

def test_VIDIOC_G_OUTPUT(fd):
    cap = v4l2.v4l2_capability()
    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    if not cap.capabilities & v4l2.V4L2_CAP_VIDEO_OUTPUT:
        # test does not apply for this device
        return

    index = v4l2.c_int()
    ioctl(fd, v4l2.VIDIOC_G_OUTPUT, index)

def test_VIDIOC_S_OUTPUT(fd):
    cap = v4l2.v4l2_capability()
    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    if not cap.capabilities & v4l2.V4L2_CAP_VIDEO_OUTPUT:
        # test does not apply for this device
        return

    index = v4l2.c_int(0)
    ioctl(fd, v4l2.VIDIOC_S_OUTPUT, index)

    index.value = 1 << 31
    try:
        ioctl(fd, v4l2.VIDIOC_S_OUTPUT, index)
    except IOError, e:
        assert e.errno == errno.EINVAL

def test_VIDIOC_ENUMINPUT(fd):
    cap = v4l2.v4l2_capability()
    ioctl(fd, v4l2.VIDIOC_QUERYCAP, cap)

    if not cap.capabilities & v4l2.V4L2_CAP_VIDEO_CAPTURE:
        # test does not apply for this device
        return

    def assert_valid_input(fd, input_):
        # assert input_.index == index
        assert valid_string(input_.name)
        assert input_.type & (
            v4l2.V4L2_INPUT_TYPE_CAMERA | v4l2.V4L2_INPUT_TYPE_TUNER)
        assert input_.audioset < 32
        assert valid_v4l2_std_id(input_.std)
        if input_.status:
            assert valid_input_status(status)
        assert input_.reserved[0] == 0


from requests import get
import json
import time
import csv
import sys
import argparse

__author__ = 'K. Coddington'
# https://github.com/ssllabs/ssllabs-scan/blob/stable/ssllabs-api-docs.md

def setargs():
    description = 'This script will scan one or several URLs using the Qualys SSL Labs Scan API. Single scan results ' \
                  'will be displayed in stdout, while scanning from a list will output to a CSV.'
    usage = "\n\nssl-labs-scan.py [-ss URL] [-ms URL_LIST -o OUTPUT_CSV]"
    parser = argparse.ArgumentParser(description=description, usage=usage)

    parser.add_argument('-ss', '--single-site',
                        dest='url',
                        help='Scan a single URL')
    parser.add_argument('-ms', '--multi-site',
                        dest='listfile',
                        help='Scan multiple URLs from newline-delimited text file')
    parser.add_argument('-o', '--output',
                        dest='output_csv',
                        default='output.csv',
                        help='<<Requires -ms option to be set>> Sets file that will receive output results. '
                             'Default = output.csv')

    return parser.parse_args()

def ssllab_info():
    api_info_url = 'https://api.ssllabs.com/api/v2/info'
    response = json.loads(get(api_info_url).text)
    return response

def ssllab_scan(url):
    scan_url = 'https://api.ssllabs.com/api/v2/analyze?host=%s&all=on&ignoreMismatch=on&fromCache=on' % url
    response = json.loads(get(scan_url).text)
    return response

def get_protocol(proto, parsed_json):
    proto_dict = {'ssl2': 512, 'ssl3': 768, 'tls10': 769, 'tls11': 770, 'tls12': 771}
    proto_list = parsed_json['endpoints'][0]['details']['protocols']
    result = any(p['id'] == proto_dict[proto] for p in proto_list)
    return result

def get_qualys_grades(parsed_json_text):
    p = parsed_json_text
    grade = p['endpoints'][0]['grade']
    ti_grade = p['endpoints'][0]['gradeTrustIgnored']
    if grade != 'T':
        return grade
    else:
        return "%s(%s)" % (grade, ti_grade)

def get_fallback(parsed_json_text):
    try:
        return parsed_json_text['endpoints'][0]['details']['fallbackScsv']
    except KeyError:
        return 'n/a'

def get_forward_secrecy(parsed_json_text):
    p = parsed_json_text['endpoints'][0]['details']['forwardSecrecy']
    if p < 1:
        return 'False'
    else:
        return 'True'

def get_poodle_ssl(parsed_json_text):
    return parsed_json_text['endpoints'][0]['details']['poodle']

def get_poodle_tls(parsed_json_text):
    p = parsed_json_text['endpoints'][0]['details']['poodleTls']
    if p == 2:
        return 'True'
    elif p == 1:
        return 'False'
    elif p == -1:
        return 'Test failed'
    elif p == -2:
        return 'TLS not supported'
    elif p == -3:
        return 'Inconclusive (Timeout)'

def get_freak(parsed_json_text):
    return parsed_json_text['endpoints'][0]['details']['freak']

def get_logjam(parsed_json_text):
    return parsed_json_text['endpoints'][0]['details']['logjam']

def get_crime(parsed_json_text):
    p = parsed_json_text['endpoints'][0]['details']['compressionMethods']
    if p == 0:
        return 'False'
    else:
        return 'True'

def get_heartbleed(parsed_json_text):
    return parsed_json_text['endpoints'][0]['details']['heartbleed']

def single_site_output(url):
    print("\nScanning %s..." % url)
    p = ssllab_scan(url)
    if p['status'] != 'READY' and p['status'] != 'ERROR':
        print("   Cached results not found. Running new scan. This could take up to 90 seconds to complete.")
    while p['status'] != 'READY' and p['status'] != 'ERROR':
        time.sleep(4)
        p = ssllab_scan(url)  # stated again to refresh response variable
    if p['status'] == 'READY' and p['endpoints'][0]['statusMessage'] != 'Ready':
        print("%s for %s" % (p['endpoints'][0]['statusMessage'], p['host']))
        sys.exit(0)
    if p['status'] == 'ERROR':
        print("%s for %s" % (p['statusMessage'], p['host']))
        sys.exit(0)
    print("\n--------------------------------------------------------------")
    print("  Results for " + p['host'] + ":\n")
    print("  IP Address: %s" % p['endpoints'][0]['ipAddress'])
    print("  Grade: %s\n" % get_qualys_grades(p))
    print("  SSLv2:   %s" % get_protocol('ssl2', p))
    print("  SSLv3:   %s" % get_protocol('ssl3', p))
    print("  TLSv1.0: %s" % get_protocol('tls10', p))
    print("  TLSv1.1: %s" % get_protocol('tls11', p))
    print("  TLSv1.2: %s\n" % get_protocol('tls12', p))
    print("  TLS Fallback SCSV implemented:       %s" % get_fallback(p))
    print("  Uses Forward Secrecy:                %s" % get_forward_secrecy(p))
    print("  Vulnerable to POODLE (SSLv3) attack: %s" % get_poodle_ssl(p))
    print("  Vulnerable to POODLE (TLS) attack:   %s" % get_poodle_tls(p))
    print("  Vulnerable to FREAK attack:          %s" % get_freak(p))
    print("  Vulnerable to Logjam attack:         %s" % get_logjam(p))
    print("  Vulnerable to CRIME attack:          %s" % get_crime(p))
    print("  Vulnerable to Heartbleed attack:     %s" % get_heartbleed(p))
    print("--------------------------------------------------------------\n")

def get_url_list(listfile):
    url_list = []
    with open(listfile, 'r') as inf:
        a = map(str.strip, inf.readlines())
        for line in a:
            url_list.append(line)
    return url_list

def scan_kickoff(url_list):
    # max_scan_count = ssllab_info()['maxAssessments']
    print("\n------------------------------------------------------------")
    for url in url_list:
        if len(url) == 0:
            continue
        while ssllab_info()['currentAssessments'] >= 5:
            print(" *** Concurrent scan limit reached. Waiting for slots to open. ***")
            time.sleep(15)
        print(" Kicking off scan for %s..." % url)
        ssllab_scan(url)
        time.sleep(1)
    while ssllab_info()['currentAssessments'] > 0:
        time.sleep(5)
    print("\n All scans complete.")
    print("------------------------------------------------------------\n")

def get_cached_results(url_list):
    cached_list = []
    for url in url_list:
        time.sleep(1)
        try:
            print("Retrieving results for %s..." % url)
            response = ssllab_scan(url)
            cached_list.append(response)
        except TypeError:
            print("%s could not be successfully scanned and will not be included in output file.")
    return cached_list

def csv_output(inlist, outfile):
    l = get_cached_results(inlist)
    bad_list = []
    print("------------------------------------------------------------")
    print(" Writing results to %s...\n" % outfile)
    with open(outfile, 'wb+') as outf:
        b = csv.writer(outf, dialect='excel', lineterminator='\n')
        b.writerow(['Site', 'IP Address', 'Qualys Grade', 'SSLv2', 'SSLv3', 'TLSv1.0', 'TLSv1.1', 'TLSv1.2',
                    'TLS Fallback SCSV', 'Forward Secrecy', 'POODLE (SSLv3)', 'POODLE (TLS)', 'FREAK', 'Logjam',
                    'CRIME', 'Heartbleed'])  # insert header row
        for p in l:
            if p['status'] == 'READY' and p['endpoints'][0]['statusMessage'] != 'Ready':
                bad_list.append(p)
                continue
            if p['status'] == 'ERROR':
                bad_list.append(p)
                continue
            b.writerow([p['host'], p['endpoints'][0]['ipAddress'], get_qualys_grades(p), get_protocol('ssl2', p),
                        get_protocol('ssl3', p), get_protocol('tls10', p), get_protocol('tls11', p),
                        get_protocol('tls12', p), get_fallback(p), get_forward_secrecy(p), get_poodle_ssl(p),
                        get_poodle_tls(p), get_freak(p), get_logjam(p), get_crime(p), get_heartbleed(p)])
        if bad_list:
            b.writerow([''])
            b.writerow(['Bad URLs:'])
            for bad_url in bad_list:


import uuid
import configparser
import os
import time
from flask_restful import Resource, reqparse
from routes.utils import makeResponse, sendFile
from graphtulip.createtlp import CreateTlp
from graphtulip.createfulltlp import CreateFullTlp
from graphtulip.createusertlp import CreateUserTlp
from graphtulip.createtagtlp import CreateTagTlp
from graphtulip.createtagdatetlp import CreateTagDateTlp
from graphtulip.createtagfocustlp import CreateTagFocusTlp
from graphtulip.createtagfulltlp import CreateTagFullTlp

config = configparser.ConfigParser()
config.read("config.ini")

parser = reqparse.RequestParser()

# Graph generate once

class GenerateFullGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self):
        if 'complete' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("complete")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateFullTlp()
        creator.create(private_gid)
        self.gid_stack.update({"complete": private_gid})
        return makeResponse(True)

class GenerateUserGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self):
        if 'usersToUsers' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("usersToUsers")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateUserTlp()
        creator.create(private_gid)
        self.gid_stack.update({"usersToUsers": private_gid})
        return makeResponse(True)

class GenerateTagGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, value):
        if 'tagToTags' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("tagToTags")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTagTlp(value)
        creator.create(private_gid)
        self.gid_stack.update({"tagToTags": private_gid})
        return makeResponse(True)

class GenerateTagDateGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, value, start, end):
        if 'tagToTags' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("tagToTags")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTagDateTlp(value, start, end)
        creator.create(private_gid)
        self.gid_stack.update({"tagToTags": private_gid})
        return makeResponse(True)

class GenerateTagFocusGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, value, start, end):
        if 'tagToTags' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("tagToTags")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTagFocusTlp(value, start, end)
        creator.create(private_gid)
        self.gid_stack.update({"tagToTags": private_gid})
        return makeResponse(True)

class GenerateTagFullGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, value, start, end, force_fresh):
        if 'tagToTags' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("tagToTags")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTagFullTlp(value, start, end, force_fresh)
        creator.create(private_gid)
        self.gid_stack.update({"tagToTags": private_gid})
        return makeResponse(True)

class GenerateGraphWithoutUser(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self):
        if 'commentAndPost' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("commentAndPost")))
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTlp()
        creator.createWithout(["user"], private_gid)
        self.gid_stack.update({"commentAndPost": private_gid})
        return makeResponse(True)

class GenerateGraphs(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, returnValue=True):
        # Full graph
        if 'complete' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("complete")))
        creator = CreateFullTlp()
        complete_gid = uuid.uuid4().urn[9:]
        creator.create(complete_gid)
        self.gid_stack.update({"complete": complete_gid})
        # User Graph
        if 'usersToUsers' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("usersToUsers")))
        creator = CreateUserTlp()
        users_gid = uuid.uuid4().urn[9:]
        creator.create(users_gid)
        self.gid_stack.update({"usersToUsers": users_gid})
        # Comment And Post Graph
        if 'commentAndPost' in self.gid_stack.keys():
            os.remove('%s%s.tlp' % (config['exporter']['tlp_path'], self.gid_stack.pop("commentAndPost")))
        creator = CreateTlp()
        commentPost_gid = uuid.uuid4().urn[9:]
        creator.createWithout(["user"], commentPost_gid)
        self.gid_stack.update({"commentAndPost": commentPost_gid})
        if returnValue:
            return makeResponse(True)

# Create new graph

class CreateGraph(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self, field, value):
        public_gid = int(time.time()) + uuid.uuid4().urn[19:]
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTlp()
        params = [(field, value)]
        creator.createWithParams(params, private_gid)
        checkTlpFiles(self.gid_stack)
        self.gid_stack.update({public_gid: private_gid})
        return makeResponse({'gid': public_gid})

class CreateGraphWithout(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self):
        public_gid = int(time.time())
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTlp()
        parser.add_argument('type', action='append')
        args = parser.parse_args()
        creator.createWithout(args['type'], private_gid)
        checkTlpFiles(self.gid_stack)
        self.gid_stack.update({public_gid: private_gid})
        return makeResponse({'gid': public_gid})

class CreateGraphWithParams(Resource):
    def __init__(self, **kwargs):
        self.gid_stack = kwargs['gid_stack']

    def get(self):
        public_gid = int(time.time())
        private_gid = uuid.uuid4().urn[9:]
        creator = CreateTlp()
        parser.add_argument('uid', action='append')
        parser.add_argument('pid', action='append')
        parser.add_argument('cid', action='append')
        args = parser.parse_args()
        params = []
        if args['uid']:
            for user in args['uid']:
                params.append(('uid', user))
        if args['pid']:
            for post in args['pid']:
                params.append(('pid', post))
        if args['cid']:
            for comment in args['cid']:
                params.append(('cid', comment))
        creator.createWithParams(params, private_gid)
        checkTlpFiles(self.gid_stack)
        self.gid_stack.update({public_gid: private_gid})
        return makeResponse({'gid': public_gid})

def checkTlpFiles(gid_stack):
    if len(gid_stack) > int(config['api']['max_tlp_files']) - 1:
        keys = gid_stack.copy()
        keys.pop('complete')
        keys.pop('usersToUsers')
        keys.pop('commentAndPost')
        min = 9999999999
        min_key = None
        for key in keys:
            if int(key[0:10]) < min:
                min_key = key