# -*- coding: utf-8 -*-

# Form implementation generated from reading ui file 'rbhusPipeNotesMod.ui'
#
# Created by: PyQt4 UI code generator 4.12
#
# WARNING! All changes made in this file will be lost!

from PyQt4 import QtCore, QtGui

try:
  _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
  def _fromUtf8(s):
    return s

try:
  _encoding = QtGui.QApplication.UnicodeUTF8
  def _translate(context, text, disambig):
    return QtGui.QApplication.translate(context, text, disambig, _encoding)
except AttributeError:
  def _translate(context, text, disambig):
    return QtGui.QApplication.translate(context, text, disambig)

class Ui_MainWindow(object):
  def setupUi(self, MainWindow):
    MainWindow.setObjectName(_fromUtf8("MainWindow"))
    MainWindow.resize(567, 780)
    MainWindow.setDocumentMode(False)
    self.centralwidget = QtGui.QWidget(MainWindow)
    self.centralwidget.setObjectName(_fromUtf8("centralwidget"))
    self.verticalLayout = QtGui.QVBoxLayout(self.centralwidget)
    self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
    self.splitter = QtGui.QSplitter(self.centralwidget)
    sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.MinimumExpanding, QtGui.QSizePolicy.Expanding)
    sizePolicy.setHorizontalStretch(0)
    sizePolicy.setVerticalStretch(0)
    sizePolicy.setHeightForWidth(self.splitter.sizePolicy().hasHeightForWidth())
    self.splitter.setSizePolicy(sizePolicy)
    self.splitter.setOrientation(QtCore.Qt.Vertical)
    self.splitter.setObjectName(_fromUtf8("splitter"))
    self.scrollArea = QtGui.QScrollArea(self.splitter)
    sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Maximum)
    sizePolicy.setHorizontalStretch(0)
    sizePolicy.setVerticalStretch(0)
    sizePolicy.setHeightForWidth(self.scrollArea.sizePolicy().hasHeightForWidth())
    self.scrollArea.setSizePolicy(sizePolicy)
    palette = QtGui.QPalette()
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.WindowText, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Button, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Light, brush)
    brush = QtGui.QBrush(QtGui.QColor(219, 218, 217))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Midlight, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Dark, brush)
    brush = QtGui.QBrush(QtGui.QColor(123, 121, 119))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Mid, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Text, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.BrightText, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.ButtonText, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Base, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Window, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.Shadow, brush)
    brush = QtGui.QBrush(QtGui.QColor(219, 218, 217))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.AlternateBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 220))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.ToolTipBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Active, QtGui.QPalette.ToolTipText, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.WindowText, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Button, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Light, brush)
    brush = QtGui.QBrush(QtGui.QColor(219, 218, 217))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Midlight, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Dark, brush)
    brush = QtGui.QBrush(QtGui.QColor(123, 121, 119))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Mid, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Text, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.BrightText, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.ButtonText, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Base, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Window, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.Shadow, brush)
    brush = QtGui.QBrush(QtGui.QColor(219, 218, 217))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.AlternateBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 220))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.ToolTipBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Inactive, QtGui.QPalette.ToolTipText, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.WindowText, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Button, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Light, brush)
    brush = QtGui.QBrush(QtGui.QColor(219, 218, 217))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Midlight, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Dark, brush)
    brush = QtGui.QBrush(QtGui.QColor(123, 121, 119))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Mid, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Text, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 255))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.BrightText, brush)
    brush = QtGui.QBrush(QtGui.QColor(92, 91, 89))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.ButtonText, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Base, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Window, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.Shadow, brush)
    brush = QtGui.QBrush(QtGui.QColor(184, 182, 179))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.AlternateBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(255, 255, 220))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.ToolTipBase, brush)
    brush = QtGui.QBrush(QtGui.QColor(0, 0, 0))
    brush.setStyle(QtCore.Qt.SolidPattern)
    palette.setBrush(QtGui.QPalette.Disabled, QtGui.QPalette.ToolTipText, brush)
    self.scrollArea.setPalette(palette)
    self.scrollArea.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
    self.scrollArea.setWidgetResizable(True)
    self.scrollArea.setAlignment(QtCore.Qt.AlignLeading|QtCore.Qt.AlignLeft|QtCore.Qt.AlignTop)
    self.scrollArea.setObjectName(_fromUtf8("scrollArea"))
    self.scrollAreaWidgetContents = QtGui.QWidget()
    self.scrollAreaWidgetContents.setGeometry(QtCore.QRect(0, 0, 551, 70))
    self.scrollAreaWidgetContents.setObjectName(_fromUtf8("scrollAreaWidgetContents"))
    self.verticalLayout_2 = QtGui.QVBoxLayout(self.scrollAreaWidgetContents)
    self.verticalLayout_2.setMargin(0)
    self.verticalLayout_2.setObjectName(_fromUtf8("verticalLayout_2"))
    self.scrollArea.setWidget(self.scrollAreaWidgetContents)
    self.textEdit = QtGui.QTextEdit(self.splitter)
    sizePolicy = QtGui.QSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
    sizePolicy.setHorizontalStretch(0)
    sizePolicy.setVerticalStretch(0)
    sizePolicy.setHeightForWidth(self.textEdit.sizePolicy().hasHeightForWidth())
    self.textEdit.setSizePolicy(sizePolicy)
    self.textEdit.setMinimumSize(QtCore.QSize(0, 2))
    self.textEdit.setMaximumSize(QtCore.QSize(16777215, 16777215))
    self.textEdit.setAutoFormatting(QtGui.QTextEdit.AutoNone)
    self.textEdit.setTextInteractionFlags(QtCore.Qt.LinksAccessibleByMouse|QtCore.Qt.TextEditable|QtCore.Qt.TextEditorInteraction|QtCore.Qt.TextSelectableByKeyboard|QtCore.Qt.TextSelectableByMouse)
    self.textEdit.setObjectName(_fromUtf8("textEdit"))
    self.verticalLayout.addWidget(self.splitter)
    self.horizontalLayout_3 = QtGui.QHBoxLayout()
    self.horizontalLayout_3.setObjectName(_fromUtf8("horizontalLayout_3"))
    spacerItem = QtGui.QSpacerItem(40, 20, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
    self.horizontalLayout_3.addItem(spacerItem)
    self.pushSend = QtGui.QPushButton(self.centralwidget)
    self.pushSend.setObjectName(_fromUtf8("pushSend"))
    self.horizontalLayout_3.addWidget(self.pushSend)
    self.verticalLayout.addLayout(self.horizontalLayout_3)
    MainWindow.setCentralWidget(self.centralwidget)


## GCEXfstests Dashboard
## Author: Harshad Shirwadkar (harshadshirwadkar@gmail.com)

import os
import csv
import subprocess

from flask import Flask
from datetime import datetime
from junitparser import JUnitXml, Property, Properties, Failure, Error, Skipped

app = Flask(__name__)

mirror_dir = os.environ.get("LOCAL_MIRROR_PATH", "/tmp/mirror")
extracted_dir = os.environ.get("LOCAL_EXTRACT_PATH", "/tmp/extracted/")
results_gs_path = os.environ.get("RESULTS_GS_PATH", "")
uncategorized_category = "Uncategorized"

def results_header():
    return """<title>GCEXfstests Dashboard</title>

GCE XFSTests Dashboard

Home	Browse

"""

def result_summary(testsuite):
    if testsuite.errors == 0 and testsuite.failures == 0:
        return ("lightgreen", "Passed at %s" % testsuite.timestamp[11:])
    return ("yellow", "Failed at %s" % testsuite.timestamp[11:])

def get_results(dirroot):
    """Return a list of files named results.xml in a directory hierarchy"""
    for dirpath, dirs, filenames in os.walk(dirroot):
        if 'results.xml' in filenames:
            yield dirpath + '/results.xml'

def get_property(props, key):
    """Return the value of the first property with the given name"""
    if props is None:
        return None
    for prop in props:
        if prop.name == key:
            return prop.value
    return None

def run_shell_command(cmd):
    return subprocess.check_output(cmd.split(' '), stderr = subprocess.STDOUT)

def gs_rsync(gs_path, mirror_path):
    output = run_shell_command("gsutil -m rsync %s %s" % (gs_path, mirror_path))
    return len(output.splitlines()) > 2

def extract_tarballs(mirror_path, extract_path):
    for dirpath, dirs, filenames in os.walk(mirror_path):
        for tarball in filenames:
            extract_dir = extract_path + "/" + os.path.basename(tarball)
            if os.path.isdir(extract_path + "/" + extract_dir):
                    continue
            run_shell_command("mkdir -p " + extract_dir)
            run_shell_command("tar -xf %s/%s -C %s" % (dirpath, tarball, extract_dir))

    for dirent in os.listdir(extract_path):
        if not os.path.isfile(mirror_path + "/" + dirent):
            run_shell_command("rm -rf %s/%s" % (mirror_path, dirent))

def setup_dirs():
    run_shell_command("mkdir -p %s" % mirror_dir)
    run_shell_command("mkdir -p %s" % extracted_dir)

@app.route("/favicon.ico")
def favicon_ico_handler():
    return "null"

@app.route("/sync")
def sync_handler():
    ret = gs_rsync(results_gs_path, mirror_dir)
    if ret != 0:
        extract_tarballs(mirror_dir, extracted_dir)
        return "Sync performed with gs://" + results_gs_path
    return "Already upto date"

class testresult:
    def __init__(self, report, link, dirpath, category):
        self.report = report
        self.link = link
        self.dirpath = dirpath
        self.category = category
        self.cfg = get_property(self.report.properties(), 'TESTCFG')
        self.timestamp = self.report.timestamp

    def __repr__(self):
        return "(Link = %s, dirpath = %s, category = %s, cfg = %s)" % (self.link, self.dirpath, self.category, self.cfg)

@app.route("/")
def root_handler():
    total_categories = set()
    testresults = []

    if results_gs_path == "":
        return "Results bucket not set."

    setup_dirs()
    sync_handler()

    for dirpath, dirs, filenames in os.walk(extracted_dir):
        if 'results.xml' in filenames:
            report = JUnitXml.fromfile(dirpath + '/results.xml')
            link = dirpath.split('/')[extracted_dir.count('/')]
            category = uncategorized_category
            if os.path.isfile(dirpath + "/../../../ltm-info"):
                with open(dirpath + "/../../../report") as f:
                    for line in f.readlines():
                        if line.startswith("CMDLINE") and "--watch" in line:
                            parts = line.split(' ')
                            category = ""
                            parse_state = ""
                            repo = ""
                            branch = ""
                            for part in parts:
                                if parse_state == "repo":
                                    repo = part
                                elif parse_state == "branch":
                                    branch = part
                                if part == "--repo":
                                    parse_state = "repo"
                                elif part == "--watch":
                                    parse_state = "branch"
                                else:
                                    parse_state = ""
                            category = "repo: %s, branch: %s" % (repo, branch)
            total_categories.add(category)
            testresults.append(testresult(report, link, dirpath, category))

    out = results_header()

    for category in sorted(total_categories, reverse = True):
        table = {}
        configs = set()
        for result in testresults:
            if result.category != category:
                continue
            date = result.timestamp[0:10]
            if date not in table:
                table[date] = {}
            if result.cfg not in table[date]:
                table[date][result.cfg] = []
            configs.add(result.cfg)
            table[date][result.cfg].append(result)

        out += "
%s
" % category
        out += "

Date	"
        for cfg in sorted(configs):
            out += "%s	" % cfg
        out += "
"
        last_date = ""
        print_timestamp = False

        for timestamp in sorted(table.keys(), reverse = True):
            max_items = 0
            print_timestamp = False
            if last_date != timestamp[0:10]:
                print_timestamp = True
                last_date = timestamp

            for cfg in table[timestamp].keys():
                max_items = max(max_items, len(table[timestamp][cfg]))

            for i in range(0, max_items):
                out += "
"
                if print_timestamp:
                    out += "%s	" % timestamp
                    print_timestamp = False
                else:
                    out += "	"
                for cfg in sorted(configs):
                    if cfg in table[timestamp] and i < len(table[timestamp][cfg]):
                        summary = result_summary(table[timestamp][cfg][i].report)
                        out += "%s	" % (summary[0], table[timestamp][cfg][i].link, summary[1])
                    else:
                        out += "	"
                out += "
"
        out += "
"

    return out

@app.route("/files/<path:path>")
def file_browser_handler(path):
    out = results_header()
    out += "
"
    if path != "/":
        parts = path.split('/')
        print(str(parts))
        out += "..\n" % ('/'.join(parts[:(len(parts) - 1)]))
    if os.path.isfile(extracted_dir + path):
        out += "
"
        with open(extracted_dir + path, "r") as f:
            out += f.read()
    else:
        for dirent in os.listdir(extracted_dir + path):
            if os.path.isdir(extracted_dir + path + "/" + dirent):
                out += "%s/
" % (path, dirent, dirent)


""" Disk namespace commands. """

import click
from click.exceptions import UsageError

from gandi.cli.core.cli import cli
from gandi.cli.core.utils import output_disk, output_generic, randomstring
from gandi.cli.core.utils.size import disk_check_size
from gandi.cli.core.params import (pass_gandi, DATACENTER, SNAPSHOTPROFILE_VM,
                                   KERNEL, SIZE, option, DISK_IMAGE)

@cli.command()
@click.option('--only-data', help='Only display data disks.', is_flag=True)
@click.option('--only-snapshot', help='Only display snapshots.', is_flag=True)
@click.option('--type', help='Display types.', is_flag=True)
@click.option('--id', help='Display ids.', is_flag=True)
@click.option('--vm', help='Display vms.', is_flag=True)
@click.option('--snapshotprofile', help='Display snapshot profile.',
              is_flag=True)
@click.option('--limit', help='Limit number of results.', default=100,
              show_default=True)
@pass_gandi
def list(gandi, only_data, only_snapshot, type, id, vm, snapshotprofile,
         limit):
    """ List disks. """
    options = {
        'items_per_page': limit,
    }

    if only_data:
        options.setdefault('type', []).append('data')
    if only_snapshot:
        options.setdefault('type', []).append('snapshot')

    output_keys = ['name', 'state', 'size']
    if type:
        output_keys.append('type')
    if id:
        output_keys.append('id')
    if vm:
        output_keys.append('vm')

    profiles = []
    if snapshotprofile:
        output_keys.append('profile')
        profiles = gandi.snapshotprofile.list()

    result = gandi.disk.list(options)
    vms = dict([(vm_['id'], vm_) for vm_ in gandi.iaas.list()])

    for num, disk in enumerate(result):
        if num:
            gandi.separator_line()
        output_disk(gandi, disk, [], vms, profiles, output_keys)

    return result

@cli.command()
@click.argument('resource', nargs=-1, required=True)
@pass_gandi
def info(gandi, resource):
    """ Display information about a disk.

    Resource can be a disk name or ID
    """
    output_keys = ['name', 'state', 'size', 'type', 'id', 'dc', 'vm',
                   'profile', 'kernel']

    resource = sorted(tuple(set(resource)))
    vms = dict([(vm['id'], vm) for vm in gandi.iaas.list()])
    datacenters = gandi.datacenter.list()

    result = []
    for num, item in enumerate(resource):
        if num:
            gandi.separator_line()
        disk = gandi.disk.info(item)
        output_disk(gandi, disk, datacenters, vms, [], output_keys)
        result.append(disk)

    return result

@cli.command()
@click.option('--bg', '--background', default=False, is_flag=True,
              help='Run command in background mode (default=False).')
@click.option('--force', '-f', is_flag=True,
              help='This is a dangerous option that will cause CLI to continue'
                   ' without prompting. (default=False).')
@pass_gandi
@click.argument('resource', nargs=-1, required=True)
def detach(gandi, resource, background, force):
    """ Detach disks from currectly attached vm.

    Resource can be a disk name, or ID
    """
    resource = sorted(tuple(set(resource)))
    if not force:
        proceed = click.confirm('Are you sure you want to detach %s?' %
                                ', '.join(resource))
        if not proceed:
            return

    result = gandi.disk.detach(resource, background)
    if background:
        gandi.pretty_echo(result)

    return result

@cli.command()
@click.option('--bg', '--background', default=False, is_flag=True,
              help='Run command in background mode (default=False).')
@click.option('-r', '--read-only', default=False, is_flag=True,
              help='Attach disk as read-only')
@click.option('--position', '-p', type=click.INT,
              help='Position where disk should be attached: 0 for system disk.'
                   ' If there is already a disk attached at the specified'
                   ' position, it will be swapped.')
@click.option('--force', '-f', is_flag=True,
              help='This is a dangerous option that will cause CLI to continue'
                   ' without prompting. (default=False).')
@pass_gandi
@click.argument('disk', nargs=1, required=True)
@click.argument('vm', nargs=1, required=True)
def attach(gandi, disk, vm, position, read_only, background, force):
    """ Attach disk to vm.

    disk can be a disk name, or ID
    vm can be a vm name, or ID
    """
    if not force:
        proceed = click.confirm("Are you sure you want to attach disk '%s'"
                                " to vm '%s'?" % (disk, vm))
        if not proceed:
            return

    disk_info = gandi.disk.info(disk)
    attached = disk_info.get('vms_id', False)
    if attached and not force:
        gandi.echo('This disk is still attached')
        proceed = click.confirm('Are you sure you want to detach %s?' % disk)

        if not proceed:
            return

    result = gandi.disk.attach(disk, vm, background, position, read_only)
    if background and result:
        gandi.pretty_echo(result)

    return result

@cli.command()
@click.option('--cmdline', type=click.STRING, default=None,
              help='Kernel cmdline.')
@click.option('--kernel', type=KERNEL, default=None, help='Kernel for disk.')
@click.option('--name', type=click.STRING, default=None, help='Disk name.')
@click.option('--size', default=None, metavar='SIZE[M|G|T]', type=SIZE,
              help=('Disk size. A size suffix (M for megabytes up to T for '
                    'terabytes) is optional, megabytes is the default if no '
                    'suffix is present.'),
              callback=disk_check_size)
@click.option('--snapshotprofile', help='Selected snapshot profile.',
              default=None, type=SNAPSHOTPROFILE_VM)
@click.option('--delete-snapshotprofile', default=False, is_flag=True,
              help='Remove snapshot profile associated to this disk.')
@click.option('--bg', '--background', default=False, is_flag=True,
              help='Run command in background mode (default=False).')
@pass_gandi
@click.argument('resource')
def update(gandi, resource, cmdline, kernel, name, size,
           snapshotprofile, delete_snapshotprofile, background):
    """ Update a disk.

    Resource can be a disk name, or ID
    """
    if snapshotprofile and delete_snapshotprofile:
        raise UsageError('You must not set snapshotprofile and '
                         'delete-snapshotprofile.')

    if delete_snapshotprofile:
        snapshotprofile = ''

    result = gandi.disk.update(resource, name, size, snapshotprofile,
                               background, cmdline, kernel)
    if background:
        gandi.pretty_echo(result)

    return result

@cli.command()
@click.option('--force', '-f', is_flag=True,
              help='This is a dangerous option that will cause CLI to continue'
                   ' without prompting. (default=False).')
@click.option('--bg', '--background', default=False, is_flag=True,
              help='run command in background mode (default=False).')
@click.argument('resource', nargs=-1, required=True)
@pass_gandi
def delete(gandi, resource, force, background):
    """ Delete a disk. """
    output_keys = ['id', 'type', 'step']

    resource = sorted(tuple(set(resource)))
    if not force:
        disk_info = "'%s'" % ', '.join(resource)
        proceed = click.confirm('Are you sure you want to delete disk %s?'
                                % disk_info)

        if not proceed:
            return

    opers = gandi.disk.delete(resource, background)
    if background:
        for oper in opers:
            output_generic(gandi, oper, output_keys)

    return opers

@cli.command()
@click.option('--name', type=click.STRING, default=None,
              help='Disk name, will be generated if not provided.')
@click.option('--vm', default=None, type=click.STRING,
              help='Attach the newly created disk to the vm.')
@click.option('--size', default='3072', metavar='SIZE[M|G|T]', type=SIZE,
              help=('Disk size. A size suffix (M for megabytes up to T for '
                    'terabytes) is optional, megabytes is the default if no '
                    'suffix is present.'),
              callback=disk_check_size)
@click.option('--snapshotprofile', help='Selected snapshot profile.',
              default=None, type=SNAPSHOTPROFILE_VM)
@click.option('--source', default=None, type=DISK_IMAGE,
              help='Create a disk from a disk or a snapshot.')
@option('--datacenter', type=DATACENTER, default='LU-BI1',
        help='Datacenter where the disk will be created.')
@click.option('--bg', '--background', default=False, is_flag=True,
              help='Run command in background mode (default=False).')
@pass_gandi
def create(gandi, name, vm, size, snapshotprofile, datacenter, source,
           background):
    """ Create a new disk. """
    output_keys = ['id', 'type', 'step']
    name = name or randomstring('vdi')

    disk_type = 'data'
    oper = gandi.disk.create(name, vm, size, snapshotprofile, datacenter,
                             source, disk_type, background)

    if background:
        output_generic(gandi, oper, output_keys)

    return oper

@cli.command()
@click.option('--name', type=click.STRING, default=None,
              help='Snapshot name, will be generated if not provided.')
@click.option('--bg', '--background', default=False, is_flag=True,
              help='Run command in background mode (default=False).')
@click.argument('resource')
@pass_gandi
def snapshot(gandi, name, resource, background):
    """ Create a snapshot on the fly. """
    name = name or randomstring('snp')

    source_info = gandi.disk.info(resource)


#!/usr/bin/env python

import os
import sys
import time
import subprocess
import signal
from multiprocessing import Process, Value

import unittest
from emilib import *

class TestEmiLib(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestEmiLib, self).__init__(*args, **kwargs)
        self.emi_core = None

    def setUp(self):
        self.emi_core = subprocess.Popen("emi_core",
                                         stdout = subprocess.PIPE, bufsize = 0)
        time.sleep(.5)

    def tearDown(self):
        self.emi_core.terminate()
        time.sleep(.5)

    def test_emi_msg(self):
        msg_data = emi_msg(
            data=b'1234'*1024, cmd=1, msgnum=2, flag=3, ipaddr="10.0.0.3")
        self.assertEqual(msg_data.data, b'1234'*1024)
        self.assertEqual(msg_data.msg, 2)
        self.assertEqual(msg_data.cmd, 1)
        self.assertEqual(msg_data.size, len(b'1234'*1024))
        self.assertEqual(str(msg_data.addr.ipv4), "10.0.0.3:1361")
        self.assertEqual(msg_data.is_block(), False)

    def test_emi_init(self):
        self.assertEqual(emi_init(), 0)

    def test_emi_msg_register(self):
        ret = emi_init()
        self.assertEqual(ret, 0)

        def func(msg):
            pass

        ret = emi_msg_register(1, func)
        self.assertEqual(ret, 0)

        self.assertRaises(EMIError, emi_msg_register, 1, func)

    def test_emi_msg_send_unblock_nosenddata(self):
        received = Value('i', 0)

        def recvprocess_unblock():
            emi_init()

            def func(msg):
                nonlocal received
                self.assertEqual(msg.msg, 2)
                self.assertEqual(msg.cmd, 1)

                with received.get_lock():
                    received.value += 1

            ret = emi_msg_register(2, func)
            self.assertEqual(ret, 0)

            signal.pause()
            time.sleep(2);

        p1 = Process(target = recvprocess_unblock)
        p2 = Process(target = recvprocess_unblock)

        p1.start()
        p2.start()

        time.sleep(1)

        msg = emi_msg(msgnum=2, cmd=1, ipaddr="127.0.0.1")
        ret = emi_msg_send(msg)
        self.assertEqual(ret, 0)

        p1.join()
        p2.join()

        self.assertEqual(received.value, 2)

    def test_emi_msg_send_unblock_senddata(self):
        received= Value('i', 0)

        def recvprocess_unblock_data():
            emi_init()

            def func(msg):
                nonlocal received
                self.assertEqual(msg.msg, 3)
                self.assertEqual(msg.cmd, 1)
                self.assertEqual(msg.data, b"11112222"*1024)

                with received.get_lock():
                    received.value += 1

            ret = emi_msg_register(3, func)
            self.assertEqual(ret, 0)

            signal.pause()
            time.sleep(2);

        p1 = Process(target = recvprocess_unblock_data)
        p2 = Process(target = recvprocess_unblock_data)

        p1.start()
        p2.start()

        time.sleep(1)

        msg = emi_msg(msgnum=3, cmd=1, ipaddr="127.0.0.1", data=b'11112222'*1024)
        ret = emi_msg_send(msg)
        self.assertEqual(ret, 0)

        p1.join()
        p2.join()

        self.assertEqual(received.value, 2)

    def test_emi_msg_send_block_noretdata(self):
        received= Value('i', 0)

        def recvprocess_block():
            emi_init()

            def func(msg):
                nonlocal received
                self.assertEqual(msg.msg, 4)
                self.assertEqual(msg.cmd, 1)
                self.assertEqual(msg.data, b"11112222"*1024)

                with received.get_lock():
                    received.value += 1

                return 0

            ret = emi_msg_register(4, func)
            self.assertEqual(ret, 0)

            signal.pause()
            time.sleep(2);

        p1 = Process(target = recvprocess_block)
        p2 = Process(target = recvprocess_block)

        p1.start()
        p2.start()

        time.sleep(1)

        msg = emi_msg(
            msgnum=4,
            cmd=1,
            ipaddr="127.0.0.1",
            data=b'11112222'*1024,
            flag=emi_flag.EMI_MSG_MODE_BLOCK)
        ret = emi_msg_send(msg)
        self.assertEqual(ret, 0)

        p1.join()
        p2.join()

        self.assertEqual(received.value, 2)

        def recvprocess_block_fail():
            emi_init()

            def func(msg):
                nonlocal received
                self.assertEqual(msg.msg, 4)
                self.assertEqual(msg.cmd, 1)
                self.assertEqual(msg.data, b"11112222"*1024)

                with received.get_lock():
                    received.value += 1

                return -1

            ret = emi_msg_register(4, func)
            self.assertEqual(ret, 0)

            signal.pause()
            time.sleep(2);

        p1 = Process(target = recvprocess_block)
        p2 = Process(target = recvprocess_block)
        p3 = Process(target = recvprocess_block_fail)

        p1.start()
        p2.start()
        p3.start()

        time.sleep(1)

        msg = emi_msg(
            msgnum=4,
            cmd=1,
            ipaddr="127.0.0.1",
            data=b'11112222'*1024,
            flag=emi_flag.EMI_MSG_MODE_BLOCK)
        ret = emi_msg_send(msg)
        self.assertEqual(ret, -1)

        p1.join()
        p2.join()
        p3.join()

        self.assertEqual(received.value, 5)

    def test_emi_msg_send_block_retdata(self):
        received= Value('i', 0)

        def recvprocess_block_data():
            emi_init()

            def func(msg):
                nonlocal received
                self.assertEqual(msg.msg, 5)
                self.assertEqual(msg.cmd, 1)
                self.assertEqual(msg.data, b"11112222"*512)

                with received.get_lock():
                    received.value += 1

                return emi_load_retdata(msg, b'abcdef'*1024)

            ret = emi_msg_register(5, func)
            self.assertEqual(ret, 0)

            signal.pause()
            time.sleep(2);

        p1 = Process(target = recvprocess_block_data)


##############################################################################################
# Copyright 2014-2015 Cloud Media Sdn. Bhd.
#
# This file is part of Xuan Automation Application.
#
# Xuan Automation Application is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Xuan Automation Application is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Xuan Automation Application.  If not, see <http://www.gnu.org/licenses/>.
##############################################################################################

from collections import deque
from concurrent.futures.thread import ThreadPoolExecutor
import copy
import json
import threading
import time

from automationApp.appConstants import AppConstants
from automationApp.automationException import AutomationException
from automationApp.core.eventController import EventController
from automationApp.core.groupController import GroupController
from automationApp.core.methodController import MethodController
from automationApp.core.ruleController import RuleController
from automationApp.core.triggerController import TriggerController
from automationApp.module.timerModule import TimerModule
from automationApp.utils.automationJSONEncoder import AutomationJSONEncoder
from automationApp.utils.methodCallGroup import MethodCallGroup
from automationApp.utils.sharedMethodWrapper import SharedMethodWrapper
from com.cloudMedia.theKuroBox.sdk.app.appinfo import AppInfo
from com.cloudMedia.theKuroBox.sdk.app.application import Application
from com.cloudMedia.theKuroBox.sdk.app.sharedMethod import SharedMethod
from com.cloudMedia.theKuroBox.sdk.util.logger import Logger
from com.cloudMedia.theKuroBox.sdk.util.util import Util
from com.cloudMedia.theKuroBox.sdk.util.validator.numberValidator import NumberValidator
from com.cloudMedia.theKuroBox.sdk.util.validator.stringValidator import StringValidator

class RuleExecResult(threading.Event):
    '''
    Rule methods execution result tracker.
    '''

    def __init__(self, totalResultRequested):
        super().__init__()
        self.__result = None
        self.__totalResultRequested = totalResultRequested
        self.__totalResultResponsed = 0
        self.__lock = threading.Lock()

    def set_result(self, result):
        with self.__lock:
            if self.__result is None:
                self.__result = not isinstance(result, Exception)
            else:
                self.__result = self.__result and not isinstance(result, Exception)

            self.__totalResultResponsed += 1
            if self.__totalResultResponsed == self.__totalResultRequested:
                self.set()

    def get_result(self):
        return self.__result is True

class RuleExecInfo:
    '''
    Keep tracks the total count of a rule being requested to trigger.
    '''

    def __init__(self):
        self.__lock = threading.Lock()
        self.__triggerCount = 0

    def get_rlock(self):
        return self.__lock

    def get_trigger_count(self):
        return self.__triggerCount

    def increase_trigger_count(self):
        self.__triggerCount = (self.__triggerCount + 1) % 100000

class RuleService:

    def __init__(self):
        # Rule processors.
        self.__ruleController = RuleController()
        self.__methodController = MethodController()
        self.__triggerController = TriggerController.instance()

        self.__ruleUpdateThreadPool = ThreadPoolExecutor(max_workers=1)
        self.__ruleExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_RULE_EXEC_THREAD_SIZE)

        # Rule run workers.
        self.__ruleExecInfos = {}
        self.__condCallGroup = MethodCallGroup()
        self.__execCallGroup = MethodCallGroup()

        # Listeners.
        self.__ruleController.listen_to_rule_status_change(self.__on_rule_status_changed)
        GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed)
        self.__methodController.listen_to_method_status_change(self.__on_method_status_changed)
        EventController.instance().listen_to_event_callback(self.__on_method_event_callback)
        self.__triggerController.listen_to_trigger_callback(self.__on_trigger_callback)

    def __on_rule_status_changed(self, ruleId, oldEnabled, newEnabled, oldStatusProcessed, newStatusProcessed):
        '''
        Trigger Source: RuleController --> This
        Callback when a rule is re-enabled OR statusProcessed changed to "updated".
        '''
        if newEnabled == True and newStatusProcessed == AppConstants.RULE_STATUS_UPDATED:
            if oldEnabled != newEnabled or oldStatusProcessed != newStatusProcessed:
                self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)

    def __on_group_icon_changed(self, kbxGroupId):
        '''
        Trigger Source: GroupController --> This
        Callback when kbxGroupIcon changed.
        '''
        ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_condition(kbxGroupId)
        ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_execution(kbxGroupId)

        # Broadcast rules updated messages.
        for ruleId in set(ruleIdsFromCond + ruleIdsFromExec):
            self.__broadcast_message__rule_updated(ruleId)

    def __on_method_status_changed(self, kbxMethodId, oldKBXMethodStatus, newKBXMethodStatus):
        '''
        Trigger Source: MethodController --> This
        Callback when kbxMethodStatus changed.
        '''
        if oldKBXMethodStatus != newKBXMethodStatus:
            ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId)
            ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_execution(kbxMethodId)

#             # Executes rules with conditions affected.
#             if newKBXMethodStatus == SharedMethod.METHOD_STATUS_ACTIVE:
#                 for ruleId in ruleIdsFromCond:
#                     self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)

            # Broadcast rules updated messages.
            for ruleId in set(ruleIdsFromCond + ruleIdsFromExec):
                self.__broadcast_message__rule_updated(ruleId)

    def __on_method_event_callback(self, kbxMethodId, eventTag, eventData):
        '''
        Trigger Source: EventController --> MethodController --> This
        Callback when a method with event broadcasted event.
        '''
        ruleIds = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId)
        for ruleId in ruleIds:
            self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId,
                                             checkCondition=True, eventTag=eventTag,
                                             eventData=eventData, eventMethodId=kbxMethodId)

    def __on_trigger_callback(self, ruleId):
        '''
        Trigger Source: TriggerController --> This
        Callback when a rule is triggered.
        '''
        self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)

    def set_rule(self, trigger, condition, execution, ruleId=None, ruleName=None, ruleProtected=False, enabled=True):
        '''
        Create/Edit(with ruleId provided) an existing rule.

        trigger:Dictionary
        condition:List
        execution:List
        ruleId:Integer <Optional>
        ruleName:String <Optional>
        ruleProtected:Boolean <Optional>
        enabled:Boolean

        Returns "ruleId"
        '''
        def process_method_list(methodList):
            #===================================================================
            # Basic type validation
            #===================================================================
            if not isinstance(methodList, list):
                Logger.log_error("RuleService.set_rule: 'condition' and 'execution' must be type of list.")
                Logger.log_debug("type:", type(methodList), "value:", methodList)
                raise AutomationException(11704, "List is required for both 'condition' and 'execution'")

            #===================================================================
            # Check allowed size, raise error if exceeded.
            #===================================================================
            methodListLen = len(methodList)
            if methodListLen > AppConstants.MAX_METHOD_SIZE:
                Logger.log_error("RuleService.set_rule: 'condition' and 'execution' cannot have more than", AppConstants.MAX_METHOD_SIZE, "items respectively.")
                raise AutomationException(11705, "Only a maximum of " + \
                                          str(AppConstants.MAX_METHOD_SIZE) + \
                                          " items is allowed for each 'condition' and 'execution' - given size " + \
                                          str(methodListLen),
                                          lambda text: str(AppConstants.MAX_METHOD_SIZE).join(text.split(":max_item_size:")))

            #===================================================================
            # Check if all kbxMethodIds are valid and all kbxMethodParams are list
            #===================================================================
            idValidator = NumberValidator(isRequired=True, decimalPoint=False)
            if not all([idValidator.is_valid(eachMethod["kbxMethodId"])
                        and isinstance(eachMethod["kbxMethodParams"], list)
                        for eachMethod in methodList]):
                raise AutomationException(11704, "'condition' and 'execution' have incorrect data structure.")

            #===================================================================
            # Check if all kbxParamName and kbxParamCurrentValue exists
            #===================================================================
            paramNameValidator = StringValidator(isRequired=True)
            for eachMethod in methodList:
                methodArgs = eachMethod["kbxMethodParams"]
                for methodArg in methodArgs:
                    if not paramNameValidator.is_valid(methodArg[AppConstants.ARG_NAME]):
                        raise AutomationException(11704, "'condition' and 'execution' have invalid params structure")

                    if not AppConstants.ARG_CURRENT_VALUE in methodArg:
                        methodArg[AppConstants.ARG_CURRENT_VALUE] = None

            return methodList

        #=======================================================================
        # Data structure validations
        #=======================================================================
        ruleId = NumberValidator(isRequired=False, decimalPoint=False).get_value(ruleId)
        triggerDTO = self.__triggerController.parse_to_trigger_dto(trigger)
        condition = process_method_list(condition)
        execution = process_method_list(execution)

        #=======================================================================
        # Add to database
        #=======================================================================
        if Util.is_empty(ruleId):
            # Validate_max_rule_size
            if self.__ruleController.count() >= AppConstants.MAX_RULE_SIZE:
                raise AutomationException(11706,
                                          "Total amount of rules cannot be more than " + str(AppConstants.MAX_RULE_SIZE),
                                          lambda text: str(AppConstants.MAX_RULE_SIZE).join(text.split(":max_rule_size:")))
            ruleId = self.__ruleController.generate_id(ruleName)
            rule = {}
        elif self.__ruleController.has(ruleId):
            ruleFromDB = self.__ruleController.get(ruleId)
            rule = dict(ruleFromDB)
            self.__check_rule_process_status(ruleId)
            self.__ruleController.change_to_updating(ruleId, ruleName)
        else:
            raise AutomationException(11704, "Rule ID provided not found - " + str(ruleId))

        #=======================================================================
        # Broadcast message: starts to update rule.
        #=======================================================================
        self.__broadcast_message__rule_update_started(ruleId, ruleName)

        #=======================================================================
        # Set basic information of the rule
        #=======================================================================
        rule["ruleId"] = ruleId
        rule["ruleName"] = ruleName
        rule["ruleProtected"] = ruleProtected
        rule["trigger"] = triggerDTO
        rule["enabled"] = enabled

        rule["condition"] = condition
        rule["execution"] = execution

        #=======================================================================
        # Update rule
        #=======================================================================
        def __update_rule(rule):
            try:
                # Fire rule update start event
                ruleId = rule["ruleId"]

                # Add methods to subscribe list
                methodIds = [kbxMethod["kbxMethodId"] for kbxMethod in rule["condition"] + rule["execution"]]
                self.__methodController.add(methodIds)

                # Update "rule" base table
                self.__ruleController.update(rule)
                self.__ruleController.commit()

            except Exception as e:
                self.__ruleController.rollback()
                self.__broadcast_message__rule_update_failed(ruleId, ruleName)
                Logger.log_error("RuleService __update_rule failed:", e, "-- rolledback")
            else:
                self.__triggerController.register_listener(ruleId, rule["trigger"])

                # Process for Timer Module
                TimerModule.delete_scheduler(ruleId)

                timerModuleHandlers = {TimerModule.METHOD_ID_DATE_TIME_RANGE:TimerModule.handle_date_time_range,
                                       TimerModule.METHOD_ID_DAY_OF_WEEK:TimerModule.handle_dow,
                                       TimerModule.METHOD_ID_TIME_RANGE:TimerModule.handle_time_range}

                for kbxMethod in rule["condition"]:
                    kbxMethodId = kbxMethod["kbxMethodId"]
                    timerModuleHandler = timerModuleHandlers.get(kbxMethodId, None)
                    if timerModuleHandler is not None:
                        timerModuleHandler(ruleId, kbxMethod["kbxMethodParams"])

                # Broadcast message: completed updating a rule
                self.__broadcast_message__rule_updated(ruleId)

        #=======================================================================
        # Submit to a thread to process other info, and return... performance...
        #=======================================================================
        self.__ruleUpdateThreadPool.submit(__update_rule, rule)

    def delete_rule(self, ruleId):
        self.__check_rule_process_status(ruleId)
        try:
            self.__ruleController.delete(ruleId)
            self.__ruleController.commit()
        except Exception as e:
            self.__ruleController.rollback()
            Logger.log_error("RuleService delete_rule ex:", e, "-- rolled back")
        else:
            self.__broadcast_message__rule_deleted(ruleId)
            self.__triggerController.unregister_listener(ruleId)
            TimerModule.delete_scheduler(ruleId)

    def trigger_rule(self, ruleId, checkCondition=False):
        '''
        self.__check_rule_process_status(ruleId) <-- Check again in self.__trigger_rule_implementation.
        '''
        self.__trigger_rule_implementation(ruleId=ruleId, checkCondition=checkCondition)

    def enable_rule(self, ruleId, enabled):
        self.__check_rule_process_status(ruleId)
        try:
            self.__ruleController.enable(ruleId, enabled)
            self.__ruleController.commit()
        except Exception as e:
            self.__ruleController.rollback()
            Logger.log_error("RuleService enable_rule ex:", e, "-- rolled back")
        else:
            self.__broadcast_message__rule_updated(ruleId)

    def get_rule(self, ruleId, language=AppInfo.DEFAULT_API_LANGUAGE):
        try:
            rule = self.__ruleController.get_detail(ruleId)
        except:
            raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId))

        kbxMethods = list(rule["condition"]) + list(rule["execution"])

        # -------------- Compile lists of kbxMethod and group IDs contains in this rule.
        kbxMethodIdsToList = {}
        kbxGroupIdsToList = set([])

        for kbxMethod in kbxMethods:
            # Variables
            kbxMethodId = kbxMethod["kbxMethodId"]
            kbxMethodAppId = kbxMethod["kbxMethodAppId"]
            kbxMethodStatus = kbxMethod["kbxMethodStatus"]
            kbxGroupId = kbxMethod["kbxGroupId"]
            kbxGroupStatus = kbxMethod["kbxGroupStatus"]

            if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
                kbxMethodIdsToList.setdefault(kbxMethodAppId, set([]))
                kbxMethodIdsToList[kbxMethodAppId].add(kbxMethodId)
            if kbxGroupId is not None and kbxGroupStatus is not -1:
                kbxGroupIdsToList.add(kbxGroupId)

        # -------------- Get methods and groups based on requested language.
        kbxMethodIdsListed = {}
        kbxGroupIdsListed = {}

        for kbxMethodAppId, kbxMethodIds in kbxMethodIdsToList.items():
            kbxMethodIdsListed[kbxMethodAppId] = SharedMethodWrapper.list_shared_methods_by_app_id(kbxMethodAppId,
                                                                                list(kbxMethodIds),
                                                                                language=language)

        groupList = SharedMethodWrapper.list_shared_method_groups(kbxGroupId=kbxGroupIdsToList, language=language)
        for row in groupList:
            kbxGroupIdsListed[row["kbxGroupId"]] = row

        # -------------- Set method and group data into rule.
        for kbxMethod in kbxMethods:
            # Variables
            kbxMethodId = kbxMethod["kbxMethodId"]
            kbxMethodAppId = kbxMethod["kbxMethodAppId"]
            kbxMethodStatus = kbxMethod["kbxMethodStatus"]
            kbxGroupId = kbxMethod["kbxGroupId"]
            kbxGroupStatus = kbxMethod["kbxGroupStatus"]

            if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
                kbxMethodParamsWithCurrentValue = {kbxMethodParam["kbxParamName"]:kbxMethodParam["kbxParamCurrentValue"] \
                                                   for kbxMethodParam in kbxMethod["kbxMethodParams"]}
                kbxMethodWithDetails = kbxMethodIdsListed[kbxMethodAppId][kbxMethodId]
                if kbxMethodWithDetails is not None:
                    kbxMethodParamsWithDetails = kbxMethodWithDetails["kbxMethodParams"]
                    kbxMethodParamsWithDetails = copy.deepcopy(kbxMethodParamsWithDetails)

                    for kbxMethodParam in kbxMethodParamsWithDetails:
                        kbxMethodParam["kbxParamCurrentValue"] = kbxMethodParamsWithCurrentValue.get(kbxMethodParam["kbxParamName"], None)

                    kbxMethod["kbxMethodParams"] = kbxMethodParamsWithDetails
                    kbxMethod["kbxMethodHasEvent"] = not Util.is_empty(kbxMethodWithDetails.get("kbxMethodEvent", None)) \
                                                        and not Util.is_empty(kbxMethodWithDetails.get("kbxMethodIdentifier", None))
                    kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel")
                    kbxMethod["kbxMethodDesc"] = kbxMethodWithDetails.get("kbxMethodDesc")

                else:
                    kbxMethod["atDebugMethod"] = "Unable to get shared method, caused by a method which never register itself on this bootup."

            else:
                kbxMethod["kbxMethodHasEvent"] = False

            if kbxGroupId is not None and kbxGroupStatus is not -1:
                try:
                    kbxMethod["kbxGroupLabel"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupLabel"]
                    kbxMethod["kbxGroupDesc"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupDesc"]
                except:
                    kbxMethod["atDebugGroup"] = "Unable to get shared method group, caused by a group which never register itself on this bootup."

        return rule

    def list_rules(self, offset=0, limit=20):
        return self.__ruleController.list(offset, limit), \
                self.__ruleController.count()

    def run_all_enabled_rules(self):
        '''
        Add the following 2 lines of code at AutomationModuleWrapper.py - start(),
        after last statement, to enable run all rules on bootup.
        # Logger.log_info("Attempts to execute all enabled rules ...")
        # self.__ruleService.run_all_enabled_rules()
        '''
        ruleIds = self.__ruleController.list_rule_ids_which_are_enabled()
        for ruleId in ruleIds:
            self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)

    def __check_rule_process_status(self, ruleId):
        try:
            statusProcessed = self.__ruleController.get_status_processed(ruleId)
            if statusProcessed != AppConstants.RULE_STATUS_UPDATED:
                raise AutomationException(11703, "edit/delete/execute is not allowed on rule update in progress")
        except:
            raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId))

    def __broadcast_message__rule_update_started(self, ruleId, ruleName=None):
        eventTag = AppConstants.EVENT_RULE_UPDATE_STARTED
        eventData = {"ruleId":ruleId, "newRuleName":ruleName}

        self.__broadcast_message(eventTag, eventData)
        Logger.log_info("Rule Start Update:", ruleName)

    def __broadcast_message__rule_updated(self, ruleId):
        try:
            rule = self.__ruleController.get_summary(ruleId)
        except Exception as e:
            Logger.log_error("RuleService.__broadcast_message__rule_updated get_summary ex:", e)
            return

        eventTag = AppConstants.EVENT_RULE_UPDATED
        eventData = rule

        self.__broadcast_message(eventTag, eventData)
        Logger.log_info("Rule Updated:", rule["ruleName"])

    def __broadcast_message__rule_update_failed(self, ruleId, ruleName=None):
        '''
        ruleName - For debugging purpose.
        '''
        try:
            rule = self.__ruleController.get_summary(ruleId)
        except Exception:
            rule = None

        eventTag = AppConstants.EVENT_RULE_UPDATE_FAILED
        eventData = {"ruleId": ruleId, "oldRuleSummary":rule}

        self.__broadcast_message(eventTag, eventData)
        Logger.log_info("Rule Update Failed:", ruleName)

    def __broadcast_message__rule_deleted(self, ruleId):
        eventTag = AppConstants.EVENT_RULE_DELETED
        eventData = {"ruleId": ruleId}

        self.__broadcast_message(eventTag, eventData)
        Logger.log_info("Rule Deleted: Id -", ruleId)

    def __broadcast_message(self, eventTag, eventData):
        eventData = json.dumps(eventData, cls=AutomationJSONEncoder)
        Application.send_web_server_event(eventTag, eventData)

    def __trigger_rule_implementation(self, ruleId, checkCondition=False, eventTag=None, eventData=None, eventMethodId=None):
        '''
        Triggers a rule by given ruleId.
        '''
        Logger.log_info("trigger rule id:", ruleId)

        # Check if rule is "updated" AND enabled.
        statusProcessed, enabled = self.__ruleController.get_status_processed_and_enabled(ruleId)
        if statusProcessed != AppConstants.RULE_STATUS_UPDATED or enabled != True:
            return

        self.__ruleExecInfos.setdefault(ruleId, RuleExecInfo())

        ruleExecInfo = self.__ruleExecInfos.get(ruleId)
        ruleExecInfo.increase_trigger_count()
        triggerCountInThisSession = ruleExecInfo.get_trigger_count()

        with ruleExecInfo.get_rlock():
            #=======================================================================
            # Check conditions
            #=======================================================================
            if checkCondition is True:
                # Check if we should proceed (stop if there is another pending request on the same ruleId).
                if triggerCountInThisSession != ruleExecInfo.get_trigger_count():
                    return

                methodListToCheck = deque()
                result = self.__ruleController.list_conditions(ruleId)
                methodCheckingTime = int(time.time())
                for row in result:
                    if row["kbxMethodStatus"] not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE):
                        return
                    else:
                        methodArgs = row["kbxMethodParams"]

                        kwargs = {methodArg[AppConstants.ARG_NAME]:methodArg[AppConstants.ARG_CURRENT_VALUE] for methodArg in methodArgs}

                        if eventTag is not None and eventMethodId == row["kbxMethodId"]:
                            kwargs[AppConstants.KEY_CONDITION_EVENT_TAG] = eventTag
                            kwargs[AppConstants.KEY_CONDITION_EVENT_DATA] = eventData

                        if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs:
                            kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE


"""
CherryPy implements a simple caching system as a pluggable Tool. This tool
tries to be an (in-process) HTTP/1.1-compliant cache. It's not quite there
yet, but it's probably good enough for most sites.

In general, GET responses are cached (along with selecting headers) and, if
another request arrives for the same resource, the caching Tool will return 304
Not Modified if possible, or serve the cached response otherwise. It also sets
request.cached to True if serving a cached representation, and sets
request.cacheable to False (so it doesn't get cached again).

If POST, PUT, or DELETE requests are made for a cached resource, they
invalidate (delete) any cached response.

Usage
=====

Configuration file example::

    [/]
    tools.caching.on = True
    tools.caching.delay = 3600

You may use a class other than the default
:class:`MemoryCache<cherrypy.lib.caching.MemoryCache>` by supplying the config
entry ``cache_class``; supply the full dotted name of the replacement class
as the config value. It must implement the basic methods ``get``, ``put``,
``delete``, and ``clear``.

You may set any attribute, including overriding methods, on the cache
instance by providing them in config. The above sets the
:attr:`delay<cherrypy.lib.caching.MemoryCache.delay>` attribute, for example.
"""

import datetime
import sys
import threading
import time

import six

import cherrypy
from cherrypy.lib import cptools, httputil
from cherrypy._cpcompat import Event

class Cache(object):

    """Base class for Cache implementations."""

    def get(self):
        """Return the current variant if in the cache, else None."""
        raise NotImplemented

    def put(self, obj, size):
        """Store the current variant in the cache."""
        raise NotImplemented

    def delete(self):
        """Remove ALL cached variants of the current resource."""
        raise NotImplemented

    def clear(self):
        """Reset the cache to its initial, empty state."""
        raise NotImplemented

# ------------------------------ Memory Cache ------------------------------- #
class AntiStampedeCache(dict):

    """A storage system for cached items which reduces stampede collisions."""

    def wait(self, key, timeout=5, debug=False):
        """Return the cached value for the given key, or None.

        If timeout is not None, and the value is already
        being calculated by another thread, wait until the given timeout has
        elapsed. If the value is available before the timeout expires, it is
        returned. If not, None is returned, and a sentinel placed in the cache
        to signal other threads to wait.

        If timeout is None, no waiting is performed nor sentinels used.
        """
        value = self.get(key)
        if isinstance(value, Event):
            if timeout is None:
                # Ignore the other thread and recalc it ourselves.
                if debug:
                    cherrypy.log('No timeout', 'TOOLS.CACHING')
                return None

            # Wait until it's done or times out.
            if debug:
                cherrypy.log('Waiting up to %s seconds' %
                             timeout, 'TOOLS.CACHING')
            value.wait(timeout)
            if value.result is not None:
                # The other thread finished its calculation. Use it.
                if debug:
                    cherrypy.log('Result!', 'TOOLS.CACHING')
                return value.result
            # Timed out. Stick an Event in the slot so other threads wait
            # on this one to finish calculating the value.
            if debug:
                cherrypy.log('Timed out', 'TOOLS.CACHING')
            e = threading.Event()
            e.result = None
            dict.__setitem__(self, key, e)

            return None
        elif value is None:
            # Stick an Event in the slot so other threads wait
            # on this one to finish calculating the value.
            if debug:
                cherrypy.log('Timed out', 'TOOLS.CACHING')
            e = threading.Event()
            e.result = None
            dict.__setitem__(self, key, e)
        return value

    def __setitem__(self, key, value):
        """Set the cached value for the given key."""
        existing = self.get(key)
        dict.__setitem__(self, key, value)
        if isinstance(existing, Event):
            # Set Event.result so other threads waiting on it have
            # immediate access without needing to poll the cache again.
            existing.result = value
            existing.set()

class MemoryCache(Cache):

    """An in-memory cache for varying response content.

    Each key in self.store is a URI, and each value is an AntiStampedeCache.
    The response for any given URI may vary based on the values of
    "selecting request headers"; that is, those named in the Vary
    response header. We assume the list of header names to be constant
    for each URI throughout the lifetime of the application, and store
    that list in ``self.store[uri].selecting_headers``.

    The items contained in ``self.store[uri]`` have keys which are tuples of
    request header values (in the same order as the names in its
    selecting_headers), and values which are the actual responses.
    """

    maxobjects = 1000
    """The maximum number of cached objects; defaults to 1000."""

    maxobj_size = 100000
    """The maximum size of each cached object in bytes; defaults to 100 KB."""

    maxsize = 10000000
    """The maximum size of the entire cache in bytes; defaults to 10 MB."""

    delay = 600
    """Seconds until the cached content expires; defaults to 600 (10 minutes).
    """

    antistampede_timeout = 5
    """Seconds to wait for other threads to release a cache lock."""

    expire_freq = 0.1
    """Seconds to sleep between cache expiration sweeps."""

    debug = False

    def __init__(self):
        self.clear()

        # Run self.expire_cache in a separate daemon thread.
        t = threading.Thread(target=self.expire_cache, name='expire_cache')
        self.expiration_thread = t
        t.daemon = True
        t.start()

    def clear(self):
        """Reset the cache to its initial, empty state."""
        self.store = {}
        self.expirations = {}
        self.tot_puts = 0
        self.tot_gets = 0
        self.tot_hist = 0
        self.tot_expires = 0
        self.tot_non_modified = 0
        self.cursize = 0

    def expire_cache(self):
        """Continuously examine cached objects, expiring stale ones.

        This function is designed to be run in its own daemon thread,
        referenced at ``self.expiration_thread``.
        """
        # It's possible that "time" will be set to None
        # arbitrarily, so we check "while time" to avoid exceptions.
        # See tickets #99 and #180 for more information.
        while time:
            now = time.time()
            # Must make a copy of expirations so it doesn't change size
            # during iteration
            items = list(six.iteritems(self.expirations))
            for expiration_time, objects in items:
                if expiration_time <= now:
                    for obj_size, uri, sel_header_values in objects:
                        try:
                            del self.store[uri][tuple(sel_header_values)]
                            self.tot_expires += 1
                            self.cursize -= obj_size
                        except KeyError:
                            # the key may have been deleted elsewhere
                            pass
                    del self.expirations[expiration_time]
            time.sleep(self.expire_freq)

    def get(self):
        """Return the current variant if in the cache, else None."""
        request = cherrypy.serving.request
        self.tot_gets += 1

        uri = cherrypy.url(qs=request.query_string)
        uricache = self.store.get(uri)
        if uricache is None:
            return None

        header_values = [request.headers.get(h, '')
                         for h in uricache.selecting_headers]
        variant = uricache.wait(key=tuple(sorted(header_values)),
                                timeout=self.antistampede_timeout,
                                debug=self.debug)
        if variant is not None:
            self.tot_hist += 1
        return variant

    def put(self, variant, size):
        """Store the current variant in the cache."""
        request = cherrypy.serving.request
        response = cherrypy.serving.response


import gensim
import pandas as pd
import random
import numpy as np
import matplotlib.pyplot as plt
import cPickle as pickle
from scipy.spatial.distance import cosine
from nltk.tokenize import PunktSentenceTokenizer
import multiprocessing as mp
import time

###########################################################################
# tokenization code

def seperatePunct(incomingString):
    outstr = ''
    characters = set(['!','@','#','$',"%","^","&","*",":","\\",
                  "(",")","+","=","?","\'","\"",";","/",
                  "{","}","[","]","<",">","~","`","|"])

    for char in incomingString:
        if char in characters:
            outstr = outstr + ' ' + char + ' '
        else:
            outstr = outstr + char

    return outstr

def hasNumbers(inputString):
     return any(char.isdigit() for char in inputString)

def text_cleaner(wordList):
    '''
    INPUT: List of words to be tokenized
    OUTPUT: List of tokenized words
    '''

    tokenziedList = []

    for word in wordList:

        #remove these substrings from the word
        word = word.replace('[deleted]','')
        word = word.replace('&gt','')

        #if link, replace with linktag
        if 'http' in word:
            tokenziedList.append('LINK_TAG')
            continue

        #if reference to subreddit, replace with reddittag
        if '/r/' in word:
            tokenziedList.append('SUBREDDIT_TAG')
            continue

        #if reference to reddit user, replace with usertag
        if '/u/' in word:
            tokenziedList.append('USER_TAG')
            continue

        #if reference to twitter user, replace with usertag
        if '@' in word:
            tokenziedList.append('USER_TAG')
            continue

        #if number, replace with numtag
        #m8 is a word, 5'10" and 54-59, 56:48 are numbers
        if hasNumbers(word) and not any(char.isalpha() for char in word):
            tokenziedList.append('NUM_TAG')
            continue

        #seperate puncuations and add to tokenizedList
        newwords = seperatePunct(word).split(" ")
        tokenziedList.extend(newwords)

    return tokenziedList

def mytokenizer(comment):
    '''
    Input: takes in a reddit comment as a str or unicode and tokenizes it
    Output: a tokenized list
    '''
    tokenizer = PunktSentenceTokenizer()
    sentenceList = tokenizer.tokenize(comment)
    wordList = []
    for sentence in sentenceList:
        wordList.extend(sentence.split(" "))

    return text_cleaner(wordList)

#############################################################################
#similarity code

def mostSimilarDoc(model,comment,k,threshold):
    '''
    Input: doc2vec model, comment is a str, k = number of similar doc vecs
    Output: an int indicating hate (1) or not hate (0),most similar subreddit
    '''

    docvecs = model.docvecs
    numdocvec = len(docvecs)
    simVals = np.zeros((numdocvec, ))

    #tokenize comment
    wordTokens = mytokenizer(comment)

    #create vector of tokenized comment
    #avg over 100 vectors
    finalVec = np.zeros((300, ))
    for i in xrange(100):
        finalVec = finalVec + model.infer_vector(wordTokens)
    commentVec = finalVec/100.0

    #compute similarity of comment to each subreddit
    for vec_ind in xrange(len(docvecs)):
        simVals[vec_ind] = 1 - cosine(commentVec,docvecs[vec_ind])

    mostSimVecInd = np.argsort(simVals)[-k:]
    hatecount = 0

    #count how many hates there are
    for index in mostSimVecInd:
        hatecount += ishateful(docvecs.index_to_doctag(index))

    #majority vote to determine hateful/nothateful
    if hatecount>=threshold*len(mostSimVecInd):
        prediction = 1
    else:
        prediction = 0

    #find most similar subreddit
    # mostSimSubreddit = docvecs.index_to_doctag(mostSimVecInd[0])

    return prediction

##############################################################################
#hate/NotHate code

def ishateful(subreddit):
    '''
    Input: str subreddit
    Output: int 1 if hateful subreddit, 0 otherwise
    '''

    # List of not hateful subreddits
    final_nothate_srs = ['politics', 'worldnews', 'history', 'blackladies', 'lgbt',
                         'TransSpace', 'women', 'TwoXChromosomes', 'DebateReligion',
                         'religion', 'islam', 'Judaism', 'BodyAcceptance', 'fatlogic'
                         'gaybros','AskMen','AskWomen']
    # List of hateful subreddits
    final_hateful_srs = ['CoonTown', 'WhiteRights', 'Trans_fags', 'SlutJustice',
                         'TheRedPill', 'KotakuInAction', 'IslamUnveiled', 'GasTheKikes',
                         'AntiPOZi', 'fatpeoplehate', 'TalesofFatHate','hamplanethatred',
                         'shitniggerssay','neofag','altright']

    if subreddit in final_hateful_srs:
        return 1
    else:
        return 0

#############################################################################
#scoring code

def test_score(model,path,param):
    '''
    Input: doc2vec model, path to csv file of cross val set, param = [k,threshold]
    Output: [k,threshold,accu,recall,precision,TP,TN,FN,FP]
    '''

    # print "loading data..."
    df = pd.read_csv(path)
    tweets = df['tweet_text'].values
    labels = df['label'].values

    predict = np.zeros((len(labels),))

    k = param[0]
    threshold = param[1]

    # print "scoring..."
    for row in xrange(len(labels)):
        tweet = tweets[row]
        prediction = mostSimilarDoc(model,tweet,k,threshold)
        predict[row] = prediction

    TP = sum(predict+labels == 2)
    TN = sum(predict+labels == 0)
    FP = sum(predict-labels == 1)
    FN = sum(predict-labels == -1)

    accu = (TP+TN)/float(len(labels))
    recall = TP/float(TP+FN)
    precision = TP/float(TP+FP)

    print ""
    print "k: {}".format(k)
    print "threshold: {}".format(threshold)
    print "accuracy: {}".format(accu)
    print "recall: {}".format(recall)
    print "precision: {}".format(precision)

    print ""
    print "TP: {}".format(TP)
    print "TN: {}".format(TN)
    print ""
    print "FN: {}".format(FN)
    print "FP: {}".format(FP)

    #output data to be saved in a pd dataframe
    return [k,threshold,accu,recall,precision,TP,TN,FN,FP]

##############################################################################
#Main

if __name__ == '__main__':

    print "starting..."

    #dataset paths
    trainpath = '../../data/labeledRedditComments.p'
    trainpath2 = '../../data/labeledRedditComments2.p'
    cvpath = '../../data/twitter_cross_val.csv'
    testpath = '../../data/twitter_test.csv'
    sqlpath = '../../data/RedditMay2015Comments.sqlite'

    #model paths
    # modelPath = '../../doc2vec_models/basemodel2/basemodel2.doc2vec'
    modelPath = '../../doc2vec_models/basemodel3/basemodel3.doc2vec'
    # modelPath = '../../doc2vec_models/modellower/modellower.doc2vec'
    # modelPath = '../../doc2vec_models/model_split/model_split.doc2vec'

    print "loading model..."
    model = gensim.models.Doc2Vec.load(modelPath)

    #find num of cores
    cores = mp.cpu_count()

    pool = mp.Pool(processes=cores)

    print "creating paramlist..."
    paramlist = []

    # for k in xrange(1,14):
    #     for threshold in [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]:
    #         paramlist.append([k,threshold])

    for k in xrange(1,3):


# ----------------------------------------------------------------------------
#
# Sally BN: An Open-Source Framework for Bayesian Networks.
#
# ----------------------------------------------------------------------------
# GNU General Public License v2
#
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation; either version 2 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License along
#   with this program; if not, write to the Free Software Foundation, Inc.,
#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# ----------------------------------------------------------------------------

### Graphic widgets

# Sally-BN
# Mavrin
# Johny Five
# Rose
# Bender
# Dorian

from gi.repository import Gtk
from gi.repository import Pango
import random
import math
from lib_sallybn.util import ulist

class StatesTable(object):
    """
    Shows a table with variable's states.
    """

    def __init__(self, disc_bn, selected_vetex,
                 state_changed_func, view, badd_state, bremove_state):

        """

        :param disc_bn:
        :param selected_vetex:
        :param state_changed_func:
        :param view:
        :param badd_state:
        :param bremove_state:
        """
        self.selected_vetex = selected_vetex
        self.disc_bn = disc_bn
        self.view = view
        self.state_changed_func = state_changed_func
        self.model = None
        self._modify_treeview_for_states()

        badd_state.connect("clicked", self._add_state)
        bremove_state.connect("clicked", self._remove_state)

    def _add_state(self, widget):
        self.model.append(["New state"])
        self.disc_bn.add_state(self.selected_vetex, "New state")
        self.state_changed_func()

    def _remove_state(self, widget):
        if len(self.model) != 0:
            (model, selected_rows) = self.view.get_selection().get_selected()
            if selected_rows is not None:
                del_state = self.model[selected_rows][0]
                # Graphic remove
                self.model.remove(selected_rows)
                # remove from graph
                self.disc_bn.add_state(self.selected_vetex, del_state)

    def _modify_treeview_for_states(self):
        ### MODEL
        self.model = Gtk.ListStore(str)
        self.view.set_model(self.model)

        states = self.disc_bn.get_states(self.selected_vetex)
        # fill states
        for state in states:
            self.model.append([state])
        # New state
        # self.model.append([NEW_STATE_NAME, Gtk.STOCK_ADD])

        ### VIEW
        cell = Gtk.CellRendererText()
        cell.props.editable = True
        # Call signal
        cell.connect("edited", self.text_edited)

        # the column is created
        col = Gtk.TreeViewColumn("States", cell, text=0)

        # append to the tree view
        self.view.append_column(col)

    def text_edited(self, widget, path, new_text):
        # Current name
        current_state = self.model[path][0]

        self.disc_bn.change_state_name(self.selected_vetex, current_state, new_text)
        # TODO validate non repeated names

        # modify gui
        self.model[path][0] = new_text
        self.state_changed_func()

class GraphicCptTable(object):
    """
    Shows a Conditional Probability Distribution to edit.
    """

    def __init__(self, disc_bn, query_v, view=Gtk.TreeView()):
        """
        Create a tree view for the CPT of the node query_v
        :param query_v:
        :return:
        """
        self.disc_bn = disc_bn

        self.view = view
        self.query_v = query_v

        ### DATA MODEL
        self.model = None
        self.editable_cells = {}
        # self.parents

        self.modify_treeview_for_cpt()

    def modify_treeview_for_cpt(self):
        """
        Modify a tree view for the CPT of the node query_v
        """
        ### Detect parents
        parents = self.disc_bn.getparents(self.query_v)

        node_states = self.disc_bn.get_states(self.query_v)
        # number of columns
        n_state_cols = len(node_states)
        # Columns for titles
        n_parent_cols = len(parents)

        # TODO Validate parents not zero states

        # number of rows
        n_rows = 1
        for p in parents:
            n_rows *= len(self.disc_bn.get_states(p))

        # data types for parents
        parent_types = [str for i in parents]
        # the data in the model (three strings for each row, one for each column)
        cpt_types = [str for i in node_states]
        # all parent + cpt types
        all_types = parent_types + cpt_types

        ### DATA MODEL
        self.model = Gtk.ListStore(*all_types)
        self.view.set_model(self.model)

        # FILL CLEAN CPT
        parents_matrix = self.disc_bn.get_parent_states(self.query_v)
        str_parent_matrix = self.disc_bn.str_parent_states(parents_matrix)
        cprob = self.disc_bn.get_cprob(self.query_v)

        # PUT CPT in model
        for l in range(n_rows):
            # Fill for no parents
            if len(parents) == 0:
                str_prob = ulist.list_to_str_list(cprob)
                self.model.append(str_prob)

            else:
                cprob_line = cprob[str_parent_matrix[l]]
                str_cprob_line = ulist.list_to_str_list(cprob_line)
                self.model.append(parents_matrix[l] + str_cprob_line)

        ### end fill data

        # Table titles
        table_titles = parents + [s + "(%)" for s in node_states]

        ### remove all columns
        cols = self.view.get_columns()
        for c in cols:
            self.view.remove_column(c)

        ####### ADD GRAPHIC COLUMNS
        for i in range(len(table_titles)):
            # cellrenderer to render the text
            cell = Gtk.CellRendererText()

            # Title in bold
            if i < n_parent_cols:
                cell.props.weight_set = True
                cell.props.weight = Pango.Weight.BOLD
                cell.props.background = "gray"

            else:
                cell.props.xalign = 1.0
                cell.props.editable = True
                self.editable_cells[cell] = i

                # Call signal
                cell.connect("edited", self.text_edited)

            # the column is created
            col = Gtk.TreeViewColumn(table_titles[i], cell, text=i)

            # append to the tree view
            self.view.append_column(col)

    ## Event for Edited cells
    def text_edited(self, widget, path, text):
        #2 validate the number between 0 and 100
        val = float(text)


import logging
import re
from collections import OrderedDict
from decimal import Decimal

from django.db import connection

from storage.models import Ingest, Collection, Request, StorageProduct, \
    CollectionProfile

class RedsReportOptions:
    MELBOURNE = 'Melbourne'  # only Melbourne storage products
    ALL = 'All'  # all storage products

class _ReportRow:
    def __init__(self, is_reds=True):
        self.is_reds = is_reds
        self._collection_name = ''
        self._node_id = 0
        self._for_1 = ''
        self._for_2 = ''
        self._for_3 = ''
        self._for_4 = ''
        self._for_5 = ''
        self._for_6 = ''
        self._for_7 = ''
        self._for_8 = ''
        self._for_9 = ''
        self._for_10 = ''
        self._approved_tb = 0
        self._available_tb = 0
        self._committed_tb = 0
        self._custodian = ''
        self._organization = ''
        self._faculty = ''
        self._link = ''
        self._description = ''

    @staticmethod
    def column_names(is_reds=True):
        """
        :param is_reds: True if FOR codes are to be included, False if not
        :return: the values of a row in the report
        """
        # Description can be quite a long multi-line string...
        result = ['Collection Name', 'Node ID']
        if is_reds:
            result += ['FOR 1', 'FOR 2',
                       'FOR 3', 'FOR 4', 'FOR 5', 'FOR 6', 'FOR 7', 'FOR 8',
                       'FOR 9', 'FOR 10']
        result += ['Research Data Approved (TB)',
                   'Research Data Available (Ready) (TB)',
                   'Total Storage Allocated (Committed) (TB)',
                   'Data Custodian',
                   'Organization', 'Faculty',
                   'Link', 'Description']
        return result

    def _values(self):
        """
        Following must be in the same order as the COLUMN_NAMES, as
        get_values(...) returns their values in the order below
        ALSO: if you add/remove an attribute, it must have a corresponding
        COLUMN_NAME entry added/removed
        :return: A list containing the attributes in the same order as the
                 COLUMN_NAMES values
        """
        result = [self._collection_name, self._node_id]
        if self.is_reds:
            result += [self._for_1, self._for_2, self._for_3, self._for_4,
                       self._for_5, self._for_6, self._for_7, self._for_8,
                       self._for_9, self._for_10]
        result += [self._approved_tb, self._available_tb,
                   self._committed_tb, self._custodian,
                   self._organization, self._faculty, self._link,
                   self._description, ]
        return result

    @property
    def description(self):
        return self._description

    @description.setter
    def description(self, value):
        """
        Substitutes every white space instance in the description with a single
        space
        """
        self._description = re.sub('\s+', ' ', value).strip()

    def set_for_code(self, code_number, value):
        """
        :param code_number: the FOR code number to set
        :param value: the value to set the FOR code to
        """
        setattr(self, '_for_%s' % code_number, value)

    def get_values(self):
        """
        :return: the values of the instance fields
        """
        return [value for value in self._values()]

MARKET_MONASH = 'Market.Monash'
VAULT_MONASH = 'Vault.Monash'

# following constants are again hard coded label values... If the label is
# changed, the code breaks...

COMPUTATIONAL_STORAGE = ['Computational.Monash.Performance',
                         'Computational.Melbourne']

MELBOURNE_STORAGE = COMPUTATIONAL_STORAGE + ['Market.Melbourne',
                                             'Vault.Melbourne.Object']

VALID_SCHEMES = ['RDSI Merit', 'RDSI Board (ReDS3)']

logger = logging.getLogger(__name__)

def _get_tb_from_gb(size_in_gb):
    return size_in_gb / 1000

def _latest_data_from_ingests(storage_products):
    """
    :param storage_products: the service providers to filter ingests by
    :return: an ordered dict containing the ingests for each collection and
             storage product, again, in ordered dicts ordered by extraction
             date descending
    """
    result = OrderedDict()
    # find the set of Ingests based on the storage product list ordered latest
    # to earliest, distinct by collection id and storage product id
    ingests = Ingest.objects.filter(
        storage_product__in=storage_products).order_by(
        'collection_id', 'storage_product_id', '-extraction_date').distinct(
        'collection_id', 'storage_product_id')
    for ingest in ingests:
        if ingest.collection.id not in result:
            result[ingest.collection.id] = OrderedDict()
        storage_product_name = ingest.storage_product.product_name.value
        result[ingest.collection.id][storage_product_name] = ingest
    return result

def _allocation_totals(allocations, rr, storage_products):
    """
    :param allocations: the allocations to be summed
    :param rr: *mutated* the allocation sums are written into the report row
    :param storage_products: the storage products whose allocations are to be
                             summed
    :return: the sum of the allocation sizes
    """
    total_used = Decimal(0.0)
    raw_alloc = {sp.product_name.value: Decimal(0.0) for sp in
                 storage_products}
    for allocation in allocations:
        sp = allocation.storage_product
        if sp in storage_products:
            sp_name = sp.product_name.value
            if sp_name in MELBOURNE_STORAGE:
                raw_alloc[
                    sp_name] += allocation.size_tb / sp.raw_conversion_factor
                if sp_name in COMPUTATIONAL_STORAGE:
                    total_used += allocation.size
            elif sp_name in [VAULT_MONASH]:
                raw_alloc[VAULT_MONASH] += allocation.size_tb * Decimal(0.069)
                raw_alloc[MARKET_MONASH] += allocation.size_tb * Decimal(2.0)
            elif sp_name in [MARKET_MONASH]:
                raw_alloc[VAULT_MONASH] += allocation.size_tb * Decimal(1.752)
                raw_alloc[MARKET_MONASH] += allocation.size_tb * Decimal(1.752)
    rr._committed_tb = sum(raw_alloc.values())
    return total_used

def _non_compute_ingests_used_capacity(collection, filtered_ingests):
    """
    :param collection: the collection whose ingests are to be summed
    :param filtered_ingests: the last ingests by storage product
    :return: the sum of the used capacity for non compute ingests
    """
    capacity_used = 0
    collection_ingests = filtered_ingests.get(collection.id)
    if collection_ingests:
        for storage_product_name in collection_ingests.keys():
            if storage_product_name in COMPUTATIONAL_STORAGE:
                continue
            ingest = collection_ingests.get(storage_product_name)
            if not ingest.used_capacity:
                continue
            capacity_used += ingest.used_capacity
    return capacity_used

def _get_organization_and_faculty(collection_id):
    """
    To match a collection to an institution:
          request <- allocation -> collection
          request -> organization
          request -> suborganization (faculty)
    So we have to select all the allocations that reference a collection,
    and then left join the organization and the faculty.

    TODO: this code makes the assumption that there will always be one row
          returned. What if there is no row? Or multiple rows?
    """
    with connection.cursor() as cursor:
        cursor.execute("""
            SELECT DISTINCT
                COALESCE(short_name, '') AS organization,
                COALESCE(name, '') AS faculty
            FROM applications_allocation
              LEFT JOIN applications_request
                ON applications_allocation.application_id =
                    applications_request.id
              LEFT JOIN contacts_organisation
                ON applications_request.institution_id =
                    contacts_organisation.id
              LEFT JOIN applications_suborganization
                ON applications_request.faculty_id =
                    applications_suborganization.id
            WHERE collection_id = %s;""", [collection_id])
        columns = [col[0] for col in cursor.description]
        result = dict(zip(columns, cursor.fetchone()))
        return result

def reds_123_calc(org_type, is_reds=True):
    """
    REDS == 'Research E-Data Scheme' ?
    REDS == 'Research Data Services' ?
    :param org_type: the organisations to return the report for
    :param is_reds: if the report is a reds one or not...
    :return: a list of rows containing the calculated values for
             each collection that belongs to the org_type
    """
    result = [_ReportRow.column_names(is_reds)]

    if RedsReportOptions.MELBOURNE == org_type:
        storage_products = StorageProduct.objects.filter(
            product_name__value__icontains='Melbourne')
    else:
        storage_products = StorageProduct.objects.all()

    filtered_ingests = _latest_data_from_ingests(storage_products)

    for collection in Collection.objects.all().order_by('name'):
        if not collection.has_allocation_for_storage_products(
                storage_products):
            continue

        try:
            request = Request.objects.get(code=collection.application_code)
            if not request or not request.status:
                continue
            status = request.status.value
            if is_reds and request.scheme.value not in VALID_SCHEMES:
                continue
        except (Request.DoesNotExist, Request.MultipleObjectsReturned):
            logger.exception("Original request not found?")
            continue  # unexpected, so do not output a row for this collection

        # we only want data for the approved applications
        if status == 'Approved':
            # set all the initial column values to 0
            rr = _ReportRow(is_reds)
            rr._collection_name = collection.name
            rr._node_id = request.code
            if is_reds:
                count = 0
                # fetch, at most 10, domains for the collection
                for domain in collection.domains.all()[:10]:
                    count += 1
                    rr.set_for_code(count, domain.field_of_research.code)
            rr._approved_tb = collection.total_allocation
            total_used = _allocation_totals(collection.allocations.all(),
                                            rr, storage_products)
            total_used += _non_compute_ingests_used_capacity(collection,
                                                             filtered_ingests)