#!/usr/bin/env python
#-------------------------------------------------------------------------------
#    FILE: genmon.py
# PURPOSE: Monitor for Generator
#
#  AUTHOR: Jason G Yates
#    DATE: 05-Oct-2016
#          23-Apr-2018
#
# MODIFICATIONS:
#-------------------------------------------------------------------------------

from __future__ import print_function       # For python 3.x compatibility with print function

import datetime, time, sys, signal, os, threading, socket
import atexit, json, collections, random, getopt
import re
from subprocess import PIPE, Popen

try:
    from genmonlib.mylog import SetupLogger
    from genmonlib.myconfig import MyConfig
    from genmonlib.mymail import MyMail
    from genmonlib.mythread import MyThread
    from genmonlib.mysupport import MySupport
    from genmonlib.mypipe import MyPipe
    from genmonlib.generac_evolution import Evolution
    from genmonlib.generac_HPanel import HPanel
    from genmonlib.generac_powerzone import PowerZone
    from genmonlib.custom_controller import CustomController
    from genmonlib.myweather import MyWeather
    from genmonlib.myplatform import MyPlatform
    from genmonlib.program_defaults import ProgramDefaults
except Exception as e1:
    print("\n\nThis program requires the modules located in the genmonlib directory in the github repository.\n")
    print("Please see the project documentation at https://github.com/jgyates/genmon.\n")
    print("Error: " + str(e1))
    sys.exit(2)

GENMON_VERSION = "V1.18.08"

#------------ Monitor class ----------------------------------------------------
class Monitor(MySupport):

    def __init__(self, ConfigFilePath = ProgramDefaults.ConfPath):
        super(Monitor, self).__init__()

        self.ProgramName = "Generator Monitor"
        self.Version = "Unknown"
        self.log = None
        self.IsStopping = False
        self.ProgramComplete = False
        if ConfigFilePath == None or ConfigFilePath == "":
            self.ConfigFilePath = ProgramDefaults.ConfPath
        else:
            self.ConfigFilePath = ConfigFilePath

        self.ConnectionList = []    # list of incoming connections for heartbeat
        # defautl values
        self.SiteName = "Home"
        self.ServerSocket = None
        self.ServerSocketPort = ProgramDefaults.ServerPort    # server socket for nagios heartbeat and command/status
        self.IncomingEmailFolder = "Generator"
        self.ProcessedEmailFolder = "Generator/Processed"

        self.FeedbackLogFile = os.path.join(self.ConfigFilePath, "feedback.json")
        self.LogLocation = ProgramDefaults.LogPath
        self.LastLogFileSize = 0
        self.NumberOfLogSizeErrors = 0
        # set defaults for optional parameters
        self.NewInstall = False         # True if newly installed or newly upgraded version
        self.FeedbackEnabled = False    # True if sending autoated feedback on missing information
        self.FeedbackMessages = {}
        self.OneTimeMessages = {}
        self.MailInit = False       # set to true once mail is init
        self.CommunicationsActive = False   # Flag to let the heartbeat thread know we are communicating
        self.Controller = None
        self.ControllerSelected = None
        self.bDisablePlatformStats = False
        self.ReadOnlyEmailCommands = False
        self.SlowCPUOptimization = False
        # weather parameters
        self.WeatherAPIKey = None
        self.WeatherLocation = None
        self.UseMetric = False
        self.WeatherMinimum = True
        self.DisableWeather = False
        self.MyWeather = None
        self.UpdateAvailable = False

        # Time Sync Related Data
        self.bSyncTime = False          # Sync gen to system time
        self.bSyncDST = False           # sync time at DST change
        self.bDST = False               # Daylight Savings Time active if True
        # simulation
        self.Simulation = False
        self.SimulationFile = None

        self.console = SetupLogger("genmon_console", log_file = "", stream = True)

        if not MySupport.PermissionsOK():
            self.LogConsole("You need to have root privileges to run this script.\nPlease try again, this time using 'sudo'.")
            sys.exit(1)

        if not os.path.isfile(os.path.join(self.ConfigFilePath, 'genmon.conf')):
            self.LogConsole("Missing config file : " + os.path.join(self.ConfigFilePath, 'genmon.conf'))
            sys.exit(1)
        if not os.path.isfile(os.path.join(self.ConfigFilePath, 'mymail.conf')):
            self.LogConsole("Missing config file : " + os.path.join(self.ConfigFilePath, 'mymail.conf'))
            sys.exit(1)

        self.config = MyConfig(filename = os.path.join(self.ConfigFilePath, 'genmon.conf'), section = "GenMon", log = self.console)
        # read config file
        if not self.GetConfig():
            self.LogConsole("Failure in Monitor GetConfig")
            sys.exit(1)

        # log errors in this module to a file
        self.log = SetupLogger("genmon", os.path.join(self.LogLocation, "genmon.log"))

        self.config.log = self.log

        if self.IsLoaded(): # this checks based on the port used for the API
            self.LogConsole("ERROR: genmon.py is already loaded.")
            self.LogError("ERROR: genmon.py is already loaded.")
            sys.exit(1)

        # this check is based on the file name.
        if MySupport.IsRunning(os.path.basename(__file__), multi_instance = self.multi_instance):
            self.LogConsole("ERROR: genmon.py is already loaded.")
            self.LogError("ERROR: genmon.py is already loaded (2).")
            sys.exit(1)

        if self.NewInstall:
            self.LogError("New version detected: Old = %s, New = %s" % (self.Version, ProgramDefaults.GENMON_VERSION))
            self.Version = ProgramDefaults.GENMON_VERSION

        self.ProgramStartTime = datetime.datetime.now()     # used for com metrics
        # this will wait one day for an update, change to
        #  datetime.datetime(1, 1, 1, 0, 0) to check immediately on load
        self.LastSofwareUpdateCheck = datetime.datetime.now()

        signal.signal(signal.SIGTERM, self.SignalClose)
        signal.signal(signal.SIGINT, self.SignalClose)

        # start thread to accept incoming sockets for nagios heartbeat and command / status clients
        self.Threads["InterfaceServerThread"] = MyThread(self.InterfaceServerThread, Name = "InterfaceServerThread")

        # init mail, start processing incoming email
        self.mail = MyMail(monitor=True, incoming_folder = self.IncomingEmailFolder,
            processed_folder =self.ProcessedEmailFolder,incoming_callback = self.ProcessCommand,
            loglocation = self.LogLocation, ConfigFilePath = ConfigFilePath)

        self.Threads = self.MergeDicts(self.Threads, self.mail.Threads)
        self.MailInit = True

        self.FeedbackPipe = MyPipe("Feedback", self.FeedbackReceiver,
            log = self.log, ConfigFilePath = self.ConfigFilePath)
        self.Threads = self.MergeDicts(self.Threads, self.FeedbackPipe.Threads)
        self.MessagePipe = MyPipe("Message", self.MessageReceiver, log = self.log,
            nullpipe = self.mail.DisableSNMP, ConfigFilePath = self.ConfigFilePath)
        self.Threads = self.MergeDicts(self.Threads, self.MessagePipe.Threads)

        try:
            #Starting device connection
            if self.Simulation:
                self.LogError("Simulation Running")
            if not self.ControllerSelected == None and len(self.ControllerSelected):
                self.LogError("Selected Controller: " + str(self.ControllerSelected))
            else:
                self.ControllerSelected = "generac_evo_nexus"

            if self.ControllerSelected.lower() == "h_100" :
                self.Controller = HPanel(self.log, newinstall = self.NewInstall, simulation = self.Simulation, simulationfile = self.SimulationFile, message = self.MessagePipe, feedback = self.FeedbackPipe, config = self.config)
            elif self.ControllerSelected.lower() == "powerzone" :
                self.Controller = PowerZone(self.log, newinstall = self.NewInstall, simulation = self.Simulation, simulationfile = self.SimulationFile, message = self.MessagePipe, feedback = self.FeedbackPipe, config = self.config)
            elif self.ControllerSelected.lower() == "custom" :
                self.Controller = CustomController(self.log, newinstall = self.NewInstall, simulation = self.Simulation, simulationfile = self.SimulationFile, message = self.MessagePipe, feedback = self.FeedbackPipe, config = self.config)
            else:
                self.Controller = Evolution(self.log, self.NewInstall, simulation = self.Simulation, simulationfile = self.SimulationFile, message = self.MessagePipe, feedback = self.FeedbackPipe, config = self.config)
            self.Threads = self.MergeDicts(self.Threads, self.Controller.Threads)

        except Exception as e1:
            self.LogErrorLine("Error opening controller device: " + str(e1))
            sys.exit(1)

        self.StartThreads()

        self.ProcessFeedbackInfo()

        # send mail to tell we are starting
        IP = self.GetNetworkIp()
        self.MessagePipe.SendMessage("Generator Monitor Starting at " + self.SiteName, "Generator Monitor Starting at " + self.SiteName + " using IP address " + IP, msgtype = "info")

        self.LogError("GenMon Loaded for site: " + self.SiteName + " using python " + str(sys.version_info.major) + "." + str(sys.version_info.minor))

    # ------------------------ Monitor::StartThreads----------------------------
    def StartThreads(self, reload = False):

        try:
            # start thread to accept incoming sockets for nagios heartbeat
            self.Threads["ComWatchDog"] = MyThread(self.ComWatchDog, Name = "ComWatchDog")

            if self.bSyncDST or self.bSyncTime:     # Sync time thread
                self.Threads["TimeSyncThread"] = MyThread(self.TimeSyncThread, Name = "TimeSyncThread")

            if not self.DisableWeather and not self.WeatherAPIKey == None and len(self.WeatherAPIKey) and not self.WeatherLocation == None and len(self.WeatherLocation):
                Unit = 'metric' if self.UseMetric else 'imperial'
                self.MyWeather = MyWeather(self.WeatherAPIKey, location = self.WeatherLocation, unit = Unit, log = self.log)
                self.Threads = self.MergeDicts(self.Threads, self.MyWeather.Threads)
        except Exception as e1:
            self.LogErrorLine("Error in StartThreads: " + str(e1))

    # -------------------- Monitor::GetConfig-----------------------------------
    def GetConfig(self):

        try:
            if self.config.HasOption('sitename'):
                self.SiteName = self.config.ReadValue('sitename')

            self.multi_instance =  self.config.ReadValue('multi_instance', return_type = bool, default = False)

            if self.config.HasOption('incoming_mail_folder'):
                self.IncomingEmailFolder = self.config.ReadValue('incoming_mail_folder')     # imap folder for incoming mail

            if self.config.HasOption('processed_mail_folder'):
                self.ProcessedEmailFolder = self.config.ReadValue('processed_mail_folder')   # imap folder for processed mail
            #  server_port, must match value in myclient.py and check_monitor_system.py and any calling client apps
            if self.config.HasOption('server_port'):
                self.ServerSocketPort = self.config.ReadValue('server_port', return_type = int)

            self.LogLocation = self.config.ReadValue('loglocation', default = ProgramDefaults.LogPath)

            self.UserDefinedDataPath = self.config.ReadValue('userdatalocation', default = os.path.dirname(os.path.realpath(__file__)))

            if self.config.HasOption('syncdst'):
                self.bSyncDST = self.config.ReadValue('syncdst', return_type = bool)
            if self.config.HasOption('synctime'):
                self.bSyncTime = self.config.ReadValue('synctime', return_type = bool)

            if self.config.HasOption('disableplatformstats'):
                self.bDisablePlatformStats = self.config.ReadValue('disableplatformstats', return_type = bool)

            if self.config.HasOption('simulation'):
                self.Simulation = self.config.ReadValue('simulation', return_type = bool)

            if self.config.HasOption('simulationfile'):
                self.SimulationFile = self.config.ReadValue('simulationfile')

            if self.config.HasOption('controllertype'):
                self.ControllerSelected = self.config.ReadValue('controllertype')

            if self.config.HasOption('disableweather'):
                self.DisableWeather = self.config.ReadValue('disableweather', return_type = bool)
            else:
                self.DisableWeather = False

            if self.config.HasOption('weatherkey'):
                self.WeatherAPIKey = self.config.ReadValue('weatherkey')

            if self.config.HasOption('weatherlocation'):
                self.WeatherLocation = self.config.ReadValue('weatherlocation')

            if self.config.HasOption('metricweather'):
                self.UseMetric = self.config.ReadValue('metricweather', return_type = bool)

            if self.config.HasOption('minimumweatherinfo'):
                self.WeatherMinimum = self.config.ReadValue('minimumweatherinfo', return_type = bool)

            if self.config.HasOption('readonlyemailcommands'):
                self.ReadOnlyEmailCommands = self.config.ReadValue('readonlyemailcommands', return_type = bool)

            if self.config.HasOption('optimizeforslowercpu'):
                self.SlowCPUOptimization = self.config.ReadValue('optimizeforslowercpu', return_type = bool)

            self.AdditionalWatchdogTime = self.config.ReadValue('watchdog_addition', return_type = int, default = 0)

            if self.config.HasOption('version'):
                self.Version = self.config.ReadValue('version')
                if not self.Version == ProgramDefaults.GENMON_VERSION:
                    self.config.WriteValue('version', ProgramDefaults.GENMON_VERSION)
                    self.NewInstall = True
            else:
                self.config.WriteValue('version', ProgramDefaults.GENMON_VERSION)
                self.NewInstall = True
                self.Version = ProgramDefaults.GENMON_VERSION
            if self.config.HasOption("autofeedback"):
                self.FeedbackEnabled = self.config.ReadValue('autofeedback', return_type = bool)
            else:
                self.config.WriteValue('autofeedback', "False")
                self.FeedbackEnabled = False
            # Load saved feedback log if log is present
            if os.path.isfile(self.FeedbackLogFile):
                try:
                    with open(self.FeedbackLogFile) as infile:
                        self.FeedbackMessages = json.load(infile)
                except Exception as e1:
                    os.remove(self.FeedbackLogFile)

            self.UpdateCheck = self.config.ReadValue('update_check', return_type = bool, default = True)
            self.UserURL = self.config.ReadValue('user_url',  default = "").strip()

        except Exception as e1:
            self.Console("Missing config file or config file entries (genmon): " + str(e1))
            return False

        return True

    #---------------------------------------------------------------------------
    def ProcessFeedbackInfo(self):

        try:
            if self.FeedbackEnabled:
                for Key, Entry in self.FeedbackMessages.items():
                    self.MessagePipe.SendMessage("Generator Monitor Submission", Entry , recipient = self.MaintainerAddress, files = self.GetLogFileNames(), msgtype = "error")
                # delete unsent Messages
                if os.path.isfile(self.FeedbackLogFile):
                    os.remove(self.FeedbackLogFile)
        except Exception as e1:
            self.LogErrorLine("Error in ProcessFeedbackInfo: " + str(e1))

    #---------------------------------------------------------------------------
    def FeedbackReceiver(self, Message):

        try:
            FeedbackDict = {}
            FeedbackDict = json.loads(Message)
            self.SendFeedbackInfo(FeedbackDict["Reason"],
                Always = FeedbackDict["Always"], Message = FeedbackDict["Message"],
                FullLogs = FeedbackDict["FullLogs"], NoCheck = FeedbackDict["NoCheck"])

        except Exception as e1:
            self.LogErrorLine("Error in  FeedbackReceiver: " + str(e1))
            self.LogError("Size : " + str(len(Message)))
            self.LogError("Message : " + str(Message))
    #---------------------------------------------------------------------------
    def MessageReceiver(self, Message):

        try:
            MessageDict = {}
            MessageDict = json.loads(Message)

            if MessageDict["onlyonce"]:
                Subject = self.OneTimeMessages.get(MessageDict["subjectstr"], None)
                if Subject == None:
                    self.OneTimeMessages[MessageDict["subjectstr"]] = MessageDict["msgstr"]
                else:
                    return

            self.mail.sendEmail(MessageDict["subjectstr"],
                MessageDict["msgstr"], recipient = MessageDict["recipient"],
                files = MessageDict["files"], deletefile= MessageDict["deletefile"],
                msgtype= MessageDict["msgtype"])

        except Exception as e1:
            self.LogErrorLine("Error in  MessageReceiver: " + str(e1))
    #---------------------------------------------------------------------------
    def SendFeedbackInfo(self, Reason, Always = False, Message = None, FullLogs = True, NoCheck = False):
        try:
            if self.NewInstall or Always:

                CheckedSent = self.FeedbackMessages.get(Reason, "")

                if not CheckedSent == "" and not NoCheck:
                    return

                if not NoCheck:
                    self.LogError(Reason + " : " + Message)

                msgbody = "Reason = " + Reason + "\n"
                if Message != None:
                    msgbody += "Message : " + Message + "\n"
                msgbody += self.printToString(self.ProcessDispatch(self.GetStartInfo(NoTile = True),""))
                if not self.bDisablePlatformStats:
                    msgbody += self.printToString(self.ProcessDispatch({"Platform Stats" : self.GetPlatformStats()},""))
                msgbody += self.printToString(self.ProcessDispatch({"Comm Stats" : self.Controller.GetCommStatus()},""))
                msgbody += self.Controller.DisplayRegisters(AllRegs = FullLogs)

                msgbody += "\n" + self.GetSupportData() + "\n"
                if self.FeedbackEnabled:
                    self.MessagePipe.SendMessage("Generator Monitor Submission", msgbody , recipient = self.MaintainerAddress, files = self.GetLogFileNames(), msgtype = "error")

                self.FeedbackMessages[Reason] = msgbody
                # if feedback not enabled, save the log to file
                if not self.FeedbackEnabled:
                    with open(self.FeedbackLogFile, 'w') as outfile:
                        json.dump(self.FeedbackMessages, outfile, sort_keys = True, indent = 4, ensure_ascii = False)
        except Exception as e1:
            self.LogErrorLine("Error in SendFeedbackInfo: " + str(e1))

    #---------- Monitor::EmailSendIsEnabled-------------------------------------
    def EmailSendIsEnabled(self):

        EmailThread = self.Threads.get("SendMailThread",None)
        if EmailThread == None:
            return False
        return True

    #---------- Monitor::GetSupportData-----------------------------------------
    def GetSupportData(self):

        SupportData = collections.OrderedDict()
        try:
            SupportData["Program Run Time"] = self.GetProgramRunTime()
            SupportData["Monitor Health"] = self.GetSystemHealth()
            SupportData["Controller Selected"] = self.ControllerSelected.lower()
            SupportData["StartInfo"] = self.GetStartInfo(NoTile = True)
            if not self.bDisablePlatformStats:
                SupportData["PlatformStats"] = self.GetPlatformStats()
            SupportData["Data"] = self.Controller.DisplayRegisters(AllRegs = True, DictOut = True)
            # Raw Modbus data
            SupportData["Registers"] = self.Controller.Registers
            SupportData["Strings"] = self.Controller.Strings
            SupportData["FileData"] = self.Controller.FileData
        except Exception as e1:
            self.LogErrorLine("Error in GetSupportData: " + str(e1))

        try:
            # indent 4 will keep some mail servers from having problems.
            return json.dumps(SupportData, indent=4, sort_keys=False)
        except Exception as e1:
            self.LogErrorLine("Error in GetSupportData (2): " + str(e1))
            return "Error Getting JSON data: " + str(e1)

    #---------- Monitor::GetLogFileNames----------------------------------------
    def GetLogFileNames(self):

        try:
            LogList = []
            FilesToSend = ["genmon.log", "genserv.log", "mymail.log", "myserial.log",
                "mymodbus.log", "gengpio.log", "gengpioin.log", "gensms.log",
                "gensms_modem.log", "genmqtt.log", "genpushover.log", "gensyslog.log",
                "genloader.log", "myserialtcp.log", "genlog.log", "genslack.log",
                "genexercise.log","genemail2sms.log", "gentankutil.log", "genalexa.log",
                "gensnmp.log","gentemp.log", "gentankdiy.log", "gengpioledblink.log",
                "gencthat.log"]
            for File in FilesToSend:
                LogFile = self.LogLocation + File
                if os.path.isfile(LogFile):
                    LogList.append(LogFile)
            return LogList
        except Exception as e1:
            return None

    #---------- Monitor::SendSupportInfo----------------------------------------
    def SendSupportInfo(self, SendLogs = True):

        try:
            if not self.EmailSendIsEnabled():
                self.LogError("Error in SendSupportInfo: send email is not enabled")
                return "Send Email is not enabled."

            msgbody = ""
            msgbody += self.printToString(self.ProcessDispatch(self.GetStartInfo(NoTile = True),""))
            if not self.bDisablePlatformStats:
                msgbody += self.printToString(self.ProcessDispatch({"Platform Stats" : self.GetPlatformStats()},""))
            msgbody += self.printToString(self.ProcessDispatch({"Comm Stats" : self.Controller.GetCommStatus()},""))

            msgbody += self.Controller.DisplayRegisters(AllRegs = True)

            # get data in JSON format
            msgbody += "\n" + self.GetSupportData()  + "\n"
            msgtitle = "Generator Monitor Log File Submission"
            if SendLogs == True:
                LogList = self.GetLogFileNames()
            else:
                msgtitle = "Generator Monitor Register Submission"
                LogList = None
            self.MessagePipe.SendMessage(msgtitle, msgbody , recipient = self.MaintainerAddress, files = LogList, msgtype = "error")
            return "Log files submitted"
        except Exception as e1:
            self.LogErrorLine("Error in SendSupportInfo: " + str(e1))

    #---------- process command from email and socket --------------------------
    def ProcessCommand(self, command, fromsocket = False):

        LocalError = False
        if isinstance(command, bytes):
            command = command.decode('utf-8')

        msgsubject = "Generator Command Response at " + self.SiteName
        if not fromsocket:
            msgbody = "\n"
        else:
            msgbody = ""

        if(len(command)) == 0:
            msgsubject = "Error in Generator Command (Lenght is zero)"
            msgbody += "Invalid GENERATOR command: zero length command."
            LocalError = True

        if not LocalError:
            if(not command.lower().startswith( 'generator:' )):
                msgsubject = "Error in Generator Command (command prefix)"
                msgbody += "Invalid GENERATOR command: all commands must be prefixed by \"generator: \""
                LocalError = True

        if LocalError:
            if not fromsocket:
                self.MessagePipe.SendMessage(msgsubject, msgbody, msgtype = "error")
                return ""       # ignored by email module
            else:
                msgbody += "EndOfMessage"
                return msgbody

        if command.lower().startswith('generator:'):
            command = command[len('generator:'):]

        CommandDict = {
            "registers"     : [self.Controller.DisplayRegisters,(False,), False],         # display registers
            "allregs"       : [self.Controller.DisplayRegisters, (True,), False],         # display registers
            "logs"          : [self.Controller.DisplayLogs, (True, False), False],
            "status"        : [self.Controller.DisplayStatus, (), False],                 # display decoded generator info
            "maint"         : [self.Controller.DisplayMaintenance, (), False],
            "monitor"       : [self.DisplayMonitor, (), False],
            "outage"        : [self.Controller.DisplayOutage, (), False],
            "settime"       : [self.StartTimeThread, (), False],                  # set time and date
            "setexercise"   : [self.Controller.SetGeneratorExerciseTime, (command.lower(),), False],
            "setquiet"      : [self.Controller.SetGeneratorQuietMode, ( command.lower(),), False],
            "setremote"     : [self.Controller.SetGeneratorRemoteCommand, (command.lower(),), False],
            "testcommand"   : [self.Controller.TestCommand, (command.lower(),), False],
            "network_status": [self.InternetConnected, (), False],
            "help"          : [self.DisplayHelp, (), False],                   # display help screen
            ## These commands are used by the web / socket interface only
            "power_log_json"    : [self.Controller.GetPowerHistory, (command.lower(),), True],
            "power_log_clear"   : [self.Controller.ClearPowerLog, (), True],
            "fuel_log_clear"    : [self.Controller.ClearFuelLog, (), True],
            "start_info_json"   : [self.GetStartInfo, (), True],
            "registers_json"    : [self.Controller.DisplayRegisters, (False, True), True],  # display registers
            "allregs_json"      : [self.Controller.DisplayRegisters, (True, True), True],   # display registers
            "logs_json"         : [self.Controller.DisplayLogs, (True, True), True],
            "status_json"       : [self.Controller.DisplayStatus, (True,), True],
            "status_num_json"   : [self.Controller.DisplayStatus, (True,True), True],
            "maint_json"        : [self.Controller.DisplayMaintenance, (True,), True],
            "maint_num_json"    : [self.Controller.DisplayMaintenance, (True,True), True],
            "monitor_json"      : [self.DisplayMonitor, (True,), True],
            "monitor_num_json"  : [self.DisplayMonitor, (True,True), True],
            "weather_json"      : [self.DisplayWeather, (True,), True],
            "outage_json"       : [self.Controller.DisplayOutage, (True,), True],
            "outage_num_json"   : [self.Controller.DisplayOutage, (True,True), True],
            "gui_status_json"   : [self.GetStatusForGUI, (), True],
            "get_maint_log_json": [self.Controller.GetMaintLogJSON, (), True],
            "add_maint_log"     : [self.Controller.AddEntryToMaintLog, (command,), True],    # Do not do command.lower() since this input is JSON
            "delete_row_maint_log" : [self.Controller.DeleteMaintLogRow, (command.lower(),), True],
            "edit_row_maint_log" : [self.Controller.EditMaintLogRow, (command,), True],    # Do not do command.lower() since this input is JSON
            "clear_maint_log"   : [self.Controller.ClearMaintLog, (), True],
            "getsitename"       : [self.GetSiteName, (), True],
            "getbase"           : [self.Controller.GetBaseStatus, (), True],    #  (UI changes color based on exercise, running , ready status)
            "gethealth"         : [self.GetSystemHealth, (), True],
            "getregvalue"       : [self.Controller.GetRegValue, (command.lower(),), True],     # only used for debug purposes, read a cached register value
            "readregvalue"      : [self.Controller.ReadRegValue, (command.lower(),), True],    # only used for debug purposes, Read Register Non Cached
            "writeregvalue"     : [self.Controller.WriteRegValue, (command.lower(),), True],    # only used for debug purposes, Read Register Non Cached
            "getdebug"          : [self.GetDeadThreadName, (), True],           # only used for debug purposes. If a thread crashes it tells you the thread name
            "sendregisters"     : [self.SendSupportInfo, (False,), True],
            "sendlogfiles"      : [self.SendSupportInfo, (True,), True],
            "support_data_json" : [self.GetSupportData, (), True],
            "set_tank_data"     : [self.Controller.SetExternalTankData, (command,), True],
            "set_temp_data"     : [self.Controller.SetExternalTemperatureData, (command,), True],
            "set_power_data"    : [self.Controller.SetExternalCTData, (command,), True]
        }

        CommandList = command.split(' ')

        ValidCommand = False
        try:
            for item in CommandList:
                if not len(item):
                    continue
                item = item.strip()
                LookUp = item
                if "=" in item:
                    BaseCmd = item.split('=')
                    LookUp = BaseCmd[0]
                # check if we disallow write commands via email
                if self.ReadOnlyEmailCommands and not fromsocket and LookUp in ["settime", "setexercise", "setquiet", "setremote"]:
                    continue

                ExecList = CommandDict.get(LookUp.lower(),None)
                if ExecList == None:
                    continue
                if ExecList[0] == None:
                    continue
                if not fromsocket and ExecList[2]:
                    continue
                # Execut Command
                ReturnMessage = ExecList[0](*ExecList[1])

                ValidCommand = True

                if LookUp.lower().endswith("_json") and not isinstance(ReturnMessage, str):
                    msgbody += json.dumps(ReturnMessage, sort_keys=False)
                else:
                    msgbody += ReturnMessage

                if not fromsocket:
                    msgbody += "\n"
        except Exception as e1:
            self.LogErrorLine("Error Processing Commands: " + command + ": "+ str(e1))

        if not ValidCommand:
            msgbody += "No valid command recognized."
        if not fromsocket:
            self.MessagePipe.SendMessage(msgsubject, msgbody, msgtype = "warn")
            return ""       # ignored by email module
        else:
            msgbody += "EndOfMessage"
            return msgbody

    #------------ Monitor::DisplayHelp -----------------------------------------
    def DisplayHelp(self):

        outstring = ""
        outstring += "Help:\n"
        outstring += self.printToString("\nCommands:")
        outstring += self.printToString("   status      - display engine and line information")
        outstring += self.printToString("   maint       - display maintenance and service information")
        outstring += self.printToString("   outage      - display current and last outage (since program launched)")
        outstring += self.printToString("                       info, also shows utility min and max values")
        outstring += self.printToString("   monitor     - display communication statistics and monitor health")
        outstring += self.printToString("   logs        - display all alarm, on/off, and maintenance logs")
        outstring += self.printToString("   registers   - display contents of registers being monitored")
        outstring += self.printToString("   settime     - set generator time to system time")
        outstring += self.printToString("   setexercise - set the exercise time of the generator. ")
        outstring += self.printToString("                      i.e. setexercise=Monday,13:30,Weekly")
        outstring += self.printToString("                  if Enhanced Exercise Frequency is supported by")
        outstring += self.printToString("                  your generator:")
        outstring += self.printToString("                      i.e. setexercise=Monday,13:30,BiWeekly")
        outstring += self.printToString("                      i.e. setexercise=15,13:30,Monthly")
        outstring += self.printToString("   setquiet    - enable or disable exercise quiet mode, ")
        outstring += self.printToString("                      i.e.  setquiet=on or setquiet=off")
        outstring += self.printToString("   setremote   - issue remote command. format is setremote=command, ")
        outstring += self.printToString("                      where command is start, stop, starttransfer,")
        outstring += self.printToString("                      startexercise. i.e. setremote=start")
        outstring += self.printToString("   help        - Display help on commands")
        outstring += self.printToString("   exit        - Exit this program")
        outstring += self.printToString("   sendlogfiles - Send log files to the developer if outbound email is setup.")
        outstring += self.printToString("                      is setup.")

        outstring += self.printToString("\n")

        outstring += self.printToString("To clear the Alarm/Warning message, press OFF on the control panel keypad")
        outstring += self.printToString("followed by the ENTER key. To access Dealer Menu on the Evolution")
        outstring += self.printToString("controller, from the top menu selection (SYSTEM, DATE/TIME,BATTERY, SUB-MENUS)")
        outstring += self.printToString("enter UP UP ESC DOWN UP ESC UP, then go to the dealer menu and press enter.")
        outstring += self.printToString("For liquid cooled models a level 2 dealer code can be entered, ESC UP UP DOWN")
        outstring += self.printToString("DOWN ESC ESC, then navigate to the dealer menu and press enter.")
        outstring += self.printToString("Passcode for Nexus controller is ESC, UP, UP ESC, DOWN, UP, ESC, UP, UP, ENTER.")
        outstring += self.printToString("\n")

        return outstring

    #------------ Monitor::GetProgramRunTime -----------------------------------
    def GetProgramRunTime(self):
        try:
            ProgramRunTime = datetime.datetime.now() - self.ProgramStartTime
            outstr = str(ProgramRunTime).split(".")[0]  # remove microseconds from string
            return self.ProgramName + " running for " + outstr + "."
        except Exception as e1:
            self.LogErrorLine("Error in GetProgramRunTime:" + str(e1))
            return "Unknown"
    #------------ Monitor::GetWeatherData --------------------------------------
    def GetWeatherData(self, ForUI = False):

        if self.MyWeather == None:
            return None

        ReturnData = self.MyWeather.GetWeather(minimum = self.WeatherMinimum, ForUI = ForUI)

        if not len(ReturnData):
            return None
        return ReturnData

    #------------ Monitor::GetUserDefinedData ----------------------------------
    # this assumes one json object, the file can be formatted (i.e. on multiple
    # lines) or can be on a single line
    def GetUserDefinedData(self):

        try:
            FileName = os.path.join(self.UserDefinedDataPath, "userdefined.json")

            if not os.path.isfile(FileName):
                return None

            if os.path.getsize(FileName) == 0:
                return None

            with open(FileName) as f:
                data = json.load(f,object_pairs_hook=collections.OrderedDict)
            return data
        except Exception as e1:
            self.LogErrorLine("Error in GetUserDefinedData: " + str(e1))
        return None

    #------------ Monitor::DisplayWeather --------------------------------------
    def DisplayWeather(self, DictOut = False):

        WeatherData = collections.OrderedDict()

        try:
            ReturnData = self.GetWeatherData()
            if not ReturnData == None and len(ReturnData):
                WeatherData["Weather"] = ReturnData

            if not DictOut:
                return self.printToString(self.ProcessDispatch(WeatherData,""))
        except Exception as e1:
            self.LogErrorLine("Error in DisplayWeather: " + str(e1))

        return WeatherData

    #------------ Monitor::DisplayMonitor --------------------------------------
    def DisplayMonitor(self, DictOut = False, JSONNum = False):

        try:
            Monitor = collections.OrderedDict()
            MonitorData = []
            Monitor["Monitor"] = MonitorData
            GenMonStats = []
            SerialStats = []
            MonitorData.append({"Generator Monitor Stats" : GenMonStats})
            MonitorData.append({"Communication Stats" : self.Controller.GetCommStatus()})

            GenMonStats.append({"Monitor Health" :  self.GetSystemHealth()})
            GenMonStats.append({"Controller" : self.Controller.GetController(Actual = False)})

            GenMonStats.append({"Run time" : self.GetProgramRunTime()})
            if self.Controller.PowerMeterIsSupported():
                GenMonStats.append({"Power log file size" : self.Controller.GetPowerLogFileDetails()})
            GenMonStats.append({"Generator Monitor Version" : ProgramDefaults.GENMON_VERSION})
            GenMonStats.append({"Update Available" : "Yes" if self.UpdateAvailable else "No"})

            if not self.bDisablePlatformStats:
                PlatformStats = self.GetPlatformStats()
                if not PlatformStats == None:
                    MonitorData.append({"Platform Stats" : PlatformStats})

            WeatherData = self.GetWeatherData()
            if not WeatherData == None and len(WeatherData):
                MonitorData.append({"Weather" : WeatherData})

            UserData = self.GetUserDefinedData()
            if UserData != None and len(UserData):
                try:
                    MonitorData.append({"External Data" : UserData})
                except Exception as e1:
                    self.LogErrorLine("Error in appending user data: " + str(e1))
            if not DictOut:
                return self.printToString(self.ProcessDispatch(Monitor,""))
        except Exception as e1:
            self.LogErrorLine("Error in DisplayMonitor: " + str(e1))
        return Monitor

    #------------ Monitor::GetStartInfo-----------------------------------------
    def GetStartInfo(self, NoTile = False):

        StartInfo = collections.OrderedDict()
        StartInfo["version"] = ProgramDefaults.GENMON_VERSION
        StartInfo["sitename"] = self.SiteName
        StartInfo["python"] = str(sys.version_info.major) + "." + str(sys.version_info.minor)
        StartInfo["platform"] = str(sys.platform)
        try:
            import platform

            StartInfo["python"] = str(platform.python_version())

            Platform = MyPlatform(log = self.log, usemetric = True)
            StartInfo["os_bits"] = Platform.PlatformBitDepth()

        except Exception as e1:
            self.LogErrorLine("Error in GetStartInfo: " + str(e1))
            pass
        try:
            import time
            if self.is_dst:
                StartInfo["zone"] = time.tzname[1]
            else:
                StartInfo["zone"] = time.tzname[0]
        except:
            pass
        ControllerStartInfo = self.Controller.GetStartInfo(NoTile = NoTile)
        StartInfo = self.MergeDicts(StartInfo, ControllerStartInfo)
        return StartInfo

    #------------ Monitor::GetStatusForGUI -------------------------------------
    def GetStatusForGUI(self):

        Status = {}

        Status["SystemHealth"] = self.GetSystemHealth()
        Status["UnsentFeedback"] = str(os.path.isfile(self.FeedbackLogFile))

        if not self.bDisablePlatformStats:
            PlatformStats = self.GetPlatformStats(usemetric = True)
            if not PlatformStats == None and len(PlatformStats):
                Status["PlatformStats"] = PlatformStats
        WeatherData = self.GetWeatherData(ForUI = True)
        if not WeatherData == None and len(WeatherData):
            Status["Weather"] = WeatherData
        # Monitor Time
        Status["MonitorTime"] = datetime.datetime.now().strftime("%m/%d/%Y %H:%M")
        # Engine run hours
        Status["RunHours"] = self.Controller.GetRunHours()
        ReturnDict = self.MergeDicts(Status, self.Controller.GetStatusForGUI())

        return ReturnDict

    #-------------Monitor::GetSystemHealth--------------------------------------
    #   returns the health of the monitor program
    def GetSystemHealth(self):

        outstr = ""
        if not self.Controller.InitComplete:
            outstr += "System Initializing. "
        if not self.AreThreadsAlive():
            outstr += " Threads are dead. "
        if  not self.CommunicationsActive:
            outstr += " Not receiving data. "
        if not self.LogFileIsOK():
            outstr += " Log file is reporting errors."

        if len(outstr) == 0:
            outstr = "OK"
        return outstr

    #----------  Monitor::StartTimeThread---------------------------------------
    def StartTimeThread(self):

        # This is done is a separate thread as not to block any return email processing
        # since we attempt to sync with generator time
        MyThread(self.Controller.SetGeneratorTimeDate, Name = "SetTimeThread")
        return "Time Set: Command Sent\n"

    #----------  Monitor::TimeSyncThread----------------------------------------
    def TimeSyncThread(self):

        self.bDST = self.is_dst()   # set initial DST state

        time.sleep(0.25)
        while True:
            if self.WaitForExit("TimeSyncThread", 1):  # ten min
                return
            if self.Controller.InitComplete:
                break

        # if we are not always syncing, then set the time once
        if not self.bSyncTime:
            self.StartTimeThread()

        while True:

            if self.bSyncDST:
                if self.bDST != self.is_dst():  # has DST changed?
                    self.bDST = self.is_dst()   # update Flag
                    # time changed so some comm stats may be off
                    self.Controller.ResetCommStats()
                    # set new time
                    self.StartTimeThread()           # start settime thread
                    self.MessagePipe.SendMessage("Generator Time Update at " + self.SiteName, "Time updated due to daylight savings time change", msgtype = "info")
                    self.LogError("DST change")

            if self.bSyncTime:
                # update gen time
                self.StartTimeThread()

            if self.WaitForExit("TimeSyncThread", 60 * 60):  # 1 hour
                return

    #----------  Monitor::is_dst------------------------------------------------
    def is_dst(self):
        #Determine whether or not Daylight Savings Time (DST) is currently in effect
        t = time.localtime()
        isdst = t.tm_isdst
        return (isdst != 0)

    #----------  Monitor::ComWatchDog-------------------------------------------
    #----------  monitors receive data status to make sure we are still communicating
    def ComWatchDog(self):

        self.CommunicationsActive = False
        time.sleep(0.25)

        NoticeSent = False
        LastActiveTime = datetime.datetime.now()
        counter = 0
        while True:
            if self.WaitForExit("ComWatchDog", 1):
                return
            if counter > 30:
                self.LogError("WARNING: Initilization not complete after 30 seconds, possible communication failure. Check cabling.")
                break
            counter += 1
            if self.Controller.InitComplete:
                break

        if self.Controller.ModBus.UseTCP:
            WatchDogPollTime = 8.0
        else:
            WatchDogPollTime = 2.0

        try:
            WatchDogPollTime += float(self.Controller.ModBus.ModBusPacketTimoutMS / 1000)
        except:
            self.LogErrorLine("Error in ComWatchDog: " + str(e1))

        WatchDogPollTime += self.AdditionalWatchdogTime

        while True:
            try:
                # check for software update
                self.CheckSoftwareUpdate()
                self.CommunicationsActive = self.Controller.ComminicationsIsActive()

                if self.CommunicationsActive:
                    LastActiveTime = datetime.datetime.now()
                    if NoticeSent:
                        NoticeSent = False
                        msgbody = "Generator Monitor communications with the controller has been restored at " + self.SiteName
                        msgbody += "\n" + self.DisplayMonitor()
                        self.MessagePipe.SendMessage("Generator Monitor Communication Restored at " + self.SiteName, msgbody , msgtype = "info")
                else:
                    if self.GetDeltaTimeMinutes(datetime.datetime.now() - LastActiveTime) > (1 + self.AdditionalWatchdogTime) :
                        if not NoticeSent:
                            NoticeSent = True
                            msgbody = "Generator Monitor is not communicating with the controller at " + self.SiteName
                            msgbody += "\n" + self.DisplayMonitor()
                            self.MessagePipe.SendMessage("Generator Monitor Communication WARNING at " + self.SiteName, msgbody , msgtype = "error")

            except Exception as e1:
                self.LogErrorLine("Error in ComWatchDog: " + str(e1))

            if self.WaitForExit("ComWatchDog", WatchDogPollTime):
                return
    #---------- Monitor::CheckSoftwareUpdate------------------------------------
    def CheckSoftwareUpdate(self):

        if not self.UpdateCheck:
            return
        try:
            if self.GetDeltaTimeMinutes(datetime.datetime.now() - self.LastSofwareUpdateCheck) > 1440 :     # check every day
                self.LastSofwareUpdateCheck = datetime.datetime.now()
                # Do the check
                try:
                    url = "https://raw.githubusercontent.com/jgyates/genmon/master/genmonlib/program_defaults.py"
                    try:
                        # For Python 3.0 and later
                        from urllib.request import urlopen
                    except ImportError:
                        # Fall back to Python 2's urllib2
                        from urllib2 import urlopen

                    data = urlopen(url).read(4000) # read only first 4000 chars
                    data = data.decode('ascii')
                    data = data.split("\n") # then split it into lines

                    for line in data:

                        if 'GENMON_VERSION = "V' in line:
                            import re
                            quoted = re.compile('"([^"]*)"')
                            for value in quoted.findall(line):
                                if value != ProgramDefaults.GENMON_VERSION:
                                    # Update Available
                                    title = self.ProgramName + " Software Update " + value + " is available for site " + self.SiteName
                                    msgbody = "\nA software update is available for the " + self.ProgramName + ". The new version (" + value + ") can be updated on the About page of the web interface. The current version installed is " + ProgramDefaults.GENMON_VERSION + ". You can disable this email from being sent on the Settings page."
                                    if len(self.UserURL):
                                        msgbody += "\n\nWeb Interface URL: " + self.UserURL
                                    msgbody += "\n\nChange Log: https://raw.githubusercontent.com/jgyates/genmon/master/changelog.md"
                                    self.MessagePipe.SendMessage(title , msgbody, msgtype = "info", onlyonce = True)
                                    self.UpdateAvailable = True

                except Exception as e1:
                    self.LogErrorLine("Error checking for software update: " + str(e1))

        except Exception as e1:
            self.LogErrorLine("Error in CheckSoftwareUpdate: " + str(e1))

    #---------- Monitor::LogFileIsOK--------------------------------------------
    def LogFileIsOK(self):

        try:
            if not self.Controller.InitComplete:
                return True

            LogFile = os.path.join(self.LogLocation, "genmon.log")

            LogFileSize = os.path.getsize(LogFile)
            if LogFileSize <= self.LastLogFileSize:     # log is unchanged or has rotated
                self.LastLogFileSize = LogFileSize
                self.NumberOfLogSizeErrors = 0
                return True

            LogFileSizeDiff = LogFileSize - self.LastLogFileSize
            self.LastLogFileSize = LogFileSize
            if LogFileSizeDiff >= 100:
                self.NumberOfLogSizeErrors += 1
                if self.NumberOfLogSizeErrors > 3:
                    return False
            else:
                self.NumberOfLogSizeErrors = 0
            return True
        except Exception as e1:
            self.LogErrorLine("Error in LogFileIsOK: " + str(e1))
        return True

    #----------  Monitor::SocketWorkThread-------------------------------------
    #  This thread spawns for each connection established by a client
    #  in InterfaceServerThread
    def SocketWorkThread(self, conn):

        try:

            statusstr = ""
            if self.Controller == None:
                outstr = "WARNING: System Initializing"
                conn.sendall(outstr.encode())
            else:
                if self.Controller.SystemInAlarm():
                    statusstr += "CRITICAL: System in alarm! "
                HealthStr = self.GetSystemHealth()
                if HealthStr != "OK":
                    statusstr += "WARNING: " + HealthStr
                if statusstr == "":
                    statusstr = "OK "

                outstr = statusstr + ": "+ self.Controller.GetOneLineStatus()
                conn.sendall(outstr.encode())

            while True:
                try:
                    data = conn.recv(1024)
                    if len(data):
                        if self.Controller == None:
                            outstr = "Retry, System Initializing"
                        else:
                            outstr = self.ProcessCommand(data, True)
                        conn.sendall(outstr.encode("utf-8"))
                    else:
                        # socket closed remotely
                        break
                except socket.timeout:
                    if self.IsStopping:
                        break
                    continue
                except socket.error as msg:
                    try:
                        self.ConnectionList.remove(conn)
                        conn.close()
                    except:
                        pass
                    break

        except socket.error as msg:
            self.LogError("Error in SocketWorkThread: " + str(msg))
            pass

        try:
            self.ConnectionList.remove(conn)
            conn.close()
        except:
            pass
        # end SocketWorkThread

    #----------  interface for heartbeat server thread -------------------------
    def InterfaceServerThread(self):

        #create an INET, STREAMing socket
        self.ServerSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # set some socket options so we can resuse the port
        self.ServerSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.ServerSocket.settimeout(.5)
        #bind the socket to a host, and a port
        self.ServerSocket.bind(('', self.ServerSocketPort))
        #become a server socket
        self.ServerSocket.listen(5)

        #wait to accept a connection - blocking call
        while True:
            try:
                conn, addr = self.ServerSocket.accept()
                #self.LogError('Connected with ' + addr[0] + ':' + str(addr[1]))
                conn.settimeout(0.5)
                self.ConnectionList.append(conn)
                SocketThread = threading.Thread(target=self.SocketWorkThread, args = (conn,), name = "SocketWorkThread")
                SocketThread.daemon = True
                SocketThread.start()       # start server thread
            except socket.timeout:
                if self.IsStopping:
                    break
                continue
            except Exception as e1:
                if self.IsStopping:
                    break
                self.LogErrorLine("Exception in InterfaceServerThread" + str(e1))
                if self.WaitForExit("InterfaceServerThread", 0.5 ):
                    break
                continue

        if self.ServerSocket != None:
            if len(self.ConnectionList):
                try:
                    self.ServerSocket.shutdown(socket.SHUT_RDWR)
                except:
                    pass
            self.ServerSocket.close()
            self.ServerSocket = None
        #

    # ----------Monitor::SignalClose--------------------------------------------
    def SignalClose(self, signum, frame):

        self.Close()
        sys.exit(1)

    #---------------------Monitor::Close----------------------------------------
    def Close(self):

        # we dont really care about the errors that may be generated on shutdown
        try:
            self.IsStopping = True

            try:
                if self.MyWeather != None:
                    self.MyWeather.Close()
            except:
                pass

            try:
                if self.bSyncDST or self.bSyncTime:
                    self.KillThread("TimeSyncThread")
            except:
                pass

            try:
                self.KillThread("ComWatchDog")
            except:
                pass

            try:
                if not self.Controller == None:
                    self.Controller.Close()
            except:
                pass

            #
            try:
                self.mail.Close()
            except:
                pass
            try:
                for item in self.ConnectionList:
                    try:
                        item.close()
                    except:
                        continue
                    self.ConnectionList.remove(item)
            except:
                pass

            try:
                if(self.ServerSocket != None):
                    self.ServerSocket.shutdown(socket.SHUT_RDWR)
                    self.ServerSocket.close()
                self.KillThread("InterfaceServerThread")
            except:
                pass

            try:
                self.FeedbackPipe.Close()
            except:
                pass
            try:
                self.MessagePipe.Close()
            except:
                pass

            # Tell any remaining threads to stop
            for name, object in self.Threads.items():
                try:
                    if self.Threads[name].IsAlive():
                        self.Threads[name].Stop()
                except Exception as e1:
                    self.LogErrorLine("Error killing thread in Monitor Close: " + name + ":" + str(e1))

        except Exception as e1:
            self.LogErrorLine("Error Closing Monitor: " + str(e1))

        with self.CriticalLock:
            self.LogError("Generator Monitor Shutdown")

        try:

            self.ProgramComplete = True
            sys.exit(0)
        except:
            pass

#------------------- Command-line interface for monitor ------------------------
if __name__=='__main__': #

    try:
        ConfigFilePath = ProgramDefaults.ConfPath
        opts, args = getopt.getopt(sys.argv[1:],"c:",["configpath="])
    except getopt.GetoptError:
        console.error("Invalid command line argument.")
        sys.exit(2)

    for opt, arg in opts:


# encoding: utf-8

import sys
import logging
import re

import sqlalchemy.engine.url as sa_url

import ckan.plugins as p
import ckan.logic as logic
import ckan.model as model
from ckan.common import config
import ckanext.timeseries.logic.action as action
import ckanext.timeseries.logic.auth as auth
import ckanext.timeseries.db as db
import ckanext.timeseries.interfaces as interfaces
import ckanext.timeseries.helpers as datastore_helpers
from ckanext.timeseries.helpers import literal_string

log = logging.getLogger(__name__)
_get_or_bust = logic.get_or_bust

DEFAULT_FORMATS = []

ValidationError = p.toolkit.ValidationError

def _is_legacy_mode(config):
    '''
        Decides if the DataStore should run on legacy mode

        Returns True if `ckan.datastore.read_url` is not set in the provided
        config object or CKAN is running on Postgres < 9.x
    '''
    write_url = config.get('ckan.datastore.write_url')

    engine = db._get_engine({'connection_url': write_url})
    connection = engine.connect()

    return (not config.get('ckan.datastore.read_url') or
            not db._pg_version_is_at_least(connection, '9.0'))

class DatastoreException(Exception):
    pass

class TimeseriesPlugin(p.SingletonPlugin):
    p.implements(p.IConfigurable, inherit=True)
    # p.implements(p.IConfigurer)
    p.implements(p.IActions)
    p.implements(p.IAuthFunctions)
    p.implements(p.IResourceUrlChange)
    p.implements(p.IDomainObjectModification, inherit=True)
    p.implements(p.IRoutes, inherit=True)
    p.implements(p.IResourceController, inherit=True)
    p.implements(interfaces.ITimeseries, inherit=True)

    legacy_mode = False
    resource_show_action = None

    def __new__(cls, *args, **kwargs):
        itimeseries_extensions = p.PluginImplementations(interfaces.ITimeseries)
        itimeseries_extensions = itimeseries_extensions.extensions()

        if itimeseries_extensions and itimeseries_extensions[0].__class__ != cls:
            msg = ('The "datastore" plugin must be the first ITimeseries '
                   'plugin loaded. Change the order it is loaded in '
                   '"ckan.plugins" in your CKAN .ini file and try again.')
            raise DatastoreException(msg)

        return super(cls, cls).__new__(cls, *args, **kwargs)

    def update_config(self, config):
        p.toolkit.add_template_directory(config, 'templates')

    def configure(self, config):
        self.config = config
        # check for ckan.datastore.write_url and ckan.datastore.read_url
        if (not 'ckan.datastore.write_url' in config):
            error_msg = 'ckan.datastore.write_url not found in config'
            raise DatastoreException(error_msg)

        # Legacy mode means that we have no read url. Consequently sql search is not
        # available and permissions do not have to be changed. In legacy mode, the
        # datastore runs on PG prior to 9.0 (for example 8.4).
        self.legacy_mode = _is_legacy_mode(self.config)

        # Check whether users have disabled datastore_search_sql
        self.enable_sql_search = p.toolkit.asbool(
            self.config.get('ckan.datastore.sqlsearch.enabled', True))

        datapusher_formats = config.get('datapusher.formats', '').split()
        self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS

        # Check whether we are running one of the paster commands which means
        # that we should ignore the following tests.
        if sys.argv[0].split('/')[-1] == 'paster' and 'timeseries' in sys.argv[1:]:
            log.warn('Omitting permission checks because you are '
                     'running paster commands.')
            return

        self.ckan_url = self.config['sqlalchemy.url']
        self.write_url = self.config['ckan.datastore.write_url']
        if self.legacy_mode:
            self.read_url = self.write_url
            log.warn('Legacy mode active. '
                     'The sql search will not be available.')
        else:
            self.read_url = self.config['ckan.datastore.read_url']

        self.read_engine = db._get_engine(
            {'connection_url': self.read_url})
        if not model.engine_is_pg(self.read_engine):
            log.warn('We detected that you do not use a PostgreSQL '
                     'database. The DataStore will NOT work and DataStore '
                     'tests will be skipped.')
            return

        if self._is_read_only_database():
            log.warn('We detected that CKAN is running on a read '
                     'only database. Permission checks and the creation '
                     'of _table_metadata are skipped.')
        else:
            self._check_urls_and_permissions()
            self._create_alias_table() # create another _table_metadata_ts for resource size

    def notify(self, entity, operation=None):
        if not isinstance(entity, model.Package) or self.legacy_mode:
            return
        # if a resource is new, it cannot have a datastore resource, yet
        if operation == model.domain_object.DomainObjectOperation.changed:
            context = {'model': model, 'ignore_auth': True}
            if entity.private:
                func = p.toolkit.get_action('datastore_ts_make_private')
            else:
                func = p.toolkit.get_action('datastore_ts_make_public')
            for resource in entity.resources:
                try:
                    func(context, {
                        'connection_url': self.write_url,
                        'resource_id': resource.id})
                except p.toolkit.ObjectNotFound:
                    pass

    def _log_or_raise(self, message):
        if self.config.get('debug'):
            log.critical(message)
        else:
            raise DatastoreException(message)

    def _check_urls_and_permissions(self):
        # Make sure that the right permissions are set
        # so that no harmful queries can be made

        if self._same_ckan_and_datastore_db():
            self._log_or_raise('CKAN and DataStore database '
                               'cannot be the same.')

        # in legacy mode, the read and write url are the same (both write url)
        # consequently the same url check and and write privilege check
        # don't make sense
        if not self.legacy_mode:
            if self._same_read_and_write_url():
                self._log_or_raise('The write and read-only database '
                                   'connection urls are the same.')

            if not self._read_connection_has_correct_privileges():
                self._log_or_raise('The read-only user has write privileges.')

    def _is_read_only_database(self):
        ''' Returns True if no connection has CREATE privileges on the public
        schema. This is the case if replication is enabled.'''
        for url in [self.ckan_url, self.write_url, self.read_url]:
            connection = db._get_engine({'connection_url': url}).connect()
            try:
                sql = u"SELECT has_schema_privilege('public', 'CREATE')"
                is_writable = connection.execute(sql).first()[0]
            finally:
                connection.close()
            if is_writable:
                return False
        return True

    def _same_ckan_and_datastore_db(self):
        '''Returns True if the CKAN and DataStore db are the same'''
        return self._get_db_from_url(self.ckan_url) == self._get_db_from_url(self.read_url)

    def _get_db_from_url(self, url):
        db_url = sa_url.make_url(url)
        return db_url.host, db_url.port, db_url.database

    def _same_read_and_write_url(self):
        return self.write_url == self.read_url

    def _read_connection_has_correct_privileges(self):
        ''' Returns True if the right permissions are set for the read
        only user. A table is created by the write user to test the
        read only user.
        '''
        write_connection = db._get_engine(
            {'connection_url': self.write_url}).connect()
        read_connection_user = sa_url.make_url(self.read_url).username

        drop_foo_sql = u'DROP TABLE IF EXISTS _foo'

        write_connection.execute(drop_foo_sql)

        try:
            write_connection.execute(u'CREATE TEMP TABLE _foo ()')
            for privilege in ['INSERT', 'UPDATE', 'DELETE']:
                test_privilege_sql = u"SELECT has_table_privilege(%s, '_foo', %s)"
                have_privilege = write_connection.execute(
                    test_privilege_sql, (read_connection_user, privilege)).first()[0]
                if have_privilege:
                    return False
        finally:
            write_connection.execute(drop_foo_sql)
            write_connection.close()
        return True

    def get_actions(self):
        actions = {'datastore_ts_create': action.datastore_create,
                   'datastore_ts_upsert': action.datastore_upsert,
                   'datastore_ts_delete': action.datastore_delete,
                   'datastore_ts_search': action.datastore_search,
                   'datastore_ts_info': action.datastore_info,
                  }
        if not self.legacy_mode:
            if self.enable_sql_search:
                # Only enable search_sql if the config does not disable it
                actions.update({'datastore_ts_search_sql':
                                 action.datastore_search_sql})
            actions.update({
                'datastore_ts_make_private': action.datastore_make_private,
                'datastore_ts_make_public': action.datastore_make_public})
        return actions

    def get_auth_functions(self):
        return {'datastore_ts_create': auth.datastore_create,
                'datastore_ts_upsert': auth.datastore_upsert,
                'datastore_ts_delete': auth.datastore_delete,
                'datastore_ts_info': auth.datastore_info,
                'datastore_ts_search': auth.datastore_search,
                'datastore_ts_search_sql': auth.datastore_search_sql,
                'datastore_ts_change_permissions': auth.datastore_change_permissions}

    def before_map(self, m):
        m.connect('/datastore_ts/dump/{resource_id}',
                  controller='ckanext.timeseries.controller.datastore_ts:TimeseriesController',
                  action='dump')
        return m

    def before_show(self, resource_dict):
        # Modify the resource url of datastore resources so that
        # they link to the datastore dumps.

        # if resource_dict.get('url_type') == 'datastore':
        connection = db._get_engine(
                {'connection_url': self.write_url}).connect()

        if db._is_timeseries({"connection": connection},resource_dict['id']):
            resource_dict['url'] = p.toolkit.url_for(
                controller='ckanext.timeseries.controller.datastore_ts:TimeseriesController',
                action='dump', resource_id=resource_dict['id'],
                qualified=True)

        if 'datastore_active' not in resource_dict:
            resource_dict[u'datastore_active'] = False

        return resource_dict

    def datastore_validate(self, context, data_dict, fields_types):
        column_names = fields_types.keys()
        fields = data_dict.get('fields')
        if fields:
            data_dict['fields'] = list(set(fields) - set(column_names))

        filters = data_dict.get('filters', {})
        for key in filters.keys():
            if key in fields_types:
                del filters[key]

        q = data_dict.get('q')
        if q:
            if isinstance(q, basestring):
                del data_dict['q']
            elif isinstance(q, dict):
                for key in q.keys():
                    if key in fields_types and isinstance(q[key], basestring):
                        del q[key]

        language = data_dict.get('language')
        if language:
            if isinstance(language, basestring):
                del data_dict['language']

        plain = data_dict.get('plain')
        if plain:
            if isinstance(plain, bool):
                del data_dict['plain']

        distinct = data_dict.get('distinct')
        if distinct:
            if isinstance(distinct, bool):
                del data_dict['distinct']

        sort_clauses = data_dict.get('sort')
        if sort_clauses:
            invalid_clauses = [c for c in sort_clauses
                               if not self._parse_sort_clause(c, fields_types)]
            data_dict['sort'] = invalid_clauses

        limit = data_dict.get('limit')
        if limit:
            is_positive_int = datastore_helpers.validate_int(limit,
                                                             non_negative=True)
            is_all = isinstance(limit, basestring) and limit.lower() == 'all'
            if is_positive_int or is_all:
                del data_dict['limit']

        offset = data_dict.get('offset')
        if offset:
            is_positive_int = datastore_helpers.validate_int(offset,
                                                             non_negative=True)
            if is_positive_int:
                del data_dict['offset']

        # Nam Giang
        fromtime = data_dict.get('fromtime')
        if fromtime:
            if isinstance(fromtime, basestring):
                del data_dict['fromtime']

        totime = data_dict.get('totime')
        if totime:
            if isinstance(totime, basestring):
                del data_dict['totime']
        # end Nam Giang

        return data_dict

    def _parse_sort_clause(self, clause, fields_types):
        clause_match = re.match(u'^(.+?)( +(asc|desc) *)?$', clause, re.I)

        if not clause_match:
            return False

        field = clause_match.group(1)
        if field[0] == field[-1] == u'"':
            field = field[1:-1]
        sort = (clause_match.group(3) or u'asc').lower()

        if field not in fields_types:
            return False

        return field, sort

    def datastore_delete(self, context, data_dict, fields_types, query_dict):
        query_dict['where'] += self._where(data_dict, fields_types)
        return query_dict

    def datastore_search(self, context, data_dict, fields_types, query_dict):
        fields = data_dict.get('fields')

        if fields:
            field_ids = datastore_helpers.get_list(fields)
        else:
            field_ids = fields_types.keys()

        ts_query, rank_column = self._textsearch_query(data_dict)
        limit = data_dict.get('limit', 100)
        offset = data_dict.get('offset', 0)

        sort = self._sort(data_dict, fields_types)
        where = self._where(data_dict, fields_types)

        select_cols = [u'"{0}"'.format(field_id) for field_id in field_ids] +\
                      [u'count(*) over() as "_full_count" %s' % rank_column]

        query_dict['distinct'] = data_dict.get('distinct', False)
        query_dict['select'] += select_cols
        query_dict['ts_query'] = ts_query
        query_dict['sort'] += sort
        query_dict['where'] += where
        query_dict['limit'] = limit
        query_dict['offset'] = offset

        return query_dict

    def _where(self, data_dict, fields_types):
        filters = data_dict.get('filters', {})
        clauses = []

        for field, value in filters.iteritems():
            if field not in fields_types:
                continue
            field_array_type = self._is_array_type(fields_types[field])
            if isinstance(value, list) and not field_array_type:
                clause_str = (u'"{0}" in ({1})'.format(field,
                              ','.join(['%s'] * len(value))))
                clause = (clause_str,) + tuple(value)
            else:
                clause = (u'"{0}" = %s'.format(field), value)
            clauses.append(clause)

        # Nam Giang
        fromtime = data_dict.get('fromtime', None)
        totime = data_dict.get('totime', None)

        try:
            if fromtime is not None:
                fromtime_postgres = datastore_helpers.timestamp_from_string(fromtime)
                clause = (u'"{0}" >= %s'.format("_autogen_timestamp"), fromtime_postgres)
                clauses.append(clause)
                # print(clause)
            if totime is not None:
                totime_postgres = datastore_helpers.timestamp_from_string(totime)
                clause = (u'"{0}" <= %s'.format("_autogen_timestamp"), totime_postgres)
                clauses.append(clause)
                # print(clause)
        except:
            raise ValidationError({
                    'fields': [u'input time string not valid, fromTime: {}, toTime: {}'.format(fromtime, totime)]
                })
        # end Nam Giang

        # add full-text search where clause
        q = data_dict.get('q')
        if q:
            if isinstance(q, basestring):
                ts_query_alias = self._ts_query_alias()
                clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
                clauses.append((clause_str,))
            elif isinstance(q, dict):
                lang = self._fts_lang(data_dict.get('lang'))
                for field, value in q.iteritems():
                    if field not in fields_types:
                        continue
                    query_field = self._ts_query_alias(field)

                    ftyp = fields_types[field]
                    if not datastore_helpers.should_fts_index_field_type(ftyp):
                        clause_str = u'_full_text @@ {0}'.format(query_field)
                        clauses.append((clause_str,))

                    clause_str = (u'to_tsvector({0}, cast("{1}" as text)) '
                                  u'@@ {2}').format(literal_string(lang),
                                                    field, query_field)
                    clauses.append((clause_str,))

        return clauses

    def _is_array_type(self, field_type):
        return field_type.startswith('_')

    def _sort(self, data_dict, fields_types):
        sort = data_dict.get('sort')
        if not sort:
            q = data_dict.get('q')
            if q: