<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2010 <NAME>
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version. 
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.

# Static Variables
VERSION = '1.2.1a1'
APP_NAME = 'Xibo Offline Download Client'

# Imports
from XiboOfflineDownloadUI import XiboOfflineDownloadUI
from XiboOfflineDownloadUI import AddDisplayUI
import wx
import os
import ConfigParser
import uuid
from threading import Thread, Semaphore
import urlparse
import xml.parsers.expat
from SOAPpy import WSDL
import SOAPpy.Types
import SOAPpy.Errors
import socket
import hashlib
import shutil

class XiboOfflineDownload(XiboOfflineDownloadUI):

    # Called after the GUI is initialised but before it's shown
    def setup_tasks(self):
        self.SetTitle(APP_NAME)
        self.txtOutput.AppendText('%s v%s\n' % (APP_NAME,VERSION));

        # Define Variables
        self.AddDisplayDialog = None
        self.downloadThread = None

        # Figure out where our config is saved
        self.__config_path = os.path.expanduser('~')
        self.__config_file = os.path.join(self.__config_path,'xibo-offline.cfg')

        global log
        log = self.writeLog

        # Read config (if exists)
        global config
        config = ConfigParser.ConfigParser()
        config.readfp(open('defaults.cfg'))

        print _("Reading user configuration")
        config.read([self.__config_file])

        global verbose
        try:
            verbose = config.getboolean('Main','verbose')
        except ValueError:
            config.set('Main','verbose','false')
            self.saveConfig()
            verbose = False

        # Setup the GUI with options from the config file
        if verbose:
            self.chkVerbose.SetValue(True)

        if config.get('Main','xmdsUrl') != '':
            self.txtServerURL.WriteText(config.get('Main','xmdsUrl'))

        if config.get('Main','xmdsKey') != '':
            self.txtServerKey.WriteText(config.get('Main','xmdsKey'))

        # Add displays defined in the config file
        self.updateDisplays()


    def updateDisplays(self):
        # Add displays defined in the config file
        displayNames = []

        for section in config.sections():
            if section != 'Main':
                displayNames.append(section)
        # End For Loop

        # Sort the names alphabetically ignoring case
        displayNames.sort(key=lambda x: x.lower())

        # Clear the list
        self.selectedDisplays.Set(displayNames)

    def writeLog(self,message,appendLineBreak=True,important=False):
        if verbose or important:
            self.txtOutput.AppendText(message)
            if appendLineBreak:
                self.txtOutput.AppendText('\n')

    def getConfigFile(self):
        return self.__config_file

    def saveConfig(self):
        try:
            fh = open(self.getConfigFile(), 'w+')
            config.write(fh)
            fh.close()
        except IOError:
            log('Unable to write configuration file at %s. Check your filesystem permissions.' % self.__parent.getConfigFile(),True,True)
            return False

        return True

    def downloadSchedule(self,key,outdir):
        # TODO: Implement
        return

    def downloadRequiredFiles(self,key,outdir):
        # TODO: Implement
        return []

    # Event Handlers
    def onSelectAll(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Select all items in the display list
        numItems = self.selectedDisplays.GetCount()
        for i in range(0,numItems):
            self.selectedDisplays.SetSelection(i)
        
        if numItems > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()

        event.Skip()

    def onSelectNone(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Clear all selections in the display list
        numItems = self.selectedDisplays.GetCount()

        for i in range(0,numItems):
            self.selectedDisplays.Deselect(i)

        self.btnRemove.Disable()
        self.btnDownload.Disable()
        event.Skip()

    def onSelectInvert(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Invert the selections in the display list.
        selected = self.selectedDisplays.GetSelections()
        numItems = self.selectedDisplays.GetCount()

        for i in range(0,numItems):
            if i in selected:
                self.selectedDisplays.Deselect(i)
            else:
                self.selectedDisplays.SetSelection(i)
        
        if len(self.selectedDisplays.GetSelections()) > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()
        else:
            self.btnRemove.Disable()
            self.btnDownload.Disable()

        event.Skip()

    def onAddDisplay(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if self.AddDisplayDialog == None:        
            self.AddDisplayDialog = AddDisplay(self,-1)
        self.AddDisplayDialog.Show()
        self.AddDisplayDialog.setParent(self)
        event.Skip()

    def onDeleteDisplay(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Are you sure
        dlg = wx.MessageDialog(self, _("Are you sure you want to delete the selected displays?"),_("Delete Displays?"),style=wx.YES_NO|wx.NO_DEFAULT|wx.ICON_QUESTION|wx.STAY_ON_TOP)
        if dlg.ShowModal() == wx.ID_YES:
            # Get a list of selected Displays
            selections = self.selectedDisplays.GetSelections()

            for i in selections:
                display = self.selectedDisplays.GetString(i)
                config.remove_section(display)
                log(_('Removed display %s') % display,True,True)

            self.saveConfig()
            self.updateDisplays()
        
        dlg.Destroy()        
        event.Skip()

    def onDownload(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        try:
            outdir = config.get('Main','outDir')
        except ConfigParser.NoOptionError:
            outdir=''

        log('Saved Output Directory: %s' % outdir)

        # Display a dialog asking for a folder to write to
        dlg = wx.DirDialog(self,_("Select output USB drive root folder"),outdir)

        if dlg.ShowModal() == wx.ID_OK:
            outdir = dlg.GetPath()
            config.set('Main','outDir',outdir)
            self.saveConfig()
            log('Selected Output Directory: %s' % outdir)

        dlg.Destroy()

        # Get the displays selected:
        displays = []
        selections = self.selectedDisplays.GetSelections()

        for i in selections:
            displays.append(self.selectedDisplays.GetString(i))

        # Displays now contains a list of display names to download content for
        # Create output folders for each display key:

        for display in displays:
            try:
                key = config.get(display,'license')
            except ConfigParser.NoOptionError:
                log(_('No license key for display %s. Please check configuration.') % display,True,True)
                # Remove the display from the list
                del displays[displays.index(display)]
                continue

            try:
                shutil.rmtree(os.path.join(outdir,key))
            except OSError:
                if os.path.isdir(os.path.join(outdir,key)):
                    log(_('Unable to remove old output directory. Check filesystem permissions.'),True,True)
                    continue
                else:
                    # Do nothing - we're golden.
                    pass

            try:
                os.mkdir(os.path.join(outdir,key))
            except IOError:
                log(_('Unable to create output directory. Check filesystem permissions.'),True,True)
                continue

            self.downloadSchedule(key,os.path.join(outdir,key))
            rf = self.downloadRequiredFiles(key,os.path.join(outdir,key))

            #TODO: Parse RF
            #for file in rf:
            #    self.downloadMedia(key,fileid,os.path.join(outdir,key))

            # End for loop of displays
            

        event.Skip()

    def onCancel(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if self.downloadThread != None:
            # TODO: Terminate running thread
            pass
        event.Skip()

    def onConfigSave(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        config.set('Main','xmdsUrl',self.txtServerURL.GetValue())
        config.set('Main','xmdsKey',self.txtServerKey.GetValue())

        if self.chkVerbose.IsChecked():
            config.set('Main','verbose','true')
        else:
            config.set('Main','verbose','false')

        if not self.saveConfig():
            event.Skip()
            return

        log('Configuration saved',True,True)
        self.btnSave.Disable()
        event.Skip()

    def onDisplayListDClick(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # print "Event handler `onDisplayListDClick' not implemented"
        event.Skip()

    def onDisplayListClick(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if len(self.selectedDisplays.GetSelections()) > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()
        else:
            self.btnRemove.Disable()
            self.btnDownload.Disable()
        event.Skip()

    def onServerUrlChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

    def onServerKeyChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

    def onVerboseChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

class AddDisplay(AddDisplayUI):
    def onCreateDisplay(self, event): # wxGlade: AddDisplayUI.<event_handler>
        # Save the new display to config
        try:
            config.add_section(self.txtClientName.GetValue())
        except ConfigParser.DuplicateSectionError:
            log('A display with that name already exists.',True,True)
            event.Skip()
            return
        except ValueError:
            log('You may not name a display "DEFAULT". Please choose another name.',True,True)
            event.Skip()
            return

        try:
            config.set(self.txtClientName.GetValue(),'license',self.txtClientKey.GetValue())
        except ConfigParser.NoSectionError:
            log('Something went wrong adding the display to your configuration file.',True,True)
            event.Skip()
            return

        if not self.__parent.saveConfig():
            event.Skip()
            return

        # Actually Add the Display
        xmds = XMDS(self.txtClientKey.GetValue(),self.txtClientName.GetValue(),config.get('Main','xmdsKey'))
        try:
            xmds.RegisterDisplay()
        except XMDSException:
            log('Webservice Exception',True,True)
            event.Skip()
            return

        # If all went well, reninitialize the form and close it
        self.txtClientKey.Clear()
        self.txtClientName.Clear()
        self.btnCreateDisplay.Disable()
        self.txtClientKey.WriteText(hashlib.sha1(uuid.uuid4().hex).hexdigest())
        self.__parent.updateDisplays()
        self.Close()
        event.Skip()

    def onGenerateKey(self, event): # wxGlade: AddDisplayUI.<event_handler>
        self.txtClientKey.Clear()
        self.txtClientKey.WriteText(hashlib.sha1(uuid.uuid4().hex).hexdigest())
        event.Skip()

    def onCancel(self, event): # wxGlade: AddDisplayUI.<event_handler>
        self.txtClientKey.Clear()
        self.txtClientName.Clear()
        self.btnCreateDisplay.Disable()
        self.Close()
        event.Skip()

    def onClientNameChange(self, event): # wxGlade: AddDisplayUI.<event_handler>
        if self.validateFields():
            self.btnCreateDisplay.Enable()
        else:
            self.btnCreateDisplay.Disable()
        event.Skip()

    def onClientKeyChange(self, event): # wxGlade: AddDisplayUI.<event_handler>
        if self.validateFields():
            self.btnCreateDisplay.Enable()
        else:
            self.btnCreateDisplay.Disable()
        event.Skip()

    def validateFields(self):
        ret = True

        if len(self.txtClientKey.GetValue()) < 1:
            ret = False

        if len(self.txtClientName.GetValue()) < 1:
            ret = False

        return ret


    def setParent(self,parent):
        self.__parent = parent

#### Webservice
class XMDSException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class XMDS:<commit_msg>[offline] Framework for actual download work. Also updated log function to be thread safe.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

#
# Xibo - Digitial Signage - http://www.xibo.org.uk
# Copyright (C) 2010 <NAME>
#
# This file is part of Xibo.
#
# Xibo is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version. 
#
# Xibo is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Xibo.  If not, see <http://www.gnu.org/licenses/>.

# Static Variables
VERSION = '1.2.1a1'
APP_NAME = 'Xibo Offline Download Client'

# Imports
from XiboOfflineDownloadUI import XiboOfflineDownloadUI
from XiboOfflineDownloadUI import AddDisplayUI
import wx
import os
import ConfigParser
import uuid
from threading import Thread, Semaphore
import urlparse
import xml.parsers.expat
from SOAPpy import WSDL
import SOAPpy.Types
import SOAPpy.Errors
import socket
import hashlib
import shutil
import Queue

class XiboOfflineDownload(XiboOfflineDownloadUI):

    # Called after the GUI is initialised but before it's shown
    def setup_tasks(self):
        self.SetTitle(APP_NAME)
        self.txtOutput.AppendText('%s v%s\n' % (APP_NAME,VERSION));

        # Define Variables
        self.AddDisplayDialog = None
        self.downloadThread = None

        # Queue for download requests
        self.downloadQueue = Queue.Queue()

        # Figure out where our config is saved
        self.__config_path = os.path.expanduser('~')
        self.__config_file = os.path.join(self.__config_path,'xibo-offline.cfg')

        global log
        log = self.writeLog

        # Read config (if exists)
        global config
        config = ConfigParser.ConfigParser()
        config.readfp(open('defaults.cfg'))

        print _("Reading user configuration")
        config.read([self.__config_file])

        global verbose
        try:
            verbose = config.getboolean('Main','verbose')
        except ValueError:
            config.set('Main','verbose','false')
            self.saveConfig()
            verbose = False

        # Setup the GUI with options from the config file
        if verbose:
            self.chkVerbose.SetValue(True)

        if config.get('Main','xmdsUrl') != '':
            self.txtServerURL.WriteText(config.get('Main','xmdsUrl'))

        if config.get('Main','xmdsKey') != '':
            self.txtServerKey.WriteText(config.get('Main','xmdsKey'))

        # Add displays defined in the config file
        self.updateDisplays()


    def updateDisplays(self):
        # Add displays defined in the config file
        displayNames = []

        for section in config.sections():
            if section != 'Main':
                displayNames.append(section)
        # End For Loop

        # Sort the names alphabetically ignoring case
        displayNames.sort(key=lambda x: x.lower())

        # Clear the list
        self.selectedDisplays.Set(displayNames)

    def writeLog(self,message,appendLineBreak=True,important=False):
        if verbose or important:
            wx.CallAfter(self.txtOutput.AppendText, message)
            if appendLineBreak:
                wx.CallAfter(self.txtOutput.AppendText, '\n')

    def getConfigFile(self):
        return self.__config_file

    def saveConfig(self):
        try:
            fh = open(self.getConfigFile(), 'w+')
            config.write(fh)
            fh.close()
        except IOError:
            log('Unable to write configuration file at %s. Check your filesystem permissions.' % self.__parent.getConfigFile(),True,True)
            return False

        return True

    def finishedDownload(self):
        log('Finished Download',True,True)
        self.btnDownload.Enable()

    # Event Handlers
    def onSelectAll(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Select all items in the display list
        numItems = self.selectedDisplays.GetCount()
        for i in range(0,numItems):
            self.selectedDisplays.SetSelection(i)
        
        if numItems > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()

        event.Skip()

    def onSelectNone(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Clear all selections in the display list
        numItems = self.selectedDisplays.GetCount()

        for i in range(0,numItems):
            self.selectedDisplays.Deselect(i)

        self.btnRemove.Disable()
        self.btnDownload.Disable()
        event.Skip()

    def onSelectInvert(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Invert the selections in the display list.
        selected = self.selectedDisplays.GetSelections()
        numItems = self.selectedDisplays.GetCount()

        for i in range(0,numItems):
            if i in selected:
                self.selectedDisplays.Deselect(i)
            else:
                self.selectedDisplays.SetSelection(i)
        
        if len(self.selectedDisplays.GetSelections()) > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()
        else:
            self.btnRemove.Disable()
            self.btnDownload.Disable()

        event.Skip()

    def onAddDisplay(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if self.AddDisplayDialog == None:        
            self.AddDisplayDialog = AddDisplay(self,-1)
        self.AddDisplayDialog.Show()
        self.AddDisplayDialog.setParent(self)
        event.Skip()

    def onDeleteDisplay(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Are you sure
        dlg = wx.MessageDialog(self, _("Are you sure you want to delete the selected displays?"),_("Delete Displays?"),style=wx.YES_NO|wx.NO_DEFAULT|wx.ICON_QUESTION|wx.STAY_ON_TOP)
        if dlg.ShowModal() == wx.ID_YES:
            # Get a list of selected Displays
            selections = self.selectedDisplays.GetSelections()

            for i in selections:
                display = self.selectedDisplays.GetString(i)
                config.remove_section(display)
                log(_('Removed display %s') % display,True,True)

            self.saveConfig()
            self.updateDisplays()
        
        dlg.Destroy()        
        event.Skip()

    def onDownload(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # Disable Download Button
        self.btnDownload.Disable()

        try:
            outdir = config.get('Main','outDir')
        except ConfigParser.NoOptionError:
            outdir=''

        log('Saved Output Directory: %s' % outdir)

        # Display a dialog asking for a folder to write to
        dlg = wx.DirDialog(self,_("Select output USB drive root folder"),outdir)

        if dlg.ShowModal() == wx.ID_OK:
            outdir = dlg.GetPath()
            config.set('Main','outDir',outdir)
            self.saveConfig()
            log('Selected Output Directory: %s' % outdir)

        dlg.Destroy()

        # Get the displays selected:
        displays = []
        selections = self.selectedDisplays.GetSelections()

        for i in selections:
            displays.append(self.selectedDisplays.GetString(i))

        # Displays now contains a list of display names to download content for
        # Create output folders for each display key:

        for display in displays:
            try:
                key = config.get(display,'license')
            except ConfigParser.NoOptionError:
                log(_('No license key for display %s. Please check configuration.') % display,True,True)
                # Remove the display from the list
                del displays[displays.index(display)]
                continue

            try:
                shutil.rmtree(os.path.join(outdir,key))
            except OSError:
                if os.path.isdir(os.path.join(outdir,key)):
                    log(_('Unable to remove old output directory. Check filesystem permissions.'),True,True)
                    continue
                else:
                    # Do nothing - we're golden.
                    pass

            try:
                os.mkdir(os.path.join(outdir,key))
            except IOError:
                log(_('Unable to create output directory. Check filesystem permissions.'),True,True)
                continue

            displayDict = {'name': display, 'key': key, 'outdir': os.path.join(outdir,key)}
            self.downloadQueue.put(displayDict)
            # Finish while loop for displays

        # Push the queue in to a download thread and start it running
        self.downloadThread = XMDSDownloadThread(self,self.downloadQueue)
        self.downloadThread.start()

        event.Skip()

    def onCancel(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if self.downloadThread != None:
            # TODO: Terminate running thread
            pass
        event.Skip()

    def onConfigSave(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        config.set('Main','xmdsUrl',self.txtServerURL.GetValue())
        config.set('Main','xmdsKey',self.txtServerKey.GetValue())

        if self.chkVerbose.IsChecked():
            config.set('Main','verbose','true')
        else:
            config.set('Main','verbose','false')

        if not self.saveConfig():
            event.Skip()
            return

        log('Configuration saved',True,True)
        self.btnSave.Disable()
        event.Skip()

    def onDisplayListDClick(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        # print "Event handler `onDisplayListDClick' not implemented"
        event.Skip()

    def onDisplayListClick(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        if len(self.selectedDisplays.GetSelections()) > 0:
            self.btnRemove.Enable()
            self.btnDownload.Enable()
        else:
            self.btnRemove.Disable()
            self.btnDownload.Disable()
        event.Skip()

    def onServerUrlChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

    def onServerKeyChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

    def onVerboseChange(self, event): # wxGlade: XiboOfflineDownloadUI.<event_handler>
        self.btnSave.Enable()
        event.Skip()

class AddDisplay(AddDisplayUI):
    def onCreateDisplay(self, event): # wxGlade: AddDisplayUI.<event_handler>
        # Save the new display to config
        try:
            config.add_section(self.txtClientName.GetValue())
        except ConfigParser.DuplicateSectionError:
            log('A display with that name already exists.',True,True)
            event.Skip()
            return
        except ValueError:
            log('You may not name a display "DEFAULT". Please choose another name.',True,True)
            event.Skip()
            return

        try:
            config.set(self.txtClientName.GetValue(),'license',self.txtClientKey.GetValue())
        except ConfigParser.NoSectionError:
            log('Something went wrong adding the display to your configuration file.',True,True)
            event.Skip()
            return

        if not self.__parent.saveConfig():
            event.Skip()
            return

        # Actually Add the Display
        xmds = XMDS(self.txtClientKey.GetValue(),self.txtClientName.GetValue(),config.get('Main','xmdsKey'))
        try:
            xmds.RegisterDisplay()
        except XMDSException:
            log('Webservice Exception',True,True)
            event.Skip()
            return

        # If all went well, reninitialize the form and close it
        self.txtClientKey.Clear()
        self.txtClientName.Clear()
        self.btnCreateDisplay.Disable()
        self.txtClientKey.WriteText(hashlib.sha1(uuid.uuid4().hex).hexdigest())
        self.__parent.updateDisplays()
        self.Close()
        event.Skip()

    def onGenerateKey(self, event): # wxGlade: AddDisplayUI.<event_handler>
        self.txtClientKey.Clear()
        self.txtClientKey.WriteText(hashlib.sha1(uuid.uuid4().hex).hexdigest())
        event.Skip()

    def onCancel(self, event): # wxGlade: AddDisplayUI.<event_handler>
        self.txtClientKey.Clear()
        self.txtClientName.Clear()
        self.btnCreateDisplay.Disable()
        self.Close()
        event.Skip()

    def onClientNameChange(self, event): # wxGlade: AddDisplayUI.<event_handler>
        if self.validateFields():
            self.btnCreateDisplay.Enable()
        else:
            self.btnCreateDisplay.Disable()
        event.Skip()

    def onClientKeyChange(self, event): # wxGlade: AddDisplayUI.<event_handler>
        if self.validateFields():
            self.btnCreateDisplay.Enable()
        else:
            self.btnCreateDisplay.Disable()
        event.Skip()

    def validateFields(self):
        ret = True

        if len(self.txtClientKey.GetValue()) < 1:
            ret = False

        if len(self.txtClientName.GetValue()) < 1:
            ret = False

        return ret


    def setParent(self,parent):
        self.__parent = parent

class XMDSDownloadThread(Thread):
    def __init__(self,parent,downloadQueue):
        Thread.__init__(self)
        self.__parent = parent
        self.__q = downloadQueue

    def run(self):
        log('Starting Download Thread')
        try:
            while True:
                # While loop will be broken by Queue empty exception
                display = self.__q.get(False)
                key = display['key']
                name = display['name']
                outdir = display['outdir']

                log('Processing display %s' % name)
                print 'Processing display %s' % name
    
                self.downloadSchedule(key,outdir)
                rf = self.downloadRequiredFiles(key,outdir)

                for tmpFile in rf:
                    fileid = tmpFile['fileid']
                    size = tmpFile['size']
                    checksum = tmpFile['checksum']
                    filetype = tmpFile['filetype']

                    log('Processing file %s for display %s' % (fileid,name))
                    print 'Processing file %s for display %s' % (fileid,name)

                    if filetype == 'media':
                        self.downloadMedia()
                    else:
                        self.downloadLayout()

        except Queue.Empty:
            # Queue is empty.
            log('Download Queue is empty')
            print 'Download Queue is empty'

        self.__parent.finishedDownload()

    def downloadSchedule(self,key,outdir):
        # TODO: Implement
        print 'Download Schedule [IN]'
        return

    def downloadRequiredFiles(self,key,outdir):
        # Download RF, save to disk and return a list of dicts of files
        # rf = [{'fileid':1, 'size':49320, 'checksum':'<checksum!>', 'filetype':'media'}]
        # TODO: Implement
        print 'Download Required Files [IN]'
        return []

    def downloadMedia(self):
        return

    def downloadLayout(self):
        return

#### Webservice
class XMDSException(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

class XMDS:
{'fraction_non_alphanumeric': 0.0984353066045811, 'fraction_numerical': 0.001780395110761119, 'mean_word_length': 2.2195767195767195, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 95, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4127140', 'n_tokens_mistral': 8234, 'n_tokens_neox': 7819, 'n_words': 1944}