## This is a script for the test of deblending using sparsity promotion with shaping regularization (Using the SNR definition from Araz).

## Copyright (C) 2013
## Yangkang Chen, The Univeristy of Texas at Austin

## The are several assumptions for implementing the following scripts:
## 1. data1 and data2 have the same dimensions
## 2. data1 and data2 are sparse in fourier or seislet domains
## 3. the unblended data is known (in other words it's used for testing).
##    Otherwise, the unblended1(2) is substituted by blended1(2), in which case
##    The error diagram has no physical meanings.

from rsf.proj import*
def Grey(data,other):
	Result(data,'grey label2=Trace unit2="" label1=Time unit1="s" title="" wherexlabel=b wheretitle=t color=b %s'%other)

def Greyplot(data,other):
	Plot(data,'grey label2=Trace unit2="" label1=Time unit1="s" title="" wherexlabel=b wheretitle=t color=b %s'%other)

def Graph(data,other):
	Result(data,'graph label1="" label2="" unit1="" unit2=""  title="" wherexlabel=b wheretitle=t %s' %other)

def Graphplot(data,other):
	Plot(data,'graph label1="" label2="" unit1="" unit2=""  title="" wherexlabel=b wheretitle=t %s' %other)

def shapeslet(sig1,sig2,dip1,dip2,mute1,mute2,padno,n2,mode,thr,i): #mute1(2) is a string, when there is no mute, mute1(2) is "cp"
    Flow(sig1+"p%g"%(i+1),[sig1,dip1],
	 '''
	 %s |pad n2=%g  |
         seislet dip=${SOURCES[1]} eps=0.1
		  adj=y inv=y unit=y type=b |
         threshold1 type=%s ifperc=1 thr=%f |
         seislet dip=${SOURCES[1]} eps=0.1
			inv=y unit=y type=b |
         window n2=%g |%s
         '''%(mute1,padno,mode,thr,n2,mute1))
    Flow(sig2+"p%g"%(i+1),[sig2,dip2],
	 '''
	 %s|pad n2=%g  |
         seislet dip=${SOURCES[1]} eps=0.1
		  adj=y inv=y unit=y type=b |
         threshold1 type=%s ifperc=1 thr=%f |
         seislet dip=${SOURCES[1]} eps=0.1
			inv=y unit=y type=b |
         window n2=%g | %s
         '''%(mute2,padno,mode,thr,n2,mute2))
    return (sig1+"p%g"%(i+1),sig2+"p%g"%(i+1))

def shapefft(sig1,sig2,mute1,mute2,mode,thr,i):#mute1(2) is a string, when there is no mute, mute1(2) is "cp"
    Flow(sig1+"p%g"%(i+1),sig1,
	 '''
	 %s|fft1 | fft3 axis=2 pad=1|
         threshold1 type=%s ifperc=1 thr=%f |
         fft3 inv=y axis=2 | fft1 inv=y |%s
         '''%(mute1,mode,thr,mute1))
    Flow(sig2+"p%g"%(i+1),sig2,
	 '''
	 %s|fft1 | fft3 axis=2 pad=1|
         threshold1 type=%s ifperc=1 thr=%f |
         fft3 inv=y axis=2 | fft1 inv=y | %s
         '''%(mute2,mode,thr,mute2))
    return (sig1+"p%g"%(i+1),sig2+"p%g"%(i+1))

def shapefxdecon(sig1,sig2,mute1,mute2,n2,i): #mute1(2) is a string, when there is no mute, mute1(2) is "cp"
    Flow(sig1+"p%g"%(i+1),[sig1],
	 '''
	 %s | fxdecon n2w=%g
         |%s
         '''%(mute1,n2,mute1))
    Flow(sig2+"p%g"%(i+1),[sig2],
	 '''
 	 %s | fxdecon n2w=%g
         |%s
         '''%(mute2,n2,mute2))
    return (sig1+"p%g"%(i+1),sig2+"p%g"%(i+1))

def step(sig1,sig2,blended1,blended2,init1,init2,shottime1,shottime2,i,n2,fraction):
#    Flow(init1+'sigtemp'+'%g'%(i+1),[sig1,sig2],'cat axis=2 ${SOURCES[1]}')
#    Flow(init1+'shottimetemp'+'%g'%(i+1),[shottime1,shottime2],'cat axis=2 ${SOURCES[1]}')
#    Flow(init1+'blendedtemp'+'%g'%(i+1),[init1+'sigtemp'+'%g'%(i+1),init1+'shottimetemp'+'%g'%(i+1)],
#    	'''
#    	blend shot_time_in=${SOURCES[1]} shot_time_out=${SOURCES[1]}
#    	''')
#
#    Flow(init1+"%g"%(i+1),[init1+'blendedtemp'+'%g'%(i+1),blended1,sig1],
#         '''
#         window n2=%g | add scale=-1,1 ${SOURCES[1]} | add scale=%g,%g ${SOURCES[2]}
#         '''%(n2,fraction,1))
#    Flow(init2+"%g"%(i+1),[init1+'blendedtemp'+'%g'%(i+1),blended2,sig2],
#         '''
#         window n2=%g f2=%g | add scale=-1,1 ${SOURCES[1]} | add scale=%g,%g ${SOURCES[2]}
#         '''%(n2,n2-1,fraction,1))

    Flow(init1+"%g"%(i+1),[sig2,shottime1,shottime2,blended1,sig1],
         '''
          blend shot_time_in=${SOURCES[2]} shot_time_out=${SOURCES[1]}
	    | add scale=%f,%f,%f ${SOURCES[3]} ${SOURCES[4]}
         '''%(-fraction,fraction,1-fraction))

    Flow(init2+"%g"%(i+1),[sig1,shottime2,shottime1,blended2,sig2],
         '''
          blend shot_time_in=${SOURCES[2]} shot_time_out=${SOURCES[1]}
	    | add scale=%f,%f,%f ${SOURCES[3]} ${SOURCES[4]}
         '''% (-fraction,fraction,1-fraction))
    return (init1+"%g"%(i+1),init2+"%g"%(i+1))

def deblendfft(unblended1,
	  unblended2,
	  blended1,
          blended2,
	  init1,
	  init2,
	  deblended1,
	  deblended2,
	  shottime1,
	  shottime2,
	  mute1,
	  mute2,
	  n1,
	  n2,
	  niter,
	  mode,
	  thr,
	  clip,
	  fraction):

    sig1, sig2 = (init1,init2)
    sigs1=[]
    sigs2=[]
    snrsa=[]
    snrsb=[]
    Flow('zero',unblended1,'math output=0')
    Flow('energy1',[unblended1, 'zero'],'diff match=${SOURCES[1]}')
    Flow('energy2',[unblended2, 'zero'],'diff match=${SOURCES[1]}')
    for i in range(niter):
 	old1,old2 = (sig1,sig2)
	snra=init1+'-snra%g'%i
	snrb=init2+'-snrb%g'%i

	Flow(snra,[unblended1,sig1,'energy1'],'diff match=${SOURCES[1]} | math a=${SOURCES[2]} output="a/input"' )
	Flow(snrb,[unblended2,sig2,'energy2'],'diff match=${SOURCES[1]} | math a=${SOURCES[2]} output="a/input"' )

	(nsig1,nsig2)=step(sig1,sig2,blended1,blended2,init1,init2,shottime1,shottime2,i,n2,fraction)
	(sig1,sig2)=shapefft(nsig1,nsig2,mute1,mute2,mode,thr,i)

    	Greyplot(sig1,'title="Esti R 1 (iter=%g)" clip=%g'% (i+1,clip))
    	Greyplot(sig2,'title="Esti R 2 (iter=%g)" clip=%g' % (i+1,clip))

	sigs1.append(sig1)
    	sigs2.append(sig2)
        snrsa.append(snra)
        snrsb.append(snrb)

    Flow(init1+'-snrsa',snrsa,'cat axis=2 ${SOURCES[1:%g]}|math output="10*log(input)/log(10)" | transp | put n1=%g o1=0 d1=1 '%(len(snrsa),len(snrsa)))
    Flow(init2+'-snrsb',snrsb,'cat axis=2 ${SOURCES[1:%g]}|math output="10*log(input)/log(10)" | transp |  put n1=%g o1=0 d1=1'%(len(snrsb),len(snrsb)))

    Graph(init1+'-snrsa','title="Data error 1" label1=Iteration label2="c(n)" symbosz=5 symbol="o" min1=0 max1=%g d1=1'%niter)
    Graph(init2+'-snrsb','title="Data error 2" label1=Iteration label2="c(n)" symbosz=5 symbol="o" min1=0 max1=%g d1=1'%niter)

#Making movie
    Plot(init1+'-sigs1',sigs1,'Movie')
    Plot(init2+'-sigs2',sigs2,'Movie')
    Flow(deblended1,sig1,'cp')
    Flow(deblended2,sig2,'cp')

def deblendslet(unblended1,
	  unblended2,
	  blended1,
          blended2,
	  init1,
	  init2,
	  deblended1,
	  deblended2,
	  shottime1,
	  shottime2,
	  mute1,
	  mute2,
	  n1,
	  n2,
	  r1,
	  r2,
	  padno,
	  niter,
	  ddip,
	  mode,
	  thr,
	  clip,
	  fhi,
	  fraction):
    Flow('mask',blended1,'math output=1 | pad n2=%g'%(padno))
    sig1, sig2 = (init1,init2)
    sigs1=[]
    sigs2=[]
    sigs=[]
    snrsa=[]
    snrsb=[]
    Flow('zero',unblended1,'math output=0')
    Flow('energy1',[unblended1, 'zero'],'diff match=${SOURCES[1]}')
    Flow('energy2',[unblended2, 'zero'],'diff match=${SOURCES[1]}')
    for i in range(niter):
#######################################################################################
# update dip map every "ddip" iterations
#######################################################################################
    	if i % ddip ==0 :
		dip1=init1+'dip%g'%int(i/ddip)
		dip2=init2+'udip%g'%int(i/ddip)
		Flow(dip1,[sig1,'mask'],
     			'''
			bandpass fhi=%g | pad n2=%g |
			dip mask=${SOURCES[1]} rect1=%g rect2=%g
			'''%(fhi,padno,r1,r2))
		Flow(dip2,[sig2,'mask'],
     			'''
			bandpass fhi=%g | pad n2=%g |
			dip mask=${SOURCES[1]} rect1=%g rect2=%g
			'''%(fhi,padno,r1,r2))
#######################################################################################
 	old1,old2 = (sig1,sig2)
	snra=init1+'-snra%g'%i
	snrb=init2+'-snrb%g'%i

	Flow(snra,[unblended1,sig1,'energy1'],'diff match=${SOURCES[1]} | math a=${SOURCES[2]} output="a/input"' )
	Flow(snrb,[unblended2,sig2,'energy2'],'diff match=${SOURCES[1]} | math a=${SOURCES[2]} output="a/input"' )

	(nsig1,nsig2)=step(sig1,sig2,blended1,blended2,init1,init2,shottime1,shottime2,i,n2,fraction)
	(sig1,sig2)=shapeslet(nsig1,nsig2,dip1,dip2,mute1,mute2,padno,n2,mode,thr,i)

    	Greyplot(sig1,'title="Esti R 1 (iter=%g)" clip=%g'% (i+1,clip))
    	Greyplot(sig2,'title="Esti R 2 (iter=%g)" clip=%g' % (i+1,clip))

	sigs1.append(sig1)
    	sigs2.append(sig2)

	Plot(sig1+'t',[sig1,sig2],'SideBySideAniso')
	sigs.append(sig1+'t')

        snrsa.append(snra)
        snrsb.append(snrb)

    Flow(init1+'-snrsa',snrsa,'cat axis=2 ${SOURCES[1:%g]}|math output="10*log(input)/log(10)" | transp | put n1=%g o1=0 d1=1 '%(len(snrsa),len(snrsb)))
    Flow(init2+'-snrsb',snrsb,'cat axis=2 ${SOURCES[1:%g]}|math output="10*log(input)/log(10)" | transp |  put n1=%g o1=0 d1=1'%(len(snrsb),len(snrsb)))

    Graph(init1+'-snrsa','title="Data error 1" label1=Iteration label2="c(n)" symbosz=5 symbol="o" min1=0 max1=%g d1=1'%niter)
    Graph(init2+'-snrsb','title="Data error 2" label1=Iteration label2="c(n)" symbosz=5 symbol="o" min1=0 max1=%g d1=1'%niter)

#Making movie
    Plot(init1+'-sigs1',sigs1,'Movie')
    Plot(init2+'-sigs2',sigs2,'Movie')

    Plot(init1+'-sigs',sigs,'Movie')

    Flow(deblended1,sig1,'cp')
    Flow(deblended2,sig2,'cp')

def deblendfxdecon(unblended1,
	  unblended2,
	  blended1,
          blended2,
	  init1,
	  init2,
	  deblended1,
	  deblended2,
	  shottime1,
	  shottime2,
	  mute1,
	  mute2,
	  n1,
	  n2,
	  niter,
	  clip,
	  fraction):
    sig1, sig2 = (init1,init2)
    sigs1=[]
    sigs2=[]
    snrsa=[]
    snrsb=[]
    Flow('zero',unblended1,'math output=0')
    Flow('energy1',[unblended1, 'zero'],'diff match=${SOURCES[1]}')
    Flow('energy2',[unblended2, 'zero'],'diff match=${SOURCES[1]}')
    for i in range(niter):
#######################################################################################


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2011 daniel (daniel.kappler@gmail.com)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""Pyqt example to demonstrate how openrave elements can be controlled by a qt-gui.

.. examplepre-block:: qtserverprocess

Description
-----------

This example shows how a qt-gui can control openrave functionality.
If an environment with a robot is loaded,
a random pose is generated for which a trajectory is planned.
This trajectory can be replayed using the qt-gui.

.. image:: ../../images/examples/qtexampleselector.jpg
  :width: 400

.. examplepost-block:: qtserverprocess
"""

from __future__ import with_statement # for python 2.5
__author__ = 'Daniel Kappler'
__copyright__ = '2011 Daniel Kappler (daniel.kappler@gmail.com)'

import sys, os, re, logging, signal
from numpy import random
from multiprocessing import Process,Pipe
from threading import Thread
from openravepy import *

from PyQt4 import QtGui, QtCore

logger = None

class CallbackHandler(QtCore.QThread):
    def __init__(self,pipe,callback=None):
        super(CallbackHandler,self).__init__()
        self.callback = callback
        self.pipe = pipe

    def run(self):
        resultValue = self.pipe.recv()
        msg = [self.callback,resultValue]
        self.emit(QtCore.SIGNAL("CallbackHandler(PyQt_PyObject)"),msg)

class OpenRaveServer(object):
    '''
    classdocs
    '''

    def __init__(self,pipe):
        '''
        Constructor
        '''
        self.pipe = pipe
        self.running = True
        self.orenv = Environment()
        self._run()

    def __del__(self):
        RaveDestroy()

    def _run(self):
        while(self.running):
            (functionName,args,handleCallback) = self.pipe.recv()
            rValue,rMessage = self.executeFunction(functionName, args)
            if handleCallback:
                self.pipe.send([rValue,rMessage])

    def executeFunction(self,name,args):
        rValue = None
        rMessage = "Function with "+name+" not available"
        if name in dir(self):
            if(args is None):
                rValue,rMessage = getattr(self,name)()
            else:
                rValue,rMessage = getattr(self,name)(args)
        return rValue,rMessage

    def StartGui(self):
        try:
            self.orenv.SetViewer('qtcoin')
            return True,None
        except:
            pass
        return None,"Please start OpenRAVE first."

    def LoadEnv(self,env):
        try:
            self.orenv.Reset()
            if((env is not None) and self.orenv.Load(env) == False):
                return None, "Could not load "+env+"."

            self.robot = self.orenv.GetRobots()[0]
            self.manip = self.robot.GetActiveManipulator()
            return True,None
        except:
            pass
        return None, "Please start OpenRAVE first."

    def UpdateTrajectory(self,pos):
        with self.orenv:
            self.robot.GetController().SetDesired(self.trajdata[pos])
        return "Trajectory position "+str(pos),[pos,len(self.trajdata)-1]

    def waitrobot(self):
        while not self.robot.GetController().IsDone():
            time.sleep(0.01)

    def CalcTrajectory(self):
        notFound = True
        with self.orenv:
            while notFound:
                logger.debug("search for goal")
                notFound = False
                try:
                    Tnewgoals = []
                    Tnewgoals.append(self.manip.GetEndEffectorTransform() +(random.rand(4,4)-0.5))
                    data = interfaces.BaseManipulation(self.robot).MoveToHandPosition(matrices=Tnewgoals,maxiter=1000,maxtries=1,seedik=4,execute=True,outputtraj=True).split(" ")
                except Exception as e:
                    notFound = True
                    logger.debug(str(e))

        skip = False
        jointValues = []
        self.trajdata = []
        size = self.robot.GetDOF()
        valueIter = iter(data)

        # remove trailing trajectory information
        valueIter.next()
        valueIter.next()
        valueIter.next()
        valueIter.next()
        try:
            while True:
                jointValues = []
                # time value
                value = valueIter.next()

                # only the arm joints
                for i in range(size):
                    value = valueIter.next()
                    jointValues.append(float(value))
                self.trajdata.append(jointValues)

                # go on till the next line
                while (len(valueIter.next()) is not 0):
                    pass

                value = valueIter.next()
        except StopIteration:
            pass

        return True, [len(self.trajdata)-1,len(self.trajdata)-1]

    def quit(self):
        RaveDestroy()
        self.running = False
        return True,None

class Server(object):
    '''
    Control server to run the benchmark in its own process.
    '''

    def __init__(self):
        '''
        Setup the shared memory data structure model and initialize the control parts.
        '''
        self.openRaveServers = []
        self.running = True
        self.orgui = None
        self.qtgui = None
        (self.pipeQtControl, self.pipeORControl) = Pipe()
        (self.pipeQtServer, self.pipeServer) = Pipe()
        self.StartQtGuiControl()
        self._run()

    def _run(self):
        '''
        Main server loop which waits for input from the qt-gui.
        '''
        while(self.running):
            (functionName,args,handleCallback) = self.pipeServer.recv()
            rValue = self.executeFunction(functionName, args)
            if handleCallback:
                self.pipeServer.send(rValue)

    def executeFunction(self,name,args):
        if name == "___START_OPENRAVE_SERVER___":
            return self.StartOpenRaveGuiServer()
        if name == "___CLOSE___":
            self.running = False
            try:
                self.qtgui.terminate()
                self.orgui.terminate()
            except:
                pass
            return True
        return False

    def StartOpenRaveGuiServer(self):
        if self.orgui:
            self.orgui.terminate()
        self.orgui = Process(target=OpenRaveServer,args=(self.pipeORControl,))
        self.orgui.start()
        return True

    def StartQtGuiControl(self):
        self.qtgui = Process(target=self._StartQtGuiControl)
        self.qtgui.start()
        return True

    def _StartQtGuiControl(self):
        app = QtGui.QApplication(sys.argv)
        form = MainWindow(self.pipeQtControl,self.pipeQtServer)
        form.show()
        app.exec_()

try:
    _fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
    _fromUtf8 = lambda s: s

class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName(_fromUtf8("MainWindow"))
        MainWindow.resize(352, 389)
        MainWindow.setMinimumSize(QtCore.QSize(352, 389))
        MainWindow.setMaximumSize(QtCore.QSize(352, 389))
        self.centralwidget = QtGui.QWidget(MainWindow)
        self.centralwidget.setObjectName(_fromUtf8("centralwidget"))
        self.teEnv = QtGui.QTextEdit(self.centralwidget)
        self.teEnv.setGeometry(QtCore.QRect(160, 220, 149, 25))
        self.teEnv.setMaximumSize(QtCore.QSize(16777215, 25))
        self.teEnv.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
        self.teEnv.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)
        self.teEnv.setObjectName(_fromUtf8("teEnv"))
        self.teOutput = QtGui.QTextEdit(self.centralwidget)
        self.teOutput.setGeometry(QtCore.QRect(160, 40, 181, 139))
        self.teOutput.setReadOnly(True)
        self.teOutput.setTextInteractionFlags(QtCore.Qt.NoTextInteraction)
        self.teOutput.setObjectName(_fromUtf8("teOutput"))
        self.label = QtGui.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(160, 20, 171, 16))
        self.label.setObjectName(_fromUtf8("label"))
        self.label_2 = QtGui.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(160, 200, 171, 16))
        self.label_2.setObjectName(_fromUtf8("label_2"))
        self.layoutWidget = QtGui.QWidget(self.centralwidget)
        self.layoutWidget.setGeometry(QtCore.QRect(10, 280, 331, 68))
        self.layoutWidget.setObjectName(_fromUtf8("layoutWidget"))
        self.verticalLayout_3 = QtGui.QVBoxLayout(self.layoutWidget)
        self.verticalLayout_3.setMargin(0)
        self.verticalLayout_3.setObjectName(_fromUtf8("verticalLayout_3"))
        self.horizontalLayout = QtGui.QHBoxLayout()
        self.horizontalLayout.setObjectName(_fromUtf8("horizontalLayout"))
        self.pbTrajectory = QtGui.QPushButton(self.layoutWidget)
        self.pbTrajectory.setObjectName(_fromUtf8("pbTrajectory"))
        self.horizontalLayout.addWidget(self.pbTrajectory)
        self.pbBackward = QtGui.QPushButton(self.layoutWidget)
        self.pbBackward.setObjectName(_fromUtf8("pbBackward"))
        self.horizontalLayout.addWidget(self.pbBackward)
        self.pbForward = QtGui.QPushButton(self.layoutWidget)
        self.pbForward.setObjectName(_fromUtf8("pbForward"))
        self.horizontalLayout.addWidget(self.pbForward)
        self.verticalLayout_3.addLayout(self.horizontalLayout)
        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
        self.verticalLayout_3.addItem(spacerItem)
        self.hsTimeLine = QtGui.QSlider(self.layoutWidget)
        self.hsTimeLine.setOrientation(QtCore.Qt.Horizontal)
        self.hsTimeLine.setObjectName(_fromUtf8("hsTimeLine"))
        self.verticalLayout_3.addWidget(self.hsTimeLine)
        self.layoutWidget1 = QtGui.QWidget(self.centralwidget)
        self.layoutWidget1.setGeometry(QtCore.QRect(10, 40, 113, 128))
        self.layoutWidget1.setObjectName(_fromUtf8("layoutWidget1"))
        self.verticalLayout = QtGui.QVBoxLayout(self.layoutWidget1)
        self.verticalLayout.setMargin(0)
        self.verticalLayout.setObjectName(_fromUtf8("verticalLayout"))
        self.pbStartOpenRAVE = QtGui.QPushButton(self.layoutWidget1)
        self.pbStartOpenRAVE.setObjectName(_fromUtf8("pbStartOpenRAVE"))
        self.verticalLayout.addWidget(self.pbStartOpenRAVE)
        self.pbStartGui = QtGui.QPushButton(self.layoutWidget1)
        self.pbStartGui.setObjectName(_fromUtf8("pbStartGui"))
        self.verticalLayout.addWidget(self.pbStartGui)
        self.pbLoadEnv = QtGui.QPushButton(self.layoutWidget1)
        self.pbLoadEnv.setObjectName(_fromUtf8("pbLoadEnv"))
        self.verticalLayout.addWidget(self.pbLoadEnv)
        self.pbClose = QtGui.QPushButton(self.layoutWidget1)
        self.pbClose.setObjectName(_fromUtf8("pbClose"))
        self.verticalLayout.addWidget(self.pbClose)
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtGui.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 352, 25))
        self.menubar.setObjectName(_fromUtf8("menubar"))
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtGui.QStatusBar(MainWindow)
        self.statusbar.setObjectName(_fromUtf8("statusbar"))
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        MainWindow.setWindowTitle(QtGui.QApplication.translate("MainWindow", "MainWindow", None, QtGui.QApplication.UnicodeUTF8))
        self.teEnv.setHtml(QtGui.QApplication.translate("MainWindow", "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n"
"<meta name=\"qrichtext\" content=\"1\" />\n"
"
data/wamtest1.env.xml
", None, QtGui.QApplication.UnicodeUTF8))
        self.label.setText(QtGui.QApplication.translate("MainWindow", "OpenRAVE Server Response", None, QtGui.QApplication.UnicodeUTF8))
        self.label_2.setText(QtGui.QApplication.translate("MainWindow", "Path to Environment file.", None, QtGui.QApplication.UnicodeUTF8))
        self.pbTrajectory.setText(QtGui.QApplication.translate("MainWindow", "Calc Trajectory", None, QtGui.QApplication.UnicodeUTF8))
        self.pbBackward.setText(QtGui.QApplication.translate("MainWindow", "step back", None, QtGui.QApplication.UnicodeUTF8))
        self.pbForward.setText(QtGui.QApplication.translate("MainWindow", "step forward", None, QtGui.QApplication.UnicodeUTF8))
        self.pbStartOpenRAVE.setText(QtGui.QApplication.translate("MainWindow", "start OpenRAVE", None, QtGui.QApplication.UnicodeUTF8))
        self.pbStartGui.setText(QtGui.QApplication.translate("MainWindow", "start GUI", None, QtGui.QApplication.UnicodeUTF8))
        self.pbLoadEnv.setText(QtGui.QApplication.translate("MainWindow", "load Env", None, QtGui.QApplication.UnicodeUTF8))
        self.pbClose.setText(QtGui.QApplication.translate("MainWindow", "Close", None, QtGui.QApplication.UnicodeUTF8))

class MainWindow(QtGui.QMainWindow,Ui_MainWindow):

    @QtCore.pyqtSignature("")
    def on_pbClose_clicked(self):
        self.closeAll()

    def __init__(self,pipeOR,pipeServer):
        super(MainWindow,self).__init__(None)
        self.pipeServer = pipeServer
        self.pipeOR = pipeOR

        self.CallbackHandler = CallbackHandler(self.pipeOR)
        self.connect(self.CallbackHandler,QtCore.SIGNAL("CallbackHandler(PyQt_PyObject)"),self.HandleCallback)
        self.__index = 0
        self.setupUi(self)

        self.buttons = []
        self.buttons.append(self.pbTrajectory)
        self.buttons.append(self.pbForward)
        self.buttons.append(self.pbBackward)
        self.buttons.append(self.hsTimeLine)
        self.pbTrajectory.setEnabled(False)
        self.pbForward.setEnabled(False)
        self.pbBackward.setEnabled(False)
        self.hsTimeLineActive = False

    def ButtonsLock(self):
        self.locks = []
        for button in self.buttons:
            self.locks.append(button.isEnabled())
            button.setEnabled(False)

    def ButtonsUnlock(self):
        for i,lock in enumerate(self.locks):
            self.buttons[i].setEnabled(lock)

    def close(self):
        self.SendToOR("___CLOSE___")
        self.SendToServer("___CLOSE___")
        QtGui.QApplication.quit()

    def closeEvent(self, event):
        self.close()

    @QtCore.pyqtSignature("")
    def on_pbClose_clicked(self):
        self.close()

    @QtCore.pyqtSignature("")
    def on_pbStartOpenRAVE_clicked(self):
        self.updateTeOutput("Starting OpenRave.")
        self.SendToServer("___START_OPENRAVE_SERVER___")

    @QtCore.pyqtSignature("")
    def on_pbStartGui_clicked(self):
        self.updateTeOutput("Starting qtcoin gui.")
        self.SendToOR("StartGui")

    @QtCore.pyqtSignature("")
    def on_pbLoadEnv_clicked(self):
        self.updateTeOutput("Loading the environment.")
        self.hsTimeLine.setEnabled(False)
        self.pbForward.setEnabled(False)
        self.pbBackward.setEnabled(False)
        self.SendToOR("LoadEnv", str(self.teEnv.toPlainText()),self.enableTrajectoryPlanning)

    @QtCore.pyqtSignature("")
    def on_pbTrajectory_clicked(self):
        self.ButtonsLock()
        self.updateTeOutput("Calculate new random trajectory.")
        self.hsTimeLineActive = False
        self.SendToOR("CalcTrajectory", callback=self.enableTrajectoryControl)

    @QtCore.pyqtSignature("")
    def on_pbForward_clicked(self):
        self.ButtonsLock()
        self.updateTeOutput("Next step of the rajectory.")
        self.hsTimeLineActive = False
        self.SendToOR("UpdateTrajectory", (self.hsTimeLine.value()+1),self.enableTrajectoryControl)

    @QtCore.pyqtSignature("")
    def on_pbBackward_clicked(self):
        self.ButtonsLock()
        self.updateTeOutput("Previous step of the trajectory.")
        self.hsTimeLineActive = False
        self.SendToOR("UpdateTrajectory", (self.hsTimeLine.value()-1),self.enableTrajectoryControl)

    @QtCore.pyqtSignature("int")
    def on_hsTimeLine_valueChanged(self,value):
        if self.hsTimeLineActive:
            self.SendToOR("UpdateTrajectory", value,self.enableTrajectoryControl)
            self.hsTimeLine.setEnabled(False)

    def enableTrajectoryPlanning(self,data):
        self.pbTrajectory.setEnabled(True)

    def enableTrajectoryControl(self,traj):
        self.ButtonsUnlock()
        self.pbForward.setEnabled(False)
        self.pbBackward.setEnabled(False)
        if len(traj) == 2:
            if traj[0] < traj[1]:
                self.pbForward.setEnabled(True)
            if traj[0] > 0:
                self.pbBackward.setEnabled(True)

            self.hsTimeLine.setMaximum(traj[1])
            self.hsTimeLine.setValue(traj[0])
            self.hsTimeLine.setEnabled(True)
            self.hsTimeLineActive = True
        else:
            self.updateTeOutput("ERROR: result was in the wrong format")

    def updateTeOutput(self,text):
        content = str(text)
        content += "\n"
        content += str(self.teOutput.toPlainText())
        self.teOutput.setText(content)

    def CallbackOR(self,args):
        self.ButtonsUnlock()

    def SendToServer(self,command,args=None,callback=None):
        handleCallback=False
        if callback:
            self.CallbackHandler.callback = callback
            self.CallbackHandler.start()
            handleCallback=True
        self.pipeServer.send([command,args,handleCallback])

    def SendToOR(self,command,args=None,callback=None):
        handleCallback=False
        if callback:
            self.CallbackHandler.callback = callback
            self.CallbackHandler.start()
            handleCallback=True
        self.pipeOR.send([command,args,handleCallback])

    def HandleCallback(self,msg):
        if(len(msg) == 2):
            if(msg[0] is not None):
                self.updateTeOutput(msg[1][0])
                try:
                    msg[0](msg[1][1])
                except Exception as e:
                    logger.error(str(e))
            else:
                self.updateTeOutput("ERROR: "+msg[1][0])
            return
        logger.error("ERROR in request format")

def main(env,options):
    "Main example code."
    global logger
    signal.signal(signal.SIGINT, signal.SIG_DFL)


#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 2008-2016 California Institute of Technology.
# Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.
# License: 3-clause BSD.  The full license text is available at:
#  - https://github.com/uqfoundation/dill/blob/master/LICENSE
"""
Methods for serialized objects (or source code) stored in temporary files
and file-like objects.
"""
#XXX: better instead to have functions write to any given file-like object ?
#XXX: currently, all file-like objects are created by the function...

__all__ = ['dump_source', 'dump', 'dumpIO_source', 'dumpIO',\
           'load_source', 'load', 'loadIO_source', 'loadIO',\
           'capture']

import contextlib
from ._dill import PY3

@contextlib.contextmanager
def capture(stream='stdout'):
    """builds a context that temporarily replaces the given stream name

    >>> with capture('stdout') as out:
    ...   print ("foo!")
    ...
    >>> print (out.getvalue())
    foo!

    """
    import sys
    if PY3:
        from io import StringIO
    else:
        from StringIO import StringIO
    orig = getattr(sys, stream)
    setattr(sys, stream, StringIO())
    try:
        yield getattr(sys, stream)
    finally:
        setattr(sys, stream, orig)

def b(x): # deal with b'foo' versus 'foo'
    import codecs
    return codecs.latin_1_encode(x)[0]

def load_source(file, **kwds):
    """load an object that was stored with dill.temp.dump_source

    file: filehandle
    alias: string name of stored object
    mode: mode to open the file, one of: {'r', 'rb'}

    >>> f = lambda x: x**2
    >>> pyfile = dill.temp.dump_source(f, alias='_f')
    >>> _f = dill.temp.load_source(pyfile)
    >>> _f(4)
    16
    """
    alias = kwds.pop('alias', None)
    mode = kwds.pop('mode', 'r')
    fname = getattr(file, 'name', file) # fname=file.name or fname=file (if str)
    source = open(fname, mode=mode, **kwds).read()
    if not alias:
        tag = source.strip().splitlines()[-1].split()
        if tag[0] != '#NAME:':
            stub = source.splitlines()[0]
            raise IOError("unknown name for code: %s" % stub)
        alias = tag[-1]
    local = {}
    exec(source, local)
    _ = eval("%s" % alias, local)
    return _

def dump_source(object, **kwds):
    """write object source to a NamedTemporaryFile (instead of dill.dump)
Loads with "import" or "dill.temp.load_source".  Returns the filehandle.

    >>> f = lambda x: x**2
    >>> pyfile = dill.temp.dump_source(f, alias='_f')
    >>> _f = dill.temp.load_source(pyfile)
    >>> _f(4)
    16

    >>> f = lambda x: x**2
    >>> pyfile = dill.temp.dump_source(f, dir='.')
    >>> modulename = os.path.basename(pyfile.name).split('.py')[0]
    >>> exec('from %s import f as _f' % modulename)
    >>> _f(4)
    16

Optional kwds:
    If 'alias' is specified, the object will be renamed to the given string.

    If 'prefix' is specified, the file name will begin with that prefix,
    otherwise a default prefix is used.

    If 'dir' is specified, the file will be created in that directory,
    otherwise a default directory is used.

    If 'text' is specified and true, the file is opened in text
    mode.  Else (the default) the file is opened in binary mode.  On
    some operating systems, this makes no difference.

NOTE: Keep the return value for as long as you want your file to exist !
    """ #XXX: write a "load_source"?
    from .source import importable, getname
    import tempfile
    kwds.pop('suffix', '') # this is *always* '.py'
    alias = kwds.pop('alias', '') #XXX: include an alias so a name is known
    name = str(alias) or getname(object)
    name = "\n#NAME: %s\n" % name
    #XXX: assumes kwds['dir'] is writable and on $PYTHONPATH
    file = tempfile.NamedTemporaryFile(suffix='.py', **kwds)
    file.write(b(''.join([importable(object, alias=alias),name])))
    file.flush()
    return file

def load(file, **kwds):
    """load an object that was stored with dill.temp.dump

    file: filehandle
    mode: mode to open the file, one of: {'r', 'rb'}

    >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
    >>> dill.temp.load(dumpfile)
    [1, 2, 3, 4, 5]
    """
    import dill as pickle
    mode = kwds.pop('mode', 'rb')
    name = getattr(file, 'name', file) # name=file.name or name=file (if str)
    return pickle.load(open(name, mode=mode, **kwds))

def dump(object, **kwds):
    """dill.dump of object to a NamedTemporaryFile.
Loads with "dill.temp.load".  Returns the filehandle.

    >>> dumpfile = dill.temp.dump([1, 2, 3, 4, 5])
    >>> dill.temp.load(dumpfile)
    [1, 2, 3, 4, 5]

Optional kwds:
    If 'suffix' is specified, the file name will end with that suffix,
    otherwise there will be no suffix.

    If 'prefix' is specified, the file name will begin with that prefix,
    otherwise a default prefix is used.

    If 'dir' is specified, the file will be created in that directory,
    otherwise a default directory is used.

    If 'text' is specified and true, the file is opened in text
    mode.  Else (the default) the file is opened in binary mode.  On
    some operating systems, this makes no difference.

NOTE: Keep the return value for as long as you want your file to exist !
    """
    import dill as pickle
    import tempfile
    file = tempfile.NamedTemporaryFile(**kwds)
    pickle.dump(object, file)
    file.flush()
    return file

def loadIO(buffer, **kwds):
    """load an object that was stored with dill.temp.dumpIO

    buffer: buffer object

    >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
    >>> dill.temp.loadIO(dumpfile)
    [1, 2, 3, 4, 5]
    """
    import dill as pickle
    if PY3:
        from io import BytesIO as StringIO
    else:
        from StringIO import StringIO
    value = getattr(buffer, 'getvalue', buffer) # value or buffer.getvalue
    if value != buffer: value = value() # buffer.getvalue()
    return pickle.load(StringIO(value))

def dumpIO(object, **kwds):
    """dill.dump of object to a buffer.
Loads with "dill.temp.loadIO".  Returns the buffer object.

    >>> dumpfile = dill.temp.dumpIO([1, 2, 3, 4, 5])
    >>> dill.temp.loadIO(dumpfile)
    [1, 2, 3, 4, 5]
    """
    import dill as pickle
    if PY3:
        from io import BytesIO as StringIO
    else:
        from StringIO import StringIO
    file = StringIO()
    pickle.dump(object, file)
    file.flush()
    return file

def loadIO_source(buffer, **kwds):
    """load an object that was stored with dill.temp.dumpIO_source

    buffer: buffer object
    alias: string name of stored object

    >>> f = lambda x:x**2
    >>> pyfile = dill.temp.dumpIO_source(f, alias='_f')
    >>> _f = dill.temp.loadIO_source(pyfile)
    >>> _f(4)
    16
    """
    alias = kwds.pop('alias', None)
    source = getattr(buffer, 'getvalue', buffer) # source or buffer.getvalue
    if source != buffer: source = source() # buffer.getvalue()
    if PY3: source = source.decode() # buffer to string
    if not alias:
        tag = source.strip().splitlines()[-1].split()
        if tag[0] != '#NAME:':


#Copyright ReportLab Europe Ltd. 2000-2012
#see license.txt for license details
#history http://www.reportlab.co.uk/cgi-bin/viewcvs.cgi/public/reportlab/trunk/reportlab/pdfbase/pdfdoc.py
__version__=''' $Id$ '''
__doc__="""
The module pdfdoc.py handles the 'outer structure' of PDF documents, ensuring that
all objects are properly cross-referenced and indexed to the nearest byte.  The
'inner structure' - the page descriptions - are presumed to be generated before
each page is saved.
pdfgen.py calls this and provides a 'canvas' object to handle page marking operators.
piddlePDF calls pdfgen and offers a high-level interface.

The classes within this generally mirror structures in the PDF file
and are not part of any public interface.  Instead, canvas and font
classes are made available elsewhere for users to manipulate.
"""
import string, types, binascii, codecs
from reportlab.pdfbase import pdfutils
from reportlab.pdfbase.pdfutils import LINEEND # this constant needed in both
from reportlab import rl_config
from reportlab.lib.utils import import_zlib, open_for_read, fp_str, _digester, makeFileName
from reportlab.pdfbase import pdfmetrics
try:
    from hashlib import md5
except ImportError:
    from md5 import md5

from sys import platform
try:
    from sys import version_info
except: # pre-2.0
    # may be inaccurate but will at least
    #work in anything which seeks to format
    # version_info into a string
    version_info = (1,5,2,'unknown',0)

if platform[:4] == 'java' and version_info[:2] == (2, 1):
    # workaround for list()-bug in Jython 2.1 (should be fixed in 2.2)
    def list(sequence):
        def f(x):
            return x
        return map(f, sequence)

def utf8str(x):
    if isinstance(x,unicode):
        return x.encode('utf8')
    else:
        return str(x)

class PDFError(Exception):
    pass

# set this flag to get more vertical whitespace (and larger files)
LongFormat = 1
##if LongFormat: (doesn't work)
##    pass
##else:
##    LINEEND = "\n" # no wasteful carriage returns!

# __InternalName__ is a special attribute that can only be set by the Document arbitrator
__InternalName__ = "__InternalName__"

# __RefOnly__ marks reference only elements that must be formatted on top level
__RefOnly__ = "__RefOnly__"

# __Comment__ provides a (one line) comment to inline with an object ref, if present
#   if it is more than one line then percentize it...
__Comment__ = "__Comment__"

# If DoComments is set then add helpful (space wasting) comment lines to PDF files
DoComments = 1
if not LongFormat:
    DoComments = 0

# name for standard font dictionary
BasicFonts = "BasicFonts"

# name for the pages object
Pages = "Pages"

### generic utilities

# for % substitutions
LINEENDDICT = {"LINEEND": LINEEND, "PERCENT": "%"}
PDF_VERSION_DEFAULT = (1, 3)
PDF_SUPPORT_VERSION = dict(     #map keyword to min version that supports it
    transparency = (1, 4),
    )

from types import InstanceType
def format(element, document, toplevel=0, InstanceType=InstanceType):
    """Indirection step for formatting.
       Ensures that document parameters alter behaviour
       of formatting for all elements.
    """
    if hasattr(element,'__PDFObject__'):
        if not toplevel and hasattr(element, __RefOnly__):
            # the object cannot be a component at non top level.
            # make a reference to it and return it's format
            return document.Reference(element).format(document)
        else:
            f = element.format(document)
            if not rl_config.invariant and DoComments and hasattr(element, __Comment__):
                f = "%s%s%s%s" % ("% ", element.__Comment__, LINEEND, f)
            return f
    elif type(element) in (float, int):
        #use a controlled number formatting routine
        #instead of str, so Jython/Python etc do not differ
        return fp_str(element)
    else:
        return str(element)

def xObjectName(externalname):
    return "FormXob.%s" % externalname

# backwards compatibility
formName = xObjectName

# no encryption
class NoEncryption:
    def encode(self, t):
        "encode a string, stream, text"
        return t
    def prepare(self, document):
        # get ready to do encryption
        pass
    def register(self, objnum, version):
        # enter a new direct object
        pass
    def info(self):
        # the representation of self in file if any (should be None or PDFDict)
        return None

class DummyDoc:
    "used to bypass encryption when required"
    __PDFObject__ = True
    encrypt = NoEncryption()

### the global document structure manager
class PDFDocument:
    __PDFObject__ = True
    # set this to define filters
    defaultStreamFilters = None
    encrypt = NoEncryption() # default no encryption
    def __init__(self,
                 dummyoutline=0,
                 compression=rl_config.pageCompression,
                 invariant=rl_config.invariant,
                 filename=None,
                 pdfVersion=PDF_VERSION_DEFAULT,
                 ):
        self._ID = None
        self.objectcounter = 0
        self.shadingCounter = 0
        self.inObject = None
        self.pageCounter = 1

        # allow None value to be passed in to mean 'give system defaults'
        if invariant is None:
            self.invariant = rl_config.invariant
        else:
            self.invariant = invariant
        self.setCompression(compression)
        self._pdfVersion = pdfVersion
        # signature for creating PDF ID
        sig = self.signature = md5()
        sig.update("a reportlab document")
        if not self.invariant:
            cat = _getTimeStamp()
        else:
            cat = 946684800.0
        sig.update(repr(cat)) # initialize with timestamp digest
        # mapping of internal identifier ("Page001") to PDF objectnumber and generation number (34, 0)
        self.idToObjectNumberAndVersion = {}
        # mapping of internal identifier ("Page001") to PDF object (PDFPage instance)
        self.idToObject = {}
        # internal id to file location
        self.idToOffset = {}
        # number to id
        self.numberToId = {}
        cat = self.Catalog = self._catalog = PDFCatalog()
        pages = self.Pages = PDFPages()
        cat.Pages = pages
        if dummyoutline:
            outlines = PDFOutlines0()
        else:
            outlines = PDFOutlines()
        self.Outlines = self.outline = outlines
        cat.Outlines = outlines
        self.info = PDFInfo()
        self.info.invariant = self.invariant
        #self.Reference(self.Catalog)
        #self.Reference(self.Info)
        self.fontMapping = {}
        #make an empty font dictionary
        DD = PDFDictionary({})
        DD.__Comment__ = "The standard fonts dictionary"
        self.Reference(DD, BasicFonts)
        self.delayedFonts = []

    def setCompression(self, onoff):
        # XXX: maybe this should also set self.defaultStreamFilters?
        self.compression = onoff

    def ensureMinPdfVersion(self, *keys):
        "Ensure that the pdf version is greater than or equal to that specified by the keys"
        for k in keys:
            self._pdfVersion = max(self._pdfVersion, PDF_SUPPORT_VERSION[k])

    def updateSignature(self, thing):
        "add information to the signature"
        if self._ID: return # but not if its used already!
        self.signature.update(utf8str(thing))

    def ID(self):
        "A unique fingerprint for the file (unless in invariant mode)"
        if self._ID:
            return self._ID
        digest = self.signature.digest()
        doc = DummyDoc()
        ID = PDFString(digest,enc='raw')
        IDs = ID.format(doc)
        self._ID = "%s %% ReportLab generated PDF document -- digest (http://www.reportlab.com) %s [%s %s] %s" % (
                LINEEND, LINEEND, IDs, IDs, LINEEND)
        return self._ID

    def SaveToFile(self, filename, canvas):
        if hasattr(getattr(filename, "write",None),'__call__'):
            myfile = 0
            f = filename
            filename = makeFileName(getattr(filename,'name',''))
        else :
            myfile = 1
            filename = makeFileName(filename)
            f = open(filename, "wb")
        f.write(self.GetPDFData(canvas))
        if myfile:
            f.close()
            import os
            if os.name=='mac':
                from reportlab.lib.utils import markfilename
                markfilename(filename) # do platform specific file junk
        if getattr(canvas,'_verbosity',None): print 'saved', filename

    def GetPDFData(self, canvas):
        # realize delayed fonts
        for fnt in self.delayedFonts:
            fnt.addObjects(self)
        # add info stuff to signature
        self.info.invariant = self.invariant
        self.info.digest(self.signature)
        ### later: maybe add more info to sig?
        # prepare outline
        self.Reference(self.Catalog)
        self.Reference(self.info)
        outline = self.outline
        outline.prepare(self, canvas)
        return self.format()

    def inPage(self):
        """specify the current object as a page (enables reference binding and other page features)"""
        if self.inObject is not None:
            if self.inObject=="page": return
            raise ValueError, "can't go in page already in object %s" % self.inObject
        self.inObject = "page"

    def inForm(self):
        """specify that we are in a form xobject (disable page features, etc)"""
        # don't need this check anymore since going in a form pushes old context at canvas level.
        #if self.inObject not in ["form", None]:
        #    raise ValueError, "can't go in form already in object %s" % self.inObject
        self.inObject = "form"
        # don't need to do anything else, I think...

    def getInternalFontName(self, psfontname):
        fm = self.fontMapping
        if psfontname in fm:
            return fm[psfontname]
        else:
            try:
                # does pdfmetrics know about it? if so, add
                fontObj = pdfmetrics.getFont(psfontname)
                if fontObj._dynamicFont:
                    raise PDFError("getInternalFontName(%s) called for a dynamic font" % repr(psfontname))
                fontObj.addObjects(self)
                #self.addFont(fontObj)
                return fm[psfontname]
            except KeyError:
                raise PDFError("Font %s not known!" % repr(psfontname))

    def thisPageName(self):
        return "Page"+repr(self.pageCounter)

    def thisPageRef(self):
        return PDFObjectReference(self.thisPageName())

    def addPage(self, page):
        name = self.thisPageName()
        self.Reference(page, name)
        self.Pages.addPage(page)
        self.pageCounter += 1
        self.inObject = None

    def addForm(self, name, form):
        """add a Form XObject."""
        # XXX should check that name is a legal PDF name
        if self.inObject != "form":
            self.inForm()
        self.Reference(form, xObjectName(name))
        self.inObject = None

    def annotationName(self, externalname):
        return "Annot.%s"%externalname

    def addAnnotation(self, name, annotation):
        self.Reference(annotation, self.annotationName(name))

    def refAnnotation(self, name):
        internalname = self.annotationName(name)
        return PDFObjectReference(internalname)

    def addShading(self, shading):
         name = "Sh%d" % self.shadingCounter
         self.Reference(shading, name)
         self.shadingCounter += 1
         return name

    def addColor(self,cmyk):
        sname = cmyk.spotName
        if not sname:
            if cmyk.cyan==0 and cmyk.magenta==0 and cmyk.yellow==0:
                sname = 'BLACK'
            elif cmyk.black==0 and cmyk.magenta==0 and cmyk.yellow==0:
                sname = 'CYAN'
            elif cmyk.cyan==0 and cmyk.black==0 and cmyk.yellow==0:
                sname = 'MAGENTA'
            elif cmyk.cyan==0 and cmyk.magenta==0 and cmyk.black==0:
                sname = 'YELLOW'
            if not sname:
                raise ValueError("CMYK colour %r used without a spotName" % cmyk)
            else:
                cmyk = cmyk.clone(spotName = sname)
        name = PDFName(sname)[1:]
        if name not in self.idToObject:
            sep = PDFSeparationCMYKColor(cmyk).value()  #PDFArray([/Separation /name /DeviceCMYK tint_tf])
            self.Reference(sep,name)
        return name,sname

    def setTitle(self, title):
        "embeds in PDF file"
        if title is None:
            self.info.title = '(anonymous)'
        else:
            self.info.title = title

    def setAuthor(self, author):
        "embedded in PDF file"
        #allow resetting to clear it
        if author is None:
            self.info.author = '(anonymous)'
        else:
            self.info.author = author

    def setSubject(self, subject):
        "embeds in PDF file"

        #allow resetting to clear it
        if subject is None:
            self.info.subject = '(unspecified)'
        else:
            self.info.subject = subject

    def setCreator(self, creator):
        "embeds in PDF file"

        #allow resetting to clear it
        if creator is None:
            self.info.creator = '(unspecified)'
        else:
            self.info.creator = creator

    def setKeywords(self, keywords):
        "embeds a string containing keywords in PDF file"

        #allow resetting to clear it but ensure it's a string
        if keywords is None:
            self.info.keywords = ''
        else:
            self.info.keywords = keywords

    def setDateFormatter(self, dateFormatter):
        self.info._dateFormatter = dateFormatter

    def getAvailableFonts(self):
        fontnames = self.fontMapping.keys()
        # the standard 14 are also always available! (even if not initialized yet)
        import _fontdata
        for name in _fontdata.standardFonts:
            if name not in fontnames:
                fontnames.append(name)
        fontnames.sort()
        return fontnames

    def format(self):
        # register the Catalog/INfo and then format the objects one by one until exhausted
        # (possible infinite loop if there is a bug that continually makes new objects/refs...)
        # Prepare encryption
        self.encrypt.prepare(self)
        cat = self.Catalog
        info = self.info
        self.Reference(self.Catalog)
        self.Reference(self.info)
        # register the encryption dictionary if present
        encryptref = None
        encryptinfo = self.encrypt.info()
        if encryptinfo:
            encryptref = self.Reference(encryptinfo)
        # make std fonts (this could be made optional
        counter = 0 # start at first object (object 1 after preincrement)
        ids = [] # the collection of object ids in object number order
        numbertoid = self.numberToId
        idToNV = self.idToObjectNumberAndVersion
        idToOb = self.idToObject
        idToOf = self.idToOffset
        ### note that new entries may be "appended" DURING FORMATTING
        done = None
        File = PDFFile(self._pdfVersion) # output collector
        while done is None:
            counter += 1 # do next object...
            if counter in numbertoid:
                id = numbertoid[counter]
                #printidToOb
                obj = idToOb[id]
                IO = PDFIndirectObject(id, obj)
                # register object number and version
                #encrypt.register(id,
                IOf = IO.format(self)
                # add a comment to the PDF output
                if not rl_config.invariant and DoComments:
                    try:
                        classname = obj.__class__.__name__
                    except:
                        classname = repr(obj)
                    File.add("%% %s: class %s %s" % (repr(id), classname[:50], LINEEND))
                offset = File.add(IOf)
                idToOf[id] = offset
                ids.append(id)
            else:
                done = 1
        # sanity checks (must happen AFTER formatting)
        lno = len(numbertoid)
        if counter-1!=lno:
            raise ValueError, "counter %s doesn't match number to id dictionary %s" %(counter, lno)
        # now add the xref
        xref = PDFCrossReferenceTable()
        xref.addsection(0, ids)
        xreff = xref.format(self)
        xrefoffset = File.add(xreff)
        # now add the trailer
        trailer = PDFTrailer(
            startxref = xrefoffset,
            Size = lno+1,
            Root = self.Reference(cat),
            Info = self.Reference(info),
            Encrypt = encryptref,
            ID = self.ID(),
            )
        trailerf = trailer.format(self)
        File.add(trailerf)
        # return string format for pdf file
        return File.format(self)

    def hasForm(self, name):
        """test for existence of named form"""
        internalname = xObjectName(name)
        return internalname in self.idToObject

    def getFormBBox(self, name, boxType="MediaBox"):
        """get the declared bounding box of the form as a list.
        If you specify a different PDF box definition (e.g. the
        ArtBox) and it has one, that's what you'll get."""
        internalname = xObjectName(name)
        if internalname in self.idToObject:
            theform = self.idToObject[internalname]
            if hasattr(theform,'_extra_pageCatcher_info'):
                return theform._extra_pageCatcher_info[boxType]
            if isinstance(theform, PDFFormXObject):
                # internally defined form
                return theform.BBoxList()
            elif isinstance(theform, PDFStream):
                # externally defined form
                return list(theform.dictionary.dict[boxType].sequence)
            else:
                raise ValueError, "I don't understand the form instance %s" % repr(name)

    def getXObjectName(self, name):
        """Lets canvas find out what form is called internally.
        Never mind whether it is defined yet or not."""
        return xObjectName(name)

    def xobjDict(self, formnames):
        """construct an xobject dict (for inclusion in a resource dict, usually)
           from a list of form names (images not yet supported)"""
        D = {}
        for name in formnames:
            internalname = xObjectName(name)
            reference = PDFObjectReference(internalname)
            D[internalname] = reference
        #print "xobjDict D", D
        return PDFDictionary(D)

    def Reference(self, object, name=None, InstanceType=InstanceType):
        ### note references may "grow" during the final formatting pass: don't use d.keys()!
        # don't make references to other references, or non instances, unless they are named!
        #print"object type is ", type(object)
        iob = hasattr(object,'__PDFObject__')
        idToObject = self.idToObject
        if name is None and (not iob or object.__class__ is PDFObjectReference):
            return object
        if hasattr(object, __InternalName__):
            # already registered
            intname = object.__InternalName__
            if name is not None and name!=intname:
                raise ValueError, "attempt to reregister object %s with new name %s" % (
                    repr(intname), repr(name))
            if intname not in idToObject:
                raise ValueError, "object named but not registered"
            return PDFObjectReference(intname)
        # otherwise register the new object
        objectcounter = self.objectcounter = self.objectcounter+1
        if name is None:
            name = "R"+repr(objectcounter)
        if name in idToObject:
            other = idToObject[name]
            if other!=object:
                raise ValueError, "redefining named object: "+repr(name)
            return PDFObjectReference(name)
        if iob:
            object.__InternalName__ = name
        #print "name", name, "counter", objectcounter
        self.idToObjectNumberAndVersion[name] = (objectcounter, 0)
        self.numberToId[objectcounter] = name
        idToObject[name] = object
        return PDFObjectReference(name)

### chapter 4 Objects
PDFtrue = "true"
PDFfalse = "false"
PDFnull = "null"

class PDFText:
    __PDFObject__ = True
    def __init__(self, t):
        self.t = t
    def format(self, document):
        result = binascii.hexlify(document.encrypt.encode(self.t))
        return "<%s>" % result
    def __str__(self):
        dummydoc = DummyDoc()
        return self.format(dummydoc)

def PDFnumber(n):
    return n

import re
_re_cleanparens=re.compile('[^()]')
del re
def _isbalanced(s):
    '''test whether a string is balanced in parens'''
    s = _re_cleanparens.sub('',s)
    n = 0
    for c in s:
        if c=='(': n+=1
        else:
            n -= 1
            if n<0: return 0
    return not n and 1 or 0

def _checkPdfdoc(utext):
    '''return true if no Pdfdoc encoding errors'''
    try:
        utext.encode('pdfdoc')
        return 1
    except UnicodeEncodeError, e:
        return 0

class PDFString:
    __PDFObject__ = True
    def __init__(self, s, escape=1, enc='auto'):
        '''s can be unicode/utf8 or a PDFString
        if escape is true then the output will be passed through escape
        if enc is raw then the string will be left alone
        if enc is auto we'll try and automatically adapt to utf_16_be if the
        effective string is not entirely in pdfdoc
        '''
        if isinstance(s,PDFString):
            self.s = s.s
            self.escape = s.escape
            self.enc = s.enc
        else:
            self.s = s
            self.escape = escape
            self.enc = enc
    def format(self, document):
        s = self.s
        enc = getattr(self,'enc','auto')
        if type(s) is str:
            if enc is 'auto':
                try:
                    u = s.decode(s.startswith(codecs.BOM_UTF16_BE) and 'utf16' or 'utf8')
                    if _checkPdfdoc(u):
                        s = u.encode('pdfdoc')
                    else:
                        s = codecs.BOM_UTF16_BE+u.encode('utf_16_be')
                except:
                    try:
                        s.decode('pdfdoc')
                    except:
                        import sys
                        print >>sys.stderr, 'Error in',repr(s)
                        raise
        elif type(s) is unicode:
            if enc is 'auto':
                if _checkPdfdoc(s):
                    s = s.encode('pdfdoc')
                else:
                    s = codecs.BOM_UTF16_BE+s.encode('utf_16_be')
            else:
                s = codecs.BOM_UTF16_BE+s.encode('utf_16_be')
        else:
            raise ValueError('PDFString argument must be str/unicode not %s' % type(s))

        escape = getattr(self,'escape',1)
        if not isinstance(document.encrypt,NoEncryption):
            s = document.encrypt.encode(s)
            escape = 1
        if escape:
            try:
                es = "(%s)" % pdfutils._escape(s)
            except:
                raise ValueError("cannot escape %s %s" % (s, repr(s)))
            if escape&2:
                es = es.replace('\\012','\n')
            if escape&4 and _isbalanced(s):
                es = es.replace('\\(','(').replace('\\)',')')
            return es
        else:
            return '(%s)' % s
    def __str__(self):
        return "(%s)" % pdfutils._escape(self.s)

def PDFName(data,lo=chr(0x21),hi=chr(0x7e)):
    # might need to change this to class for encryption
    #  NOTE: RESULT MUST ALWAYS SUPPORT MEANINGFUL COMPARISONS (EQUALITY) AND HASH
    # first convert the name
    L = list(data)
    for i,c in enumerate(L):
        if c<lo or c>hi or c in "%()<>{}[]#":
            L[i] = "#"+hex(ord(c))[2:] # forget the 0x thing...
    return "/"+(''.join(L))

class PDFDictionary:
    __PDFObject__ = True
    multiline = LongFormat
    def __init__(self, dict=None):
        """dict should be namestring to value eg "a": 122 NOT pdfname to value NOT "/a":122"""
        if dict is None:
            self.dict = {}
        else:
            self.dict = dict.copy()
    def __setitem__(self, name, value):
        self.dict[name] = value
    def __getitem__(self, a):
        return self.dict[a]
    def __contains__(self,a):
        return a in self.dict
    def Reference(self, name, document):
        self.dict[name] = document.Reference(self.dict[name])
    def format(self, document,IND=LINEEND+' '):
        dict = self.dict
        try:
            keys = dict.keys()
        except:
            print repr(dict)
            raise
        keys.sort()
        L = [(format(PDFName(k),document)+" "+format(dict[k],document)) for k in keys]
        if self.multiline:
            L = IND.join(L)
        else:
            # break up every 6 elements anyway
            t=L.insert
            for i in xrange(6, len(L), 6):
                t(i,LINEEND)
            L = " ".join(L)
        return "<< %s >>" % L

    def copy(self):
        return PDFDictionary(self.dict)

    def normalize(self):
        #normalize the names to use RL standard ie Name not /Name
        D = self.dict
        K = [k for k in D.iterkeys() if k.startswith('/')]
        for k in K:
            D[k[1:]] = D.pop(k)

class checkPDFNames:
    def __init__(self,*names):
        self.names = map(PDFName,names)
    def __call__(self,value):
        if not value.startswith('/'):
            value=PDFName(value)
        if value in self.names:
            return value

def checkPDFBoolean(value):
    if value in ('true','false'): return value

class CheckedPDFDictionary(PDFDictionary):
    validate = {}
    def __init__(self,dict=None,validate=None):
        PDFDictionary.__init__(self,dict)
        if validate: self.validate = validate

    def __setitem__(self,name,value):
        if name not in self.validate:
            raise ValueError('invalid key, %r' % name)
        cvalue = self.validate[name](value)
        if cvalue is None:
            raise ValueError('Bad value %r for key %r' % (value,name))
        PDFDictionary.__setitem__(self,name,cvalue)

class ViewerPreferencesPDFDictionary(CheckedPDFDictionary):
    validate=dict(
                HideToolbar=checkPDFBoolean,
                HideMenubar=checkPDFBoolean,
                HideWindowUI=checkPDFBoolean,
                FitWindow=checkPDFBoolean,
                CenterWindow=checkPDFBoolean,
                DisplayDocTitle=checkPDFBoolean,    #contributed by mark Erbaugh
                NonFullScreenPageMode=checkPDFNames(*'UseNone UseOutlines UseThumbs UseOC'.split()),
                Direction=checkPDFNames(*'L2R R2L'.split()),
                ViewArea=checkPDFNames(*'MediaBox CropBox BleedBox TrimBox ArtBox'.split()),
                ViewClip=checkPDFNames(*'MediaBox CropBox BleedBox TrimBox ArtBox'.split()),
                PrintArea=checkPDFNames(*'MediaBox CropBox BleedBox TrimBox ArtBox'.split()),
                PrintClip=checkPDFNames(*'MediaBox CropBox BleedBox TrimBox ArtBox'.split()),
                PrintScaling=checkPDFNames(*'None AppDefault'.split()),
                )

# stream filters are objects to support round trip and
# possibly in the future also support parameters
class PDFStreamFilterZCompress:
    pdfname = "FlateDecode"
    def encode(self, text):
        from reportlab.lib.utils import import_zlib
        zlib = import_zlib()
        if not zlib: raise ImportError, "cannot z-compress zlib unavailable"
        return zlib.compress(text)
    def decode(self, encoded):
        from reportlab.lib.utils import import_zlib
        zlib = import_zlib()
        if not zlib: raise ImportError, "cannot z-decompress zlib unavailable"
        return zlib.decompress(encoded)

# need only one of these, unless we implement parameters later
PDFZCompress = PDFStreamFilterZCompress()

class PDFStreamFilterBase85Encode:
    pdfname = "ASCII85Decode"
    def encode(self, text):
        from pdfutils import _AsciiBase85Encode, _wrap
        text = _AsciiBase85Encode(text)
        if rl_config.wrapA85:
            text = _wrap(text)
        return text
    def decode(self, text):
        from pdfutils import _AsciiBase85Decode
        return _AsciiBase85Decode(text)

# need only one of these too
PDFBase85Encode = PDFStreamFilterBase85Encode()

STREAMFMT = ("%(dictionary)s%(LINEEND)s" # dictionary
             "stream" # stream keyword
             "%(LINEEND)s" # a line end (could be just a \n)
             "%(content)s" # the content, with no lineend
             "endstream%(LINEEND)s" # the endstream keyword
             )
class PDFStream:
    '''set dictionary elements explicitly stream.dictionary[name]=value'''
    __PDFObject__ = True
    ### compression stuff not implemented yet
    __RefOnly__ = 1 # must be at top level
    def __init__(self, dictionary=None, content=None, filters=None):
        if dictionary is None:
            dictionary = PDFDictionary()
        self.dictionary = dictionary
        self.content = content
        self.filters = filters
    def format(self, document):
        dictionary = self.dictionary
        # copy it for modification
        dictionary = PDFDictionary(dictionary.dict.copy())
        content = self.content
        filters = self.filters
        if self.content is None:
            raise ValueError, "stream content not set"
        if filters is None:
            filters = document.defaultStreamFilters
        # only apply filters if they haven't been applied elsewhere
        if filters is not None and "Filter" not in dictionary.dict:
            # apply filters in reverse order listed
            rf = list(filters)
            rf.reverse()
            fnames = []
            for f in rf:
                #print "*****************content:"; print repr(content[:200])
                #print "*****************filter", f.pdfname
                content = f.encode(content)
                fnames.insert(0, PDFName(f.pdfname))
            #print "*****************finally:"; print content[:200]
            #print "****** FILTERS", fnames
            #stop
            dictionary["Filter"] = PDFArray(fnames)
        # "stream encoding is done after all filters have been applied"
        content = document.encrypt.encode(content)
        fc = format(content, document)
        #print "type(content)", type(content), len(content), type(self.dictionary)
        lc = len(content)
        #if fc!=content: burp
        # set dictionary length parameter
        dictionary["Length"] = lc
        fd = format(dictionary, document)
        sdict = LINEENDDICT.copy()
        sdict["dictionary"] = fd
        sdict["content"] = fc
        return STREAMFMT % sdict

def teststream(content=None):
    #content = "" # test
    if content is None:
        content = teststreamcontent
    content = string.strip(content)
    content = string.replace(content, "\n", LINEEND) + LINEEND
    S = PDFStream(content = content,
                    filters=rl_config.useA85 and [PDFBase85Encode,PDFZCompress] or [PDFZCompress])
    # nothing else needed...
    S.__Comment__ = "test stream"
    return S

teststreamcontent = """
1 0 0 1 0 0 cm BT /F9 12 Tf 14.4 TL ET
1.00 0.00 1.00 rg
n 72.00 72.00 432.00 648.00 re B*
"""
class PDFArray:
    __PDFObject__ = True
    multiline = LongFormat
    _ZLIST = list(9*' ')+[LINEEND]
    def __init__(self, sequence):
        self.sequence = list(sequence)
    def References(self, document):
        """make all objects in sequence references"""
        self.sequence = map(document.Reference, self.sequence)
    def format(self, document, IND=LINEEND+' '):
        L = [format(e, document) for e in self.sequence]
        if self.multiline:
            L = IND.join(L)
        else:
            n=len(L)
            if n>10:
                # break up every 10 elements anyway
                m,r = divmod(n,10)
                L = ''.join([l+z for l,z in zip(L,m*self._ZLIST+list(r*' '))])
                L = L.strip()
            else:
                L = ' '.join(L)
        return "[ %s ]" % L

class PDFArrayCompact(PDFArray):
    multiline=False

INDIRECTOBFMT = "%(n)s %(v)s obj%(LINEEND)s%(content)s%(CLINEEND)sendobj%(LINEEND)s"
class PDFIndirectObject:
    __PDFObject__ = True
    __RefOnly__ = 1
    def __init__(self, name, content):
        self.name = name
        self.content = content
    def format(self, document):
        name = self.name
        n, v = document.idToObjectNumberAndVersion[name]
        # set encryption parameters
        document.encrypt.register(n, v)
        fcontent = format(self.content, document, toplevel=1) # yes this is at top level
        D = LINEENDDICT.copy()
        D["n"] = n
        D["v"] = v
        D["content"] = fcontent
        D['CLINEEND'] = (LINEEND,'')[fcontent.endswith(LINEEND)]
        return INDIRECTOBFMT % D

class PDFObjectReference:
    __PDFObject__ = True
    def __init__(self, name):
        self.name = name
    def format(self, document):
        try:
            return "%s %s R" % document.idToObjectNumberAndVersion[self.name]
        except:
            raise KeyError, "forward reference to %s not resolved upon final formatting" % repr(self.name)

### chapter 5
# Following Ken Lunde's advice and the PDF spec, this includes
# some high-order bytes.  I chose the characters for Tokyo
# in Shift-JIS encoding, as these cannot be mistaken for
# any other encoding, and we'll be able to tell if something
# has run our PDF files through a dodgy Unicode conversion.
PDFHeader = (
"%%PDF-%s.%s"+LINEEND+
"%%\223\214\213\236 ReportLab Generated PDF document http://www.reportlab.com"+LINEEND)

class PDFFile:
    __PDFObject__ = True
    ### just accumulates strings: keeps track of current offset
    def __init__(self,pdfVersion=PDF_VERSION_DEFAULT):
        self.strings = []
        self.write = self.strings.append
        self.offset = 0
        self.add(PDFHeader % pdfVersion)

    def closeOrReset(self):
        pass

    def add(self, s):
        """should be constructed as late as possible, return position where placed"""
        result = self.offset
        self.offset = result+len(s)
        self.write(s)
        return result
    def format(self, document):
        strings = map(str, self.strings) # final conversion, in case of lazy objects
        return string.join(strings, "")

XREFFMT = '%0.10d %0.5d n'

class PDFCrossReferenceSubsection:
    __PDFObject__ = True
    def __init__(self, firstentrynumber, idsequence):
        self.firstentrynumber = firstentrynumber
        self.idsequence = idsequence
    def format(self, document):
        """id sequence should represent contiguous object nums else error. free numbers not supported (yet)"""
        firstentrynumber = self.firstentrynumber
        idsequence = self.idsequence
        entries = list(idsequence)
        nentries = len(idsequence)
        # special case: object number 0 is always free
        taken = {}
        if firstentrynumber==0:
            taken[0] = "standard free entry"
            nentries = nentries+1
            entries.insert(0, "0000000000 65535 f")
        idToNV = document.idToObjectNumberAndVersion
        idToOffset = document.idToOffset
        lastentrynumber = firstentrynumber+nentries-1
        for id in idsequence:
            (num, version) = idToNV[id]
            if num in taken:
                raise ValueError, "object number collision %s %s %s" % (num, repr(id), repr(taken[id]))
            if num>lastentrynumber or num<firstentrynumber:
                raise ValueError, "object number %s not in range %s..%s" % (num, firstentrynumber, lastentrynumber)
            # compute position in list
            rnum = num-firstentrynumber
            taken[num] = id
            offset = idToOffset[id]
            entries[num] = XREFFMT % (offset, version)
        # now add the initial line
        firstline = "%s %s" % (firstentrynumber, nentries)
        entries.insert(0, firstline)
        # make sure it ends with a LINEEND
        entries.append("")
        if LINEEND=="\n" or LINEEND=="\r":
            reflineend = " "+LINEEND # as per spec
        elif LINEEND=="\r\n":
            reflineend = LINEEND
        else:
            raise ValueError, "bad end of line! %s" % repr(LINEEND)
        return string.join(entries, LINEEND)

class PDFCrossReferenceTable:
    __PDFObject__ = True

    def __init__(self):
        self.sections = []
    def addsection(self, firstentry, ids):
        section = PDFCrossReferenceSubsection(firstentry, ids)
        self.sections.append(section)
    def format(self, document):
        sections = self.sections
        if not sections:
            raise ValueError, "no crossref sections"
        L = ["xref"+LINEEND]
        for s in self.sections:
            fs = format(s, document)
            L.append(fs)
        return string.join(L, "")

TRAILERFMT = ("trailer%(LINEEND)s"
              "%(dict)s%(LINEEND)s"
              "startxref%(LINEEND)s"
              "%(startxref)s%(LINEEND)s"
              "%(PERCENT)s%(PERCENT)sEOF%(LINEEND)s")

class PDFTrailer:
    __PDFObject__ = True

    def __init__(self, startxref, Size=None, Prev=None, Root=None, Info=None, ID=None, Encrypt=None):
        self.startxref = startxref
        if Size is None or Root is None:
            raise ValueError, "Size and Root keys required"
        dict = self.dict = PDFDictionary()
        for (n,v) in [("Size", Size), ("Prev", Prev), ("Root", Root),
                      ("Info", Info), ("ID", ID), ("Encrypt", Encrypt)]:
            if v is not None:
                dict[n] = v
    def format(self, document):
        fdict = format(self.dict, document)
        D = LINEENDDICT.copy()
        D["dict"] = fdict
        D["startxref"] = self.startxref
        return TRAILERFMT % D

#### XXXX skipping incremental update,
#### encryption

#### chapter 6, doc structure

class PDFCatalog:
    __PDFObject__ = True
    __Comment__ = "Document Root"
    __RefOnly__ = 1
    # to override, set as attributes
    __Defaults__ = {"Type": PDFName("Catalog"),
                "PageMode": PDFName("UseNone"),
                }
    __NoDefault__ = string.split("""
        Dests Outlines Pages Threads AcroForm Names OpenAction PageMode URI
        ViewerPreferences PageLabels PageLayout JavaScript StructTreeRoot SpiderInfo"""
                                 )
    __Refs__ = __NoDefault__ # make these all into references, if present

    def format(self, document):
        self.check_format(document)
        defaults = self.__Defaults__
        Refs = self.__Refs__
        D = {}
        for k in defaults.keys():
            default = defaults[k]
            v = None
            if hasattr(self, k) and getattr(self,k) is not None:
                v = getattr(self, k)
            elif default is not None:
                v = default
            if v is not None:
                D[k] = v
        for k in self.__NoDefault__:
            if hasattr(self, k):
                v = getattr(self,k)
                if v is not None:
                    D[k] = v
        # force objects to be references where required
        for k in Refs:
            if k in D:
                #print"k is", k, "value", D[k]
                D[k] = document.Reference(D[k])


# -*- coding: utf-8 -*-
#
# Atizo - The Open Innovation Platform
# http://www.atizo.com/
#
# Copyright (c) 2008-2010 Atizo AG. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
from __future__ import unicode_literals
from django.conf.urls.defaults import patterns, url
import datetime
from django.http import HttpResponse
from django.utils import simplejson
import time
from django.utils.functional import wraps
from django.template.loader import render_to_string
from django.db.models.aggregates import Count
from django.utils.datastructures import SortedDict
from googleanalytics import Connection
from django.conf import settings
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse
import logging
from django.template.defaultfilters import slugify

logger = logging.getLogger(__name__)

class BaseStatistics(object):

    name = _('Statistiken')
    render_template = 'djangojames/statistics.html'
    default_options = {}

    def __init__( self, *args, **kwargs):
        self.prefix = slugify(self.__class__.__name__)

    def get_statsmethod(self):
        return [(m.replace('stats_', ''), getattr(self,m)) for m in dir(self) if m.startswith('stats_')]

    def __unicode__(self):
        return self.get_html()

    def get_urls(self):
        self._urls = None

        def _add_pattern(pattern):
            if self._urls:
                self._urls += pattern
            else:
                self._urls = pattern

        for m in self.get_statsmethod():
            _add_pattern(patterns("", url("^%s/$" % m[0], m[1], name='%s' % m[0]),))

        if hasattr(self, 'summary'):
            _add_pattern(patterns("", url("^%s/$" % 'summary', self.summary, name='%s' % 'summary'),))

        return self._urls

    @classmethod
    def datetime_to_stamp(cls, a_datetime):

        if isinstance(a_datetime, unicode):
            a_datetime = BaseStatistics.str_to_date(a_datetime)

        a_datetime = a_datetime + datetime.timedelta(days=1)
        if isinstance(a_datetime, datetime.datetime):
            return time.mktime(a_datetime.replace(hour=0, minute=0, second=0, microsecond=0).timetuple()) * 1000
        else:
            return time.mktime(a_datetime.timetuple()) * 1000

    @classmethod
    def str_to_date(cls, astring, last_second=False, default=None):
        """
        parses a date 'yy-mm-dd' (eg 2011-05-24)
        """
        if astring:
            l = [int(t) for t in astring.split('-')]
            if len(l) == 3:
                if last_second:
                    return datetime.datetime(l[0], l[1], l[2], 23, 59, 59)
                else:
                    return datetime.datetime(l[0], l[1], l[2])

        return default

    def get_html(self):
        context = {'stats_config': simplejson.dumps(self.get_statsconfig())}
        context['earliest_date'] = '-3years'
        context['latest_date'] = '+2years'
        context['choice_per_col'] = '3'
        context['prefix'] = self.prefix

        if hasattr(self, 'get_extra_config'):
            context.update(self.get_extra_config())

        return render_to_string(self.render_template, context)

    def get_statsconfig(self):
        config_lists = []
        for m in self.get_statsmethod():
            name = unicode(getattr(m[1], "name", m[0]))
            options = getattr(m[1], 'options', self.default_options)
            modes = getattr(m[1], 'modes', {})
            type = getattr(m[1], 'type', {'name': 'line', 'unity': 'linear'})
            selected = getattr(m[1], "selected", False)
            multiple_series = getattr(m[1], "multiple_series", False)
            index = getattr(m[1], "index", 0)
            config_lists.append({'name':m[0], 'index':index, 'dataurl': self.get_data_url(m[0]), 'multiple_series': multiple_series,
                                 'label': name, 'options': options, 'selected': selected, 'modes': modes, 'type': type})

        config_lists=sorted(config_lists, key=lambda aconfig: aconfig['index'])
        configs = SortedDict()
        for aconfig in config_lists:
            configs['%s_%s' % (aconfig['name'], self.prefix)] = aconfig

        return configs

    def get_data_url(self, method):
        return reverse(method)

def apply_days_range(days):
    def decorator(view_func):
        def _wrapped_view_func(instance, request, *args, **kwargs):
            try:

                today = datetime.date.today()
                startdate = today - datetime.timedelta(days=days)

                from_date = BaseGoogleAnalyticsStatistics.str_to_date(request.GET.get('from', ''), default=startdate)
                to_date = BaseGoogleAnalyticsStatistics.str_to_date(request.GET.get('to', ''), last_second=True, default=today)
                kwargs['from_date'] = from_date
                kwargs['to_date'] = to_date
                data = view_func(instance, request, *args, **kwargs)
            except Exception as e:
                logger.exception('Error: Could not get statistic')
                data = []
            return HttpResponse(simplejson.dumps(data), mimetype='application/javascript')

        return wraps(view_func)(_wrapped_view_func)
    return decorator

class BaseSequenceStatistics(BaseStatistics):

    default_options = {'lines': { 'show': True }}

    @classmethod
    def get_null_secquence(cls, start_date, end_date):

        if start_date is None or end_date is None:
            return {}

        currentdate = datetime.date(start_date.year, start_date.month, start_date.day)
        enddate = datetime.date(end_date.year, end_date.month, end_date.day)
        timeline_dict = {}
        while currentdate <= enddate:
            timeline_dict[cls.datetime_to_stamp(currentdate)] = 0
            currentdate += datetime.timedelta(days=1)
        return timeline_dict

    def handle_sequence(self, queryset, mode, from_date, to_date, timeline_dict, datefield_name='created'):

        if from_date is None or to_date is None:
            return timeline_dict

        kwargs = {'%s__gte' % datefield_name: from_date,
                  '%s__lte' % datefield_name: to_date + datetime.timedelta(days=1)}
        new_name = 'date_%s' % datefield_name

        # count all objects per datefield_name
        queryset = queryset.filter(**kwargs).extra({new_name:"date(%s.%s)" % (queryset.model._meta.db_table,datefield_name)}) \
        .values(new_name).annotate(created_count=Count('id')).order_by(new_name)

        for idict in queryset:
            stamp = self.datetime_to_stamp(idict[new_name])
            timeline_dict[stamp] = idict['created_count']

        return timeline_dict

    def datetime_sequence(self, queryset, from_date, to_date, datefield_name='created'):
        timeline_dict = self.get_null_secquence(from_date, to_date)

        kwargs = {'%s__gte' % datefield_name: from_date,
                  '%s__lt' % datefield_name: to_date + datetime.timedelta(days=1)}
        new_name = 'date_%s' % datefield_name

        # count all objects per datefield_name
        queryset = queryset.filter(**kwargs).extra({new_name:"date(%s.%s)" % (queryset.model._meta.db_table,datefield_name)}) \
        .values(new_name).annotate(created_count=Count('id')).order_by(new_name)

        for idict in queryset:
            stamp = self.datetime_to_stamp(idict[new_name])
            timeline_dict[stamp] = idict['created_count']

        data = []
        for day, count in timeline_dict.items():
            data.append([day, count])

        return data

class BaseFractionStatistics(BaseStatistics):

    def _flatten(self, data):
        if data and len(data) == 1:
            data = data[0]
            if isinstance(data, basestring):
                if data == '(not set)':
                    data = _('(Unbekannt)')
            return data
        else:
            return data

    def normalize(self, keyvalue_list_mix, calc_percent=False, summerize=True, order=True):

        def value_compare(x, y):
                cmp = x[1] < y[1]
                if cmp:
                    return 1
                else:
                    return -1

        data_norm = []
        if len(keyvalue_list_mix) == 0:
            return data_norm

        keyvalue_list = []

        for k,v in keyvalue_list_mix:
            if type(v) not in (tuple, list):
                v = [v]
            keyvalue_list.append([k,v])

        totals = [0 for m in keyvalue_list[0][1]]

        for item in keyvalue_list:
            for idx, val in enumerate(totals):
                totals[idx] = val + item[1][idx]

            if not calc_percent:
                data_norm.append([self._flatten(item[0]), self._flatten(item[1])])

        if calc_percent:
            totals_per = [100.0/t for t in totals]
            for item in keyvalue_list:
                for idx, val in enumerate(totals_per):
                    item[1][idx] = round(item[1][idx] * val,1)

                data_norm.append([self._flatten(item[0]), self._flatten(item[1])])

        if order:
            data_norm = sorted(data_norm, cmp=value_compare)

        return data_norm

class BaseGoogleAnalyticsStatistics(BaseFractionStatistics):
    # http://code.google.com/intl/de-CH/apis/analytics/docs/gdata/gdataReferenceDimensionsMetrics.html

    def __init__(self, *args, **kwargs):
        super(BaseGoogleAnalyticsStatistics, self).__init__()
        self._account = None
        self.filters = []

    def get_account(self):

        if not self._account:
            connection = Connection(settings.GOOGLE_ACCOUNT, settings.GOOGLE_ACCOUNT_PASSWORD)
            self._account = connection.get_account(settings.GOOGLE_ACCOUNT_PROFILE_ID)

        return self._account

    def get_data(self, start_date=None, end_date=None, dimensions=None, metrics=None, sort=None, filters=None, calc_percent=False, summerize=True):

        try:
            if start_date > end_date:
                start_date = end_date

            data = self.get_account().get_data(start_date=start_date, end_date=end_date, dimensions=dimensions, metrics=metrics, sort=sort, filters=filters).list

            if dimensions and 'date' in dimensions:
                index = dimensions.index('date')
                for item in data:
                    data_str = item[0][index]
                    item[0][index] = self.datetime_to_stamp(datetime.datetime(int(data_str[:4]),  int(data_str[4:6]), int(data_str[6:])))
        except Exception as e:
            logger.error('Could not get Google Analytics statistic: %s' % str(e))
            data = []

        return self.normalize(data, calc_percent, summerize)

    def _get_visitor(self, from_date, to_date):
        return self.get_data(start_date=from_date, end_date=to_date, dimensions=['date'], metrics=['visitors',], sort=['-date',], filters=self.filters, calc_percent=False)

    def _get_country(self, from_date, to_date):
        return self.get_data(start_date=from_date, end_date=to_date, dimensions=['country'], metrics=['visitors'], sort=['-visitors',], filters=self.filters, calc_percent=True)

    def _get_browser(self, from_date, to_date):
        return self.get_data(start_date=from_date, end_date=to_date, dimensions=['browser'], metrics=['visitors',], sort=['-visitors',], filters=self.filters, calc_percent=True)

    def _get_operatingsystem(self, from_date, to_date):
        return self.get_data(start_date=from_date, end_date=to_date, dimensions=['operatingsystem'], metrics=['visitors',], sort=['-visitors',], filters=self.filters, calc_percent=True)

    def _get_visitortype(self, from_date, to_date):
        return self.get_data(start_date=from_date, end_date=to_date, dimensions=['visitortype'], metrics=['visitors',], sort=['-visitors',], filters=self.filters, calc_percent=True)

class VisitorsStatistics(BaseSequenceStatistics,
                          BaseGoogleAnalyticsStatistics):

    name = _('Besucher')

    @apply_days_range(days=120)
    def stats_visitor(self, request, from_date, to_date):
        return self._get_visitor(from_date, to_date)
    stats_visitor.name = _('Eindeutige Besucher')
    stats_visitor.index = 20
    stats_visitor.options = {'xAxis': { 'type': 'datetime'}}
    stats_visitor.type = {'name': 'line', 'unity': 'datetime'}
    stats_visitor.selected = True

    @apply_days_range(days=500)
    def stats_country(self, request, from_date, to_date):
        return self._get_country(from_date, to_date)
    stats_country.name = _('Herkunftsland')
    stats_country.index = 30
    stats_country.type = {'name': 'pie', 'unity': 'percent'}

    @apply_days_range(days=500)
    def stats_browser(self, request, from_date, to_date):
        return self._get_browser(from_date, to_date)
    stats_browser.name = _('Browser')
    stats_browser.index = 40