#!/usr/bin/env python
"""linuxfd: Python interface for the Linux system calls eventfd/signalfd/timerfd
Copyright (C) 2014-2020 Frank Abelbeck <frank.abelbeck@googlemail.com>

linuxfd is free software: you can redistribute it and/or modify it under the
terms of the GNU Lesser General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your option)
any later version.

linuxfd is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public License
along with linuxfd.  If not, see <http://www.gnu.org/licenses/>.

Written in Python V3."""

# import helper modules for the syscalls and constants
import linuxfd.eventfd_c
import linuxfd.signalfd_c
import linuxfd.timerfd_c
import linuxfd.inotify_c

# modules used for raising own OSError
import errno,os

# define constants
# inotify event mask constants (inotify.add(), inotify.read())
IN_ACCESS        =  inotify_c.IN_ACCESS
IN_ATTRIB        = inotify_c.IN_ATTRIB
IN_CLOSE_WRITE   = inotify_c.IN_CLOSE_WRITE
IN_CLOSE_NOWRITE = inotify_c.IN_CLOSE_NOWRITE
IN_CREATE        = inotify_c.IN_CREATE
IN_DELETE        = inotify_c.IN_DELETE
IN_DELETE_SELF   = inotify_c.IN_DELETE_SELF
IN_MODIFY        = inotify_c.IN_MODIFY
IN_MOVE_SELF     = inotify_c.IN_MOVE_SELF
IN_MOVED_FROM    = inotify_c.IN_MOVED_FROM
IN_MOVED_TO      = inotify_c.IN_MOVED_TO
IN_OPEN          = inotify_c.IN_OPEN
IN_ALL_EVENTS    = inotify_c.IN_ALL_EVENTS
IN_MOVE          = inotify_c.IN_MOVE
IN_CLOSE         = inotify_c.IN_CLOSE
# additional inotify event mask constants for inotify.add()
IN_DONT_FOLLOW   = inotify_c.IN_DONT_FOLLOW
IN_EXCL_UNLINK   = inotify_c.IN_EXCL_UNLINK
IN_ONESHOT       = inotify_c.IN_ONESHOT
IN_ONLYDIR       = inotify_c.IN_ONLYDIR
# additional inotify event mask constants for inotify.read()
IN_IGNORED       = inotify_c.IN_IGNORED
IN_ISDIR         = inotify_c.IN_ISDIR
IN_Q_OVERFLOW    = inotify_c.IN_Q_OVERFLOW
IN_UNMOUNT       = inotify_c.IN_UNMOUNT

class eventfd:
	"""Class to manage a file descriptor for event notification.

An event file descriptor is created, which represents a kernel-managed counter.
Reading and writing this file increments/decrements/resets this counter. As the
file is pollable via select/poll/epoll it can be used as event notification in
asynchronous I/O algorithms."""

	def __init__(self,initval=0,semaphore=False,nonBlocking=False,closeOnExec=False):
		"""Constructor: Initialise an event file descriptor. The descriptor itself can be
retrieved via the fileno() method.

If the current value of this event file is zero, any reading operation will
either fail with error EAGAIN (if "nonBlocking" is set to True) or will block.

Args:
   initval: an integer, range [0;2**64-2], defining an initial counter value.
   semaphore: a boolean; if True, this event file will act like a counting
              semaphore -- every reading operation will decrease the counter by
              one and returns 1; otherwise every reading operation will return
              the last counter value and reset the counter to zero.
   nonBlocking: a boolean.
   closeOnExec: a boolean; if True, the close-on-exec flag for this event file
                descriptor is set. This can be useful in multithreaded programs
                to close a parent's file descriptors when a child takes control
                via exec(). Please refer to the documentation on exec() for
                further details.

Raises:
   OSError.EINVAL: initval is out of bounds or unsupported value in flags.
   OSError.EMFILE: per-process limit on number of open file descriptors reached.
   OSError.ENFILE: system-wide limit on total number of open files reached.
   OSError.ENODEV: could not mount (internal) anonymous inode device.
   OSError.ENOMEM: insufficient memory to create a new eventfd file descriptor."""
		self._isNonBlocking = bool(nonBlocking)
		self._isCloseOnExec = bool(closeOnExec)
		self._isSemaphore   = bool(semaphore)
		flags = 0
		if self._isNonBlocking: flags |= eventfd_c.EFD_NONBLOCK
		if self._isCloseOnExec: flags |= eventfd_c.EFD_CLOEXEC
		if self._isSemaphore:   flags |= eventfd_c.EFD_SEMAPHORE
		try:
			# convert initval to an integer and clip it to uint (assuming 64 bit)
			initval = min(max(int(initval),0),0xfffffffffffffffe)
		except:
			# something went wrong: raise EINVAL (like providing wrong flags)
			raise OSError(errno.EINVAL,os.strerror(errno.EINVAL))
		self._fd = eventfd_c.eventfd(initval,flags)

	def __del__(self):
		"""Destructor: Close the file descriptor."""
		self.close()

	def close(self):
		"""Close the file descriptor."""
		try:
			if self._fd: os.close(self._fd)
		except: pass
		self._fd = None

	def fileno(self):
		"""Return the file descriptor of this event file object.

Returns:
   An integer."""
		return self._fd

	def read(self):
		"""Read the event file and return its value.

If this file acts like a counting semaphore, the counter's value is decreased by
one and 1 is returned. Otherwise the current counter value is returned and the
counter value is reset to zero.

If the counter value is zero and this event file is set non-blocking, this
method will block until the counter value becomes non-zero

Returns:
   An integer.

Raises:
   OSError.EAGAIN: file is non-blocking and value is zero.
   OSError.EBADF: eventfd file descriptor already closed."""
		return eventfd_c.eventfd_read(self._fd)

	def write(self,value=1):
		"""Write a value to the event file. The value is added to the counter.
No value is returned. If the counter will reach its maximum value, the operation
will either block or raise OSError.EAGAIN (if set to non-blocking behaviour).

Args:
   value: an integer in range [0;2**64-2], defaults to one.

Raises:
   OSError.EINVAL: value is out of bounds.
   OSError.EAGAIN: maximum counter value reached and file is non-blocking.
   OSError.EBADF: eventfd file descriptor already closed."""
		try:
			# convert value to an integer and clip it to uint64
			value = min(max(int(value),0),0xfffffffffffffffe)
		except:
			# something went wrong: raise EINVAL (like providing wrong flags)
			raise OSError(errno.EINVAL,os.strerror(errno.EINVAL))
		eventfd_c.eventfd_write(self._fd,value)

	def isSemaphore(self):
		"""Return True if this event file has semaphore properties.

Returns:
   A boolean."""
		return self._isSemaphore

	def isNonBlocking(self):
		"""Return True if this event file does not block when no data is available.

Returns:
   A boolean."""
		return self._isNonBlocking

	def isCloseOnExec(self):
		"""Return True if the close-on-exec flag is set.

Returns:
   A boolean."""
		return self._isCloseOnExec

class signalfd:
	"""Class to manage a file descriptor for signal notification.

A signal file descriptor is created, which accepts signals targeted at the
caller and becomes readable if triggered. Reading this file returns the pending
signals for the process. As the file is pollable via select/poll/epoll it can be
used as an alternative to the usual signal handlers."""

	def __init__(self,signalset,nonBlocking=False,closeOnExec=False):
		"""Constructor: Initialise a signal file descriptor. The descriptor itself can be
retrieved via the fileno() method.

If no signals are currently pending, any reading operation will either fail with
error EAGAIN (if "nonBlocking" is set to True) or will block.

Args:
   signalset: a set of valid signal numbers; please refer to the 'signal' module
              for valid constants (like signal.SIGTERM); a list or tuple of
              signal numbers is also accepted and will be cast to a set.
   nonBlocking: a boolean.
   closeOnExec: a boolean; if True, the close-on-exec flag for this event file
                descriptor is set. This can be useful in multithreaded programs
                to close a parent's file descriptors when a child takes control
                via exec(). Please refer to the documentation on exec() for
                further details.

Raises:
   OSError.EINVAL: invalid signalset.
   OSError.EMFILE: per-process limit on number of open file descriptors reached.
   OSError.ENFILE: system-wide limit on total number of open files reached.
   OSError.ENODEV: could not mount (internal) anonymous inode device.
   OSError.ENOMEM: insufficient memory to create a new singalfd file descriptor."""
		self._isNonBlocking = bool(nonBlocking)
		self._isCloseOnExec = bool(closeOnExec)
		flags = 0
		if self._isNonBlocking: flags |= signalfd_c.SFD_NONBLOCK
		if self._isCloseOnExec: flags |= signalfd_c.SFD_CLOEXEC
		try:
			# evaluate signal set
			self._signalset = tuple([int(i) for i in set(signalset)])
		except:
			# something went wrong: raise EINVAL (like providing wrong flags)
			raise OSError(errno.EINVAL,os.strerror(errno.EINVAL))
		self._fd = signalfd_c.signalfd(-1,self._signalset,flags)

	def __del__(self):
		"""Destructor: Close the file descriptor."""
		self.close()

	def close(self):
		"""Close the file descriptor."""
		try:
			if self._fd: os.close(self._fd)
		except: pass
		self._fd = None

	def fileno(self):
		"""Return the file descriptor of this event file object.

Returns:
   An integer."""
		return self._fd

	def modify(self,signalset,nonBlocking,closeOnExec):
		"""Modify the signalset guarded by this signal file descriptor. The descriptor
itself can be retrieved via the fileno() method. For details on the arguments,
please refer to __init__().

Args:
   signalset: a set of valid signal numbers.
   nonBlocking: a boolean.
   closeOnExec: a boolean.

Raises:
   OSError.EINVAL: invalid signalset.
   OSError.ENODEV: could not mount (internal) anonymous inode device.
   OSError.EBADF: signalfd file descriptor already closed."""
		self._isNonBlocking = bool(nonBlocking)
		self._isCloseOnExec = bool(closeOnExec)
		flags = 0
		if self._isNonBlocking: flags |= signalfd_c.SFD_NONBLOCK
		if self._isCloseOnExec: flags |= signalfd_c.SFD_CLOEXEC
		try:
			# evaluate signal set
			self._signalset = tuple([int(i) for i in set(signalset)])
		except:
			# something went wrong: raise EINVAL (like providing wrong flags)
			raise OSError(errno.EINVAL,os.strerror(errno.EINVAL))
		self._fd = signalfd_c.signalfd(self._fd,self._signalset,flags)

	def read(self):
		"""Read the signal file and return a dictionary holding information on
the recently received signal. In addition, the signal is consumed, so that it is
no longer pending for the process.

If multiple signals got caught, multiple calls to read are necessary until all
pending signals are consumed. When there are no signals pending, this method
either blocks or fails with error EAGAIN (if in non-blocking mode).

Returns:
   A dictionary of the following structure (according to signalfd(2)):
   {
      "signo":   int # Signal number
      "errno":   int # Error number (unused)
      "code":    int # Signal code
      "pid":     int # PID of sender
      "uid":     int # Real UID of sender
      "fd":      int # File descriptor (SIGIO)
      "tid":     int # Kernel timer ID (POSIX timers)
      "band":    int # Band event (SIGIO)
      "overrun": int # POSIX timer overrun count
      "trapno":  int # Trap number that caused signal
      "status":  int # Exit status or signal (SIGCHLD)
      "int":     int # Integer sent by sigqueue(3)
      "ptr":     int # Pointer sent by sigqueue(3)
      "utime":   int # User CPU time consumed (SIGCHLD)
      "stime":   int # System CPU time consumed (SIGCHLD)
      "addr":    int # Address that generated this signal (hardware-generated?)
   }

Raises:
   OSError.EAGAIN: no pending signals.
   OSError.EWOULDBLOCK: no pending signals.
   OSError.EINTR: read() call interrupted by a signal.
   OSError.EBADF: signalfd file descriptor already closed."""
		return signalfd_c.signalfd_read(self._fd)

	def signals(self):
		"""Return the set of guarded signal numbers.

Returns:
   A set of integers (signal numbers like signal.SIGTERM)."""
		return self._signalset

	def isNonBlocking(self):
		"""Return True if this event file does not block when no data is available.

Returns:
   A boolean."""
		return self._isNonBlocking

	def isCloseOnExec(self):
		"""Return True if the close-on-exec flag is set.

Returns:
   A boolean."""
		return self._isCloseOnExec

class timerfd:
	"""Class to manage a file descriptor for timer notification.

A timer file descriptor is created, which represents a timer and becomes
readable if this timer expires. Reading this file returns the number of
expirations that have occurred since the last read operation."""

	def __init__(self,rtc=False,nonBlocking=False,closeOnExec=False):
		"""Constructor: Initialise a timer file descriptor. The descriptor itself can be
retrieved via the fileno() method.

If the timer does not have expired yet, any reading operation will either fail
with error EAGAIN (if "nonBlocking" is set to True) or will block.

Args:
   rtc: a boolean; if True, this timer will access the system-wide realtime clock
        for time measurement. Otherwise a monotonic timed counter will be used.
        While the RTC can be changed by the SysAdmin and thus might exhibit
        discontinuous jumps, the monotonic clock is only affected through
        incremental adjustments by adjtime(3) and NTP.
   nonBlocking: a boolean.
   closeOnExec: a boolean; if True, the close-on-exec flag for this event file
                descriptor is set. This can be useful in multithreaded programs
                to close a parent's file descriptors when a child takes control
                via exec(). Please refer to the documentation on exec() for
                further details.

Raises:
   OSError.EMFILE: per-process limit on number of open file descriptors reached.
   OSError.ENFILE: system-wide limit on total number of open files reached.
   OSError.ENODEV: could not mount (internal) anonymous inode device.
   OSError.ENOMEM: insufficient memory to create a new singalfd file descriptor."""
		self._isRTC         = bool(rtc)
		self._isNonBlocking = bool(nonBlocking)
		self._isCloseOnExec = bool(closeOnExec)
		if self._isRTC:
			clockid = timerfd_c.CLOCK_REALTIME
		else:
			clockid = timerfd_c.CLOCK_MONOTONIC
		flags = 0
		if self._isNonBlocking: flags |= timerfd_c.TFD_NONBLOCK
		if self._isCloseOnExec: flags |= timerfd_c.TFD_CLOEXEC
		self._fd = timerfd_c.timerfd_create(clockid,flags)

	def __del__(self):
		"""Destructor: Close the file descriptor."""
		self.close()

	def close(self):
		"""Close the file descriptor."""
		try:
			if self._fd: os.close(self._fd)
		except: pass
		self._fd = None

	def fileno(self):
		"""Return the file descriptor of this event file object.

Returns:
   An integer."""
		return self._fd

	def gettime(self):
		"""Return the current timer setting.

Returns:
    A 2-tuple (value, interval) of floats;
    "value" specifies the amount of time in seconds until the timer will expire;
    "interval" defines the period in seconds of a periodically expiring timer.

    If "value" is zero, the timer is disabled.
    If "interval" is zero, the timer will just expire once.

Raises:
   OSError.EBADF: timerfd file descriptor already closed."""
		return timerfd_c.timerfd_gettime(self._fd)

	def settime(self,value=0,interval=0,absolute=False):
		"""Start or stop the timer.

In case of an absolute timer based on an RTC source, "value" is expected to be
a UNIX time value of the desired point of time.

Args:
   value: a float >= 0 defining the initial expiration time in seconds;
          if zero (default), the timer is disabled.
   interval: a float >= 0 defining the period in seconds of a periodically
             expiring timer; if zero (default), the timer will only expire once.
   absolute: a boolean; if True, an absolute timer is started; in this case
             "value" is interpreted as an absolute clock value; otherwise
             a relative timer is started (default).

Returns:
   A 2-tuple (value,interval) of floats; the old timer setting.

Raises:
   OSError.EINVAL: invalid timer values specified.
   OSError.EBADF: timerfd file descriptor already closed."""
		if bool(absolute):
			flags = timerfd_c.TFD_TIMER_ABSTIME
		else:
			flags = 0
		return timerfd_c.timerfd_settime(self._fd,flags,value,interval)

	def read(self):
		"""Read the timer file and return an integer denoting the number of
expirations since the last reading operation.

If the timer has not expired yet, this method will either block or fail with
error EAGAIN if in non-blocking mode.

Raises:
   OSError.EAGAIN: timer has not yet expired.
   OSError.EBADF: timerfd file descriptor already closed."""
		return timerfd_c.timerfd_read(self._fd)

	def isRTC(self):
		"""Return True if this timer uses the system-wide realtime clock.
If this returns False, a monotonic clock source is used.

Returns:
   A boolean."""
		return self._isRTC

	def isNonBlocking(self):
		"""Return True if this event file does not block when no data is available.

Returns:
   A boolean."""
		return self._isNonBlocking

	def isCloseOnExec(self):
		"""Return True if the close-on-exec flag is set.

Returns:
   A boolean."""
		return self._isCloseOnExec

class inotify:
	"""Class to manage an inotify instance.

An inotify file descriptor is created, which represents an inotify instance.
Files and directories can be added in order to monitor them. The inotify file
descriptor becomes readable when such a file alternation event occurs."""

	def __init__(self,nonBlocking=False,closeOnExec=False):
		"""Constructor: Initialise an inotify file descriptor. The descriptor itself can be
retrieved via the fileno() method.

If no file changes were detected by this inotify instance, any reading operation
will either fail with error EAGAIN (if "nonBlocking" is set) or will block.

Args:
   nonBlocking: a boolean.
   closeOnExec: a boolean; if True, the close-on-exec flag for this event file
                descriptor is set. This can be useful in multithreaded programs
                to close a parent's file descriptors when a child takes control
                via exec(). Please refer to the documentation on exec() for
                further details.

Raises:
   OSError.EMFILE: user limit on total number of inotify instances reached.
   OSError.EMFILE: per-process limit on number of open file descriptors reached.
   OSError.ENFILE: system-wide limit on total number of open files reached.
   OSError.ENOMEM: insufficient kernel memory available."""
		self._isNonBlocking = bool(nonBlocking)
		self._isCloseOnExec = bool(closeOnExec)
		self._wd = dict() # mapping pathnames to watch descriptors
		self._name = dict() # mapping watch descriptors to pathnames
		flags = 0
		if self._isNonBlocking: flags |= inotify_c.IN_NONBLOCK
		if self._isCloseOnExec: flags |= inotify_c.IN_CLOEXEC
		self._fd = inotify_c.inotify_init(flags)

	def __del__(self):
		"""Destructor: Close the file descriptor."""
		self.close()

	def close(self):
		"""Close the file descriptor."""
		try:
			if self._fd: os.close(self._fd)
		except: pass
		self._fd = None

	def fileno(self):
		"""Return the file descriptor of this event file object.

Returns:
   An integer."""
		return self._fd

	def add(self,pathname,mask=IN_ALL_EVENTS,replace=True):
		"""Add another file or directory to this inotify instance in order to monitor it.

The type of event to look for is described by OR-ing the following constants:
   linuxfd.IN_ACCESS: file was accessed, for example by read().
   linuxfd.IN_ATTRIB: metadata changed (e.g. permissions, extended attributes,
                      link count and user/group ID.
   linuxfd.IN_CLOSE_WRITE: a file opened for writing was closed.
   linuxfd.IN_CLOSE_NOWRITE: a file/directory not opened for writing was closed.
   linuxfd.IN_CREATE: a file/directory was created in watched directory.
   linuxfd.IN_DELETE: a file/directory was deleted from watched directory.
   linuxfd.IN_DELETE_SELF: watched file/directory was deleted. Moving files can
                           cause this event, too (copy+delete); afterwards
                           inotify will generate IN_IGNORED events.
   linuxfd.IN_MODIFY: file was modified, for example by write().
   linuxfd.IN_MOVE_SELF: watched file/directory was moved.
   linuxfd.IN_MOVED_FROM: generated for a directory containing the old filename
                          when a file is renamed.
   linuxfd.IN_MOVED_TO: generated for a directory containing the new filename
                        when a file is renamed.
   linuxfd.IN_OPEN: a file/directory was opened.
   linuxfd.IN_ALL_EVENTS = all of the above OR-ed together.
   linuxfd.IN_MOVE = linuxfd.IN_MOVED_FROM | linuxfd.IN_MOVED_TO.
   linuxfd.IN_CLOSE = linuxfd.IN_CLOSE_WRITE | linuxfd.IN_CLOSE_NOWRITE

In addition, the following flags can be set to tune the behaviour of this
inotify instance for this file:
   linuxfd.IN_DONT_FOLLOW: do not dereference pathname if it is a symbolic link.
   linuxfd.IN_EXCL_UNLINK: events are not generated for children after they have
                           been unlinked from the watched directory.
   linuxfd.IN_ONESHOT: only monitor a file/directory for one event, then remove
                       it from the watch list.
   linuxfd.IN_ONLYDIR: only watch pathname if it is a directory.

Args:
   pathname: a string.
   mask: an integer, a bitmask describing file alternation events; defaults to
         IN_ALL_EVENTS, i.e. watch for all file events.
   replace: a boolean; if it is True (=default) and a watch instance alrady
            exists for the given file, replace its mask; otherwise add the
            specified events to the existing mask (by OR-ing them).

Raises:
   OSError.EACCES: read access to given file is not permitted.
   OSError.EBADF: inotify file descriptor already closed.
   OSError.EFAULT: pathname points outside of the process's address space.
   OSError.EINVAL: given event mask contains no valid events.
   OSError.ENAMETOOLONG: pathname too long.
   OSError.ENOENT: a directory component in pathname does not exist or is a
                   dangling symbolic link.
   OSError.ENOMEM: insufficient kernel memory available.
   OSError.ENOSPC: user limit on total number of inotify watches was reached or
                   the kernel failed to allocate a needed resource."""
		if bool(replace):
			mask = mask & ~inotify_c.IN_MASK_ADD # make sure MASK_ADD is not set
		else:
			mask = mask | inotify_c.IN_MASK_ADD # make sure MASK_ADD is set
		wd = inotify_c.inotify_add_watch(self._fd,pathname,mask)
		self._wd[pathname] = wd
		self._name[wd] = pathname

	def remove(self,pathname):
		"""
Raises:
   OSError.EBADF: inotify file descriptor already closed.
"""
		wd = self._wd[pathname]
		inotify_c.inotify_rm_watch(self._fd,wd)
		del self._wd[pathname]
		del self._name[wd]

	def read(self,buffersize=1024):
		"""Read the inotify file and return a tuple of events.

If there are no inotify events, this method will either block or fail with error
EAGAIN if in non-blocking mode.

Args:
   buffersize: an integer, defining the maximum read buffer size in bytes;
               default = 1024 bytes.

Returns:
   A tuple of 4-tuples (pathname,name,mask,cookie):
    - "pathname" is the name string previously registered using add();
    - if "pathname" is a directory, the string "name" refers to a file below
      "pathname"; empty string otherwise;
    - "mask" is an integer bitmask describing the occurred events;
    - "cookie" is a unique integer connecting related events; this applies only
      for the events IN_MOVED_FROM and IN_MOVED_TO, so that the calling
      application can group these events; in any other case "cookie" is zero.

Raises:
   ValueError,TypeError: buffersize is not integer-castable.
   OSError.EAGAIN: no inotify events occurred.
   OSError.EBADF: inotify file descriptor already closed.
   OSError.EINVAL: buffer size too small.
   OSError.ENOMEM: insufficient memory for buffer allocation."""
		eventlist = inotify_c.inotify_read(self._fd,int(buffersize))
		result = list()
		for wd,mask,cookie,name in eventlist:
			result.append((self._name[wd],name,mask,cookie))
		return tuple(result)

	def watchedPaths(self):
		"""Return a tuple of all pathnames watched by this inotify instance.

Returns:
   A tuple of strings."""
		return tuple(self._wd.keys())

	def isNonBlocking(self):
		"""Return True if this event file does not block when no data is available.

Returns:
   A boolean."""
		return self._isNonBlocking

	def isCloseOnExec(self):
		"""Return True if the close-on-exec flag is set.

Returns:
   A boolean."""
		return self._isCloseOnExec

	def eventStrings(self,mask):
		"""Return a tuple of event flags identifier strings set in given mask.

Args:
   mask: an integer, a bitmask describing file alternation events.

Returns:
   A tuple of strings."""
		retval = list()
		if mask & inotify_c.IN_ACCESS: retval.append("IN_ACCESS")
		if mask & inotify_c.IN_ATTRIB: retval.append("IN_ATTRIB")
		if mask & inotify_c.IN_CLOSE_WRITE: retval.append("IN_CLOSE_WRITE")
		if mask & inotify_c.IN_CLOSE_NOWRITE: retval.append("IN_CLOSE_NOWRITE")
		if mask & inotify_c.IN_CREATE: retval.append("IN_CREATE")
		if mask & inotify_c.IN_DELETE: retval.append("IN_DELETE")
		if mask & inotify_c.IN_DELETE_SELF: retval.append("IN_DELETE_SELF")
		if mask & inotify_c.IN_MODIFY: retval.append("IN_MODIFY")


import os
import re
import sys
import unittest
import unittest.mock
from pkg_resources import VersionConflict

from coalib.coala_main import run_coala
from coalib.output.printers.LogPrinter import LogPrinter
from coalib import assert_supported_version, coala
from pyprint.ConsolePrinter import ConsolePrinter
from coala_utils.ContextManagers import prepare_file
from coalib.output.Logging import configure_logging

from tests.TestUtilities import execute_coala, bear_test_module

class coalaTest(unittest.TestCase):

    def setUp(self):
        self.old_argv = sys.argv

    def tearDown(self):
        sys.argv = self.old_argv

    def test_coala(self):
        with bear_test_module(), \
                prepare_file(['#fixme'], None) as (lines, filename):
            retval, stdout, stderr = execute_coala(
                             coala.main,
                             'coala', '-c', os.devnull,
                             '--non-interactive', '--no-color',
                             '-f', re.escape(filename),
                             '-b', 'LineCountTestBear')
            self.assertIn('This file has 1 lines.',
                          stdout,
                          'The output should report count as 1 lines')
            self.assertEqual(1, len(stderr.splitlines()))
            self.assertIn(
                'LineCountTestBear: This result has no patch attached.',
                stderr)
            self.assertNotEqual(retval, 0,
                                'coala must return nonzero when errors occured')

    @unittest.mock.patch('sys.version_info', tuple((2, 7, 11)))
    def test_python_version_27(self):
        with self.assertRaises(SystemExit):
            assert_supported_version()
            self.assertEqual(cm.error_code, 4)

    @unittest.mock.patch('sys.version_info', tuple((3, 3, 6)))
    def test_python_version_33(self):
        with self.assertRaises(SystemExit):
            assert_supported_version()
            self.assertEqual(cm.error_code, 4)

    def test_python_version_34(self):
        assert_supported_version()

    def test_did_nothing(self, debug=False):
        retval, stdout, stderr = execute_coala(coala.main, 'coala', '-I',
                                               '-S', 'cli.enabled=false',
                                               debug=debug)
        self.assertEqual(retval, 2)
        self.assertIn('Did you forget to give the `--files`', stderr)
        self.assertFalse(stdout)

        retval, stdout, stderr = execute_coala(coala.main, 'coala', '-I',
                                               '-b', 'JavaTestBear', '-f',
                                               '*.java',
                                               '-S', 'cli.enabled=false',
                                               debug=debug)
        self.assertEqual(retval, 2)
        self.assertIn('Nothing to do.', stderr)
        self.assertFalse(stdout)

    def test_did_nothing_debug(self):
        self.test_did_nothing(debug=True)

    def test_show_all_bears(self, debug=False):
        with bear_test_module():
            retval, stdout, stderr = execute_coala(
                coala.main, 'coala', '-B', '-I', debug=debug)
            self.assertEqual(retval, 0)
            # 8 bears plus 1 line holding the closing colour escape sequence.
            self.assertEqual(len(stdout.strip().splitlines()), 9)
            self.assertFalse(stderr)

    def test_show_all_bears_debug(self):
        return self.test_show_all_bears(debug=True)

    def test_show_language_bears(self, debug=False):
        with bear_test_module():
            retval, stdout, stderr = execute_coala(
                coala.main, 'coala', '-B', '-l', 'java', '-I', debug=debug)
            self.assertEqual(retval, 0)
            # 2 bears plus 1 line holding the closing colour escape sequence.
            self.assertEqual(len(stdout.splitlines()), 3)
            self.assertFalse(stderr)

    def test_show_language_bears_debug(self):
        self.test_show_language_bears(debug=True)

    def test_show_capabilities_with_supported_language(self, debug=False):
        with bear_test_module():
            retval, stdout, stderr = execute_coala(
                coala.main, 'coala', '-p', 'R', '-I', debug=debug)
            self.assertEqual(retval, 0)
            self.assertEqual(len(stdout.splitlines()), 2)
            self.assertFalse(stderr)

    def test_show_capabilities_with_supported_language_debug(self):
        self.test_show_capabilities_with_supported_language(debug=True)

    @unittest.mock.patch('coalib.parsing.DefaultArgParser.get_all_bears_names')
    @unittest.mock.patch('coalib.collecting.Collectors.icollect_bears')
    def test_version_conflict_in_collecting_bears(self, import_fn, _):
        with bear_test_module():
            import_fn.side_effect = VersionConflict('msg1', 'msg2')
            retval, stdout, stderr = execute_coala(coala.main, 'coala', '-B')
            self.assertEqual(retval, 13)
            self.assertIn(('There is a conflict in the version of a '
                           'dependency you have installed'), stderr)
            self.assertIn('pip install "msg2"', stderr)
            self.assertFalse(stdout)
            self.assertNotEqual(retval, 0,
                                'coala must return nonzero when errors occured')

    @unittest.mock.patch('coalib.collecting.Collectors._import_bears')
    def test_unimportable_bear(self, import_fn):
        with bear_test_module():
            import_fn.side_effect = SyntaxError
            retval, stdout, stderr = execute_coala(coala.main, 'coala', '-B')
            self.assertEqual(retval, 0)
            self.assertIn('Unable to collect bears from', stderr)
            self.assertIn('No bears to show.', stdout)

            import_fn.side_effect = VersionConflict('msg1', 'msg2')
            retval, stdout, stderr = execute_coala(coala.main, 'coala', '-B')
            # Note that bear version conflicts don't give exitcode=13,
            # they just give a warning with traceback in log_level debug.
            self.assertEqual(retval, 0)
            self.assertRegex(stderr,
                             'Unable to collect bears from .* because there '
                             'is a conflict with the version of a dependency '
                             'you have installed')
            self.assertIn('pip install "msg2"', stderr)
            self.assertIn('No bears to show.', stdout)

    def test_run_coala_no_autoapply(self, debug=False):
        with bear_test_module(), \
                prepare_file(['#fixme  '], None) as (lines, filename):
            self.assertEqual(
                1,
                len(run_coala(
                    console_printer=ConsolePrinter(),
                    log_printer=LogPrinter(),
                    arg_list=(
                        '-c', os.devnull,
                        '-f', re.escape(filename),
                        '-b', 'SpaceConsistencyTestBear',
                        '--apply-patches',
                        '-S', 'use_spaces=yeah'
                    ),
                    autoapply=False,
                    debug=debug
                )[0]['cli'])
            )

            self.assertEqual(
                0,
                len(run_coala(
                    console_printer=ConsolePrinter(),
                    log_printer=LogPrinter(),
                    arg_list=(
                        '-c', os.devnull,
                        '-f', re.escape(filename),
                        '-b', 'SpaceConsistencyTestBear',
                        '--apply-patches',
                        '-S', 'use_spaces=yeah'
                    ),
                    debug=debug
                )[0]['cli'])
            )

    def test_run_coala_no_autoapply_debug(self):
        self.test_run_coala_no_autoapply(debug=True)

    def test_logged_error_causes_non_zero_exitcode(self):
        configure_logging()
        with bear_test_module(), \
                prepare_file(['#fixme  '], None) as (lines, filename):
            _, exitcode, _ = run_coala(
                console_printer=ConsolePrinter(),
                log_printer=LogPrinter(),
                arg_list=(
                    '-c', os.devnull,
                    '-f', re.escape(filename),
                    '-b', 'ErrorTestBear'
                ),
                autoapply=False
            )

            assert exitcode == 1

    def test_coala_with_color(self):


#!/usr/bin/python
from __future__ import absolute_import, division, print_function, unicode_literals
from builtins import (ascii, bytes, chr, dict, filter, hex, input, #pip install future
                      int, map, next, oct, open, pow, range, round,
                      str, super, zip)
from future.utils import with_metaclass
from abc import ABCMeta, abstractmethod, abstractproperty
import numpy as np
import scipy.stats
import warnings
import logging
import itertools as it
import time
import math
import os

from commandline_parsable import parsable_base
from logging_exceptions import log_to_exception

from ..utils import get_version_string

log = logging.getLogger(__name__)

DEFAULT_ENERGY_PREFACTOR = 30
INCR = 0.01

@parsable_base(False, required_kwargs=["cg"], factory_function="from_cg",
               name_attr="_shortname", helptext_sep="\n", help_attr="HELPTEXT",
               allow_pre_and_post_number=True, help_intro_list_sep="\n")
class EnergyFunction(with_metaclass(ABCMeta, object)):
    '''
    The base class for energy functions.
    '''
    __metaclass__ = ABCMeta

    @classmethod
    def from_cg(cls, prefactor, adjustment, cg, **kwargs):
        """
        Factory function. Return this energy for the given cg.

        :returns: An instance if this class or a CombinedEnergy containing instances of this class.
        """
        return cls(prefactor, adjustment)

    def __init__(self, prefactor=None, adjustment=None):
        self.log = logging.getLogger(self.__class__.__module__+"."+self.__class__.__name__)
        if prefactor is None:
            prefactor = DEFAULT_ENERGY_PREFACTOR
        if adjustment is None:
            adjustment = 1.
        #: Used by constraint energies, to store tuples of stems that clash.
        #: Updated every time eval_energy is called.
        self.bad_bulges = []

        #: The measure encountered during last energy evaluation
        self._last_measure = None
        #: The reference distribution.
        #: In the case of EnergyFunctions that are not CoarseGrainEnergy instances,
        #: this is only used to dump the measures to a file.
        self.accepted_measures = []

        #: The energy function can be adjusted with a prefactor (weight)
        #: and an adjustment (offset from the target value)
        self.prefactor, self._pf_stepwidth, self._pf_update_freq = self._parse_prefactor(prefactor, DEFAULT_ENERGY_PREFACTOR)
        self.adjustment, self._adj_stepwidth, self._adj_update_freq = self._parse_prefactor(adjustment, 1.0)

        #: How many sampling steps have been performed
        #: (Used for reference ratio method and simulated annealing)
        self.step=0

        #: Name and shortname of the energy
        # We need to check the class, not the instance, because implementation of name as a property
        # in subclasses may raise an error on incompletely initialized instanzes.
        if not hasattr(type(self), "name"):
            self.name = self.__class__.__name__.lower()

    @property
    def last_accepted_measure(self):
        return self.accepted_measures[-1]

    def accept_last_measure(self):
        """
        The last measure should contribute to the new reference distribution.

        The structure evaluated with the last call to eval_energy was accepted
        during sampling. Thus the corresponding measure should be included in
        the reference distribution. (Reference ratio method)
        """
        if self._last_measure is not None:
            self.accepted_measures.append(self._last_measure)
        self._step_complete()

    def reject_last_measure(self):
        """
        The last measure of the reference distribution should be duplicated.

        The structure evaluated with the last call to eval_energy was rejected
        during sampling. Thus the last accepted measure should be added a second
        time to the reference distribution. (Reference ratio method)
        """
        if len(self.accepted_measures) > 0:
            self.accepted_measures.append(self.accepted_measures[-1])
        self._step_complete()

    @abstractproperty #!Note: Can be overwritten by a simple class-level variable, does not have to be a property.
    def _shortname(self):
        """
        A shortcut for the energy name. Ideally 3 to 4 letters, all-caps.

        This should be implemented by a simple class-level variable.

        It is used to generate energies from commandline-options
        (via fess.builder.energy.energies_from_string).
        Further more, the `shortname` property `_shortname` it for the description of the energy.
        """
        raise NotImplementedError

    @abstractproperty
    def HELPTEXT(self):
        """
        A preformatted helptext describing the energy.

        This is used by get_argparse_help and intended for
        output on the commandline via argparse's `--help` option.
        """
        raise NotImplementedError

    @property
    def shortname(self):
        if self.prefactor==DEFAULT_ENERGY_PREFACTOR:
            pre=""
        else:
            pre=self.prefactor
        if self.adjustment==1.0:
            adj=""
        else:
            adj=self.adjustment
        return "{}{}{}".format(pre, self._shortname, adj)

        return

    @abstractmethod
    def eval_energy(self, cg, background=True, nodes=None, **kwargs):
        raise NotImplementedError

    def dump_measures(self, base_directory, iteration=None):
        '''
        Dump all of the accepted measures collected so far
        to a file.
        '''
        out = " ".join(map("{:.4f}".format,self.accepted_measures))+"\n"
        output_file = op.join(base_directory, self.name+"_"+str(hex(id(self)))+".measures")
        with open(output_file, 'w') as f:
            f.write(out)

        if iteration is not None:
            with open(output_file + ".{:d}".format(iteration), 'w') as f:
                f.write(out)

    def _parse_prefactor(self, value, default):
        """
        Convert prefactor and adjustment to a 3-tuple

        :param value: A number, tuple or a string with either a number or 3 '_'-separated numbers.
        :returns: A tuple prefactor, stepwidth, update_frequency
        """
        if not value:
            return default, 0, float("inf")
        try:
            if isinstance(value[0], (float, int)): # A tuple of numbers:
                return value # Return the tuple unmodified. Unpacking the tuple will check its length
        except TypeError:
            return value,  0, float("inf") # Value is a number
        # Now value is a string or not a legal value.
        if "_" in value:
            p, s, f = value.split("_")
            return float(p), float(s), int(f)
        else:
            return float(value),  0, float("inf")

    def _step_complete(self):
        """
        Bookkeeping after every performed sampling step (NOT energy evaluation).

        Called by accept/reject last measure.
        If required, update "temperature" in simulated annealing simulations.
        """
        log.debug("step complete called on %s instance at %s.",type(self).__name__, hex(id(self)))
        self.step+=1
        if self.step % self._pf_update_freq <1 and self.step>0:
            self._update_pf()
        log.debug("step: %s  %% _adj_update_freq: %s = %s", self.step, self._adj_update_freq, self.step % self._adj_update_freq)
        if self.step % self._adj_update_freq < 1 and self.step>0:
            self._update_adj()
    def _update_pf(self):
        self.prefactor += self._pf_stepwidth
    def _update_adj(self):
        log.debug("Updating adjustment from %s to %s", self.adjustment, self.adjustment+self._adj_stepwidth)
        self.adjustment += self._adj_stepwidth

    def __bool__(self):
        """
        Returns always True, because a normal energy is never empty.
        This is implemented here to provide compatibility of the
        interface of Energy with CombinedEnergy
        """
        return True

    __nonzero__=__bool__

class InteractionEnergy(EnergyFunction):
    """
    A base-class for A-Minor and SLD energy. Uses the reference ratio method
    """
    def __init__(self, rna_length, num_loops, prefactor, adjustment):
        self.num_loops = num_loops
        super(InteractionEnergy, self).__init__(prefactor, adjustment)
        if num_loops == 0:
            log.warning("The number of loops is 0 for %s", self.shortname)
        self.reset_distributions(rna_length)

    @classmethod
    def qualifying_loops(cls, cg, loop_iterator):
        logger = logging.getLogger(cls.__module__+"."+cls.__name__)
        for l in loop_iterator:
            if l in cg.incomplete_elements or l in cg.interacting_elements:
                logger.debug("Loop %s does not qualify because it is interacting or incomplete.", l)
                logger.debug("Interacting %s.", cg.interacting_elements)
                logger.debug("Incomplete %s.", cg.incomplete_elements)
                continue
            yield l

    @abstractmethod
    def reset_distributions(self, rna_length):
        """
        This should set the following attributes:

        self.reference_interactions: A list with the fraction of loops that interact per entry.
                                     Each entry must correspond to self.num_loops loops.
        self.target_interactions: The fraction of loops that interact in the PDB
        """
        raise NotImplementedError

    @abstractmethod
    def _get_cg_measure(self, cg):
        """
        Return the fraction of loops that interact, i.e. the interaction-count/self.num_loops
        """
        raise NotImplementedError

    def eval_energy(self, cg, background=True, nodes=None, use_accepted_measure=False,
                    plot_debug=False, **kwargs):
        if plot_debug or nodes is not None:
            raise NotImplementedError("'plot_debug' and 'nodes' args are not implemented"
                                      " for InteractionEnergies")
        if use_accepted_measure:
            m = self.accepted_measures[-1]
        else:
            m = self._get_cg_measure(cg)
        self._last_measure = m

        reference_perc = sum(self.reference_interactions)/len(self.reference_interactions)
        target_perc    = self.target_interactions
        if background:
            e_i = -np.log( target_perc) + np.log(reference_perc)
            e_noi = -np.log( 1-target_perc) + np.log(1-reference_perc)
        else:
            e_i=-np.log(target_perc)
            e_noi=-np.log(1-target_perc)
        self.log.debug("%s , %s", e_i, e_noi)
        energy_contrib = e_i-e_noi

        num_interactions = int(round(m*self.num_loops))
        self.log.debug("Energy contribution of %s = %s, "
                  "with %s interactions (background=%s)",
                  self.shortname,
                  energy_contrib, num_interactions, background)
        energy=energy_contrib*num_interactions
        return self.prefactor*energy

    def _set_target_distribution(self):
        """
        Not needed, since the adjustment is multiplied to the
        target value in every eval_energy call
        """
        pass

    def _resample_background_kde(self):
        self.reference_interactions = self.accepted_measures[:]

class CoarseGrainEnergy(EnergyFunction):
    """
    A base-class for Energy functions that use a background distribution.
    """
    #: Change this to anything but "kde" to use a beta distribution (UNTESTED).
    dist_type = "kde"

    @classmethod
    def from_cg(cls, prefactor, adjustment, cg, **kwargs):
        """
        Factory function. Return this energy for the given cg.

        :returns: An instance if this class or a CombinedEnergy which is empty or contains instances of this class.
        """
        return cls(rna_length = cg.seq_length, prefactor=prefactor, adjustment=adjustment)

    def __init__(self, rna_length, prefactor=None, adjustment=None):
        super(CoarseGrainEnergy, self).__init__(prefactor, adjustment)

        self.reset_distributions(rna_length)

        #: The previous evaluated energy
        self.prev_energy = None

        #: Resample the reference distribution every n steps
        self.kde_resampling_frequency = 3

    @abstractproperty
    def sampled_stats_fn(self):
        """
        Can be overridden by class-level variable.

        A filename containing the target_distribution or None
        """
        raise NotImplementedError
    @abstractproperty
    def HELPTEXT(self):
        """
        Can be overridden by class-level variable.

        A filename containing the reference_distribution
        """
        raise NotImplementedError

    @classmethod
    @abstractmethod
    def generate_target_distribution(cls, *args, **kwargs):
        """
        Provide an implementation how the target distribution (and all other required files)
        can be generated. Document this method well to ensure reproducibility.

        This must be implemented as a classmethod, so changes to the class will
        be reflected in the target distribution.
        """
        raise NotImplementedError

    @staticmethod
    def _print_file_header(file_, cg_filenames):
        """
        Print a header to files generated with `generate_target_distribution`.

        This should allow for easy reproducibility where possible.

        :param file_: A file opened for writing
        :param cg_filenames: A list of strings (filenames)
        """
        print("# Generated on "+ time.strftime("%d %b %Y %H:%M:%S %Z"), file=file_)
        print("# Generated from the following files: ", file=file_)
        for fn in cg_filenames:
            print("#   {}".format(fn), file=file_)
        print("# Working directory: {}".format(os.getcwd()), file=file_)
        print("# Version: {}".format(get_version_string().strip()), file=file_)

    def plot_distributions(self, from_=None, to_=None, val = None):
        """
        :param from_: Minimal value of x-axis
        :param to_:   Maximal value of x-axis
        """
        import matplotlib.pyplot as plt

        if from_ is None:
            try:
                from_ = min(it.chain(self.accepted_measures, self.target_values))
            except AttributeError:
                from_ = min(self.accepted_measures)*0.8
            except ValueError:
                from_ = 0
        if to_ is None:
            try:
                to_ = max(it.chain(self.accepted_measures, self.target_values))
            except AttributeError:
                to_ = max(self.accepted_measures)*1.2
            except ValueError:
                to_ = 10
        xs=np.linspace(from_, to_, 2000)
        fig,ax1 = plt.subplots()
        ax2 = ax1.twinx()
        ax1.plot(xs, self.reference_distribution(xs), label="sampled")
        ax1.plot(xs, self.target_distribution(xs), label="target")
        ax2.plot(xs, -(np.log(self.target_distribution(xs) + 0.00000001 *
                     self.reference_distribution(xs)) - np.log(self.reference_distribution(xs))),
                 label="energy", color="red")
        plt.title(self.shortname)
        ax1.legend(loc="lower left")
        ax2.legend()
        if val is not None:
            ax1.plot([val,val], [0, max(self.target_distribution(xs))], "g-")

        plt.show(block = False)

    def reset_distributions(self, rna_length):
        """
        Reset the reference and target distribution to the values from files.

        :param rna_length: The length of the RNA in nucleotides.
        """
        if self.sampled_stats_fn is not None:
            log.debug("Loading sapmled measures into accepted_measures")
            self.accepted_measures = list(self._get_values_from_file(self.sampled_stats_fn, rna_length))
        #If sampled_stats_fn is None, we assume accepted_measures is given in the constructor
        if self.accepted_measures:
            self.reference_distribution = self._get_distribution_from_values(self.accepted_measures)
        else:
            raise ValueError("Either sampled_stats_fn or accepted_measures has to be set "
                             "before calling CoarseGrainEnergy.__init__ or "
                             "CoarseGrainEnergy.reset_distribution "
                             "for {}.".format(type(self).__name__))
        if self.real_stats_fn is not None:
            self.target_values =  self._get_values_from_file(self.real_stats_fn, rna_length)
        self._set_target_distribution()

    def _step_complete(self):
        """
        Call superclass _step_complete and resample background_kde every n steps.
        """
        log.info("CoarseGrainedEnergy step complete")
        super(CoarseGrainEnergy, self)._step_complete()
        if self.step % self.kde_resampling_frequency == 0:
            self._resample_background_kde()

    def _resample_background_kde(self):
        """
        Update the reference distribution based on the accepted values
        """
        log.debug("Resampling background KDE for %s. Now %d accepted measures", type(self).__name__, len(self.accepted_measures))
        values = self.accepted_measures
        new_kde = self._get_distribution_from_values(values)
        if new_kde is not None:
            self.reference_distribution = new_kde
            log.debug("Density of ref AFTER resampling = %s", self.reference_distribution(self.accepted_measures[-1]))
        else:
            log.warning("Distribution is None. Cannot change background_kde")

    @classmethod
    @abstractmethod
    def _get_values_from_file(cls, filename, nt_length):
        raise NotImplementedError

    @staticmethod
    def _values_within_nt_range(data, length, target_col, length_col="nt_length", target_len=500):
        rdata = []

        distribution_lower_bound = 1.
        distribution_upper_bound = 1.

        while (len(rdata) < target_len and len(rdata)<len(data)):
            try:
                distribution_lower_bound -= INCR
                distribution_upper_bound += INCR
                rdata = data[data[length_col] > distribution_lower_bound * length]
                rdata = rdata[rdata[length_col] < length * distribution_upper_bound]
            except KeyboardInterrupt:
                log.error("len(rdata) is {}, len(data)={}, bound= {}...{}".format(len(rdata),len(data),
                     distribution_lower_bound ) * length, length * ( distribution_upper_bound ))
                raise
        if len(rdata)==0:
            raise ValueError("No data found for distribution")
        log.info("%d datapoints", len(rdata))
        return rdata[target_col]

    @classmethod
    def _get_distribution_from_values(cls, values):
        '''
        Return a probability distribution from the given values.

        :param values: A list of values to fit a distribution to.
        :return: A probability distribution fit to the values.
        '''

        log.debug("Getting distribution from values of shape {}".format(np.shape(values)))
        if cls.dist_type == "kde":
            try:
                k = scipy.stats.gaussian_kde(values)
            except np.linalg.linalg.LinAlgError:
                log.exception("Setting KDE for %s to None because of", values)
                return None
        else:
            floc = -0.1
            fscale =  1.5 * max(values)
            f = scipy.stats.beta.fit(values, floc=floc, fscale=fscale)
            k = lambda x: scipy.stats.beta.pdf(x, f[0], f[1], f[2], f[3])
        return k

    @abstractmethod
    def _get_cg_measure(self, cg):
        raise NotImplementedError

    def eval_energy(self, cg, background=True, nodes=None, use_accepted_measure=False, plot_debug=False, **kwargs):
        '''
        A generic function which simply evaluates the energy based on the
        previously calculated probability distributions.

        :param use_accepted_measure: Do not calculate the measure from the cg. Instead use the
                                     last accepted measure.
        :param plot_debug: Plot the distributions for debugging
        '''
        if use_accepted_measure:
            m = self.accepted_measures[-1]
        else:


# -*- coding: utf-8 -*-

"""
***************************************************************************
    ImportIntoPostGIS.py
    ---------------------
    Date                 : October 2012
    Copyright            : (C) 2012 by Victor Olaya
    Email                : volayaf at gmail dot com
***************************************************************************
*                                                                         *
*   This program is free software; you can redistribute it and/or modify  *
*   it under the terms of the GNU General Public License as published by  *
*   the Free Software Foundation; either version 2 of the License, or     *
*   (at your option) any later version.                                   *
*                                                                         *
***************************************************************************
"""

__author__ = 'Victor Olaya'
__date__ = 'October 2012'
__copyright__ = '(C) 2012, Victor Olaya'

# This will get replaced with a git SHA1 when you do a git archive

__revision__ = '$Format:%H$'

from qgis.core import (QgsVectorLayerExporter,
                       QgsSettings,
                       QgsFeatureSink,
                       QgsProcessingException,
                       QgsProcessingParameterFeatureSource,
                       QgsProcessingParameterString,
                       QgsProcessingParameterField,
                       QgsProcessingParameterBoolean,
                       QgsWkbTypes)

from processing.algs.qgis.QgisAlgorithm import QgisAlgorithm
from processing.tools import postgis

class ImportIntoPostGIS(QgisAlgorithm):

    DATABASE = 'DATABASE'
    TABLENAME = 'TABLENAME'
    SCHEMA = 'SCHEMA'
    INPUT = 'INPUT'
    OVERWRITE = 'OVERWRITE'
    CREATEINDEX = 'CREATEINDEX'
    GEOMETRY_COLUMN = 'GEOMETRY_COLUMN'
    LOWERCASE_NAMES = 'LOWERCASE_NAMES'
    DROP_STRING_LENGTH = 'DROP_STRING_LENGTH'
    FORCE_SINGLEPART = 'FORCE_SINGLEPART'
    PRIMARY_KEY = 'PRIMARY_KEY'
    ENCODING = 'ENCODING'

    def group(self):
        return self.tr('Database')

    def groupId(self):
        return 'database'

    def __init__(self):
        super().__init__()

    def initAlgorithm(self, config=None):
        self.addParameter(QgsProcessingParameterFeatureSource(self.INPUT,
                                                              self.tr('Layer to import')))

        db_param = QgsProcessingParameterString(
            self.DATABASE,
            self.tr('Database (connection name)'))
        db_param.setMetadata({
            'widget_wrapper': {
                'class': 'processing.gui.wrappers_postgis.ConnectionWidgetWrapper'}})
        self.addParameter(db_param)

        schema_param = QgsProcessingParameterString(
            self.SCHEMA,
            self.tr('Schema (schema name)'), 'public', False, True)
        schema_param.setMetadata({
            'widget_wrapper': {
                'class': 'processing.gui.wrappers_postgis.SchemaWidgetWrapper',
                'connection_param': self.DATABASE}})
        self.addParameter(schema_param)

        table_param = QgsProcessingParameterString(
            self.TABLENAME,
            self.tr('Table to import to (leave blank to use layer name)'), '', False, True)
        table_param.setMetadata({
            'widget_wrapper': {
                'class': 'processing.gui.wrappers_postgis.TableWidgetWrapper',
                'schema_param': self.SCHEMA}})
        self.addParameter(table_param)

        self.addParameter(QgsProcessingParameterField(self.PRIMARY_KEY,
                                                      self.tr('Primary key field'), None, self.INPUT, QgsProcessingParameterField.Any, False, True))
        self.addParameter(QgsProcessingParameterString(self.GEOMETRY_COLUMN,
                                                       self.tr('Geometry column'), 'geom'))
        self.addParameter(QgsProcessingParameterString(self.ENCODING,
                                                       self.tr('Encoding'), 'UTF-8',
                                                       False, True))
        self.addParameter(QgsProcessingParameterBoolean(self.OVERWRITE,
                                                        self.tr('Overwrite'), True))
        self.addParameter(QgsProcessingParameterBoolean(self.CREATEINDEX,
                                                        self.tr('Create spatial index'), True))
        self.addParameter(QgsProcessingParameterBoolean(self.LOWERCASE_NAMES,
                                                        self.tr('Convert field names to lowercase'), True))
        self.addParameter(QgsProcessingParameterBoolean(self.DROP_STRING_LENGTH,
                                                        self.tr('Drop length constraints on character fields'), False))
        self.addParameter(QgsProcessingParameterBoolean(self.FORCE_SINGLEPART,
                                                        self.tr('Create single-part geometries instead of multi-part'), False))

    def name(self):
        return 'importintopostgis'

    def displayName(self):
        return self.tr('Import into PostGIS')

    def processAlgorithm(self, parameters, context, feedback):
        connection = self.parameterAsString(parameters, self.DATABASE, context)
        db = postgis.GeoDB.from_name(connection)

        schema = self.parameterAsString(parameters, self.SCHEMA, context)
        overwrite = self.parameterAsBool(parameters, self.OVERWRITE, context)
        createIndex = self.parameterAsBool(parameters, self.CREATEINDEX, context)
        convertLowerCase = self.parameterAsBool(parameters, self.LOWERCASE_NAMES, context)
        dropStringLength = self.parameterAsBool(parameters, self.DROP_STRING_LENGTH, context)
        forceSinglePart = self.parameterAsBool(parameters, self.FORCE_SINGLEPART, context)
        primaryKeyField = self.parameterAsString(parameters, self.PRIMARY_KEY, context) or 'id'
        encoding = self.parameterAsString(parameters, self.ENCODING, context)

        source = self.parameterAsSource(parameters, self.INPUT, context)
        if source is None:
            raise QgsProcessingException(self.invalidSourceError(parameters, self.INPUT))

        table = self.parameterAsString(parameters, self.TABLENAME, context)
        if table:
            table.strip()
        if not table or table == '':
            table = source.sourceName()
            table = table.replace('.', '_')
        table = table.replace(' ', '').lower()[0:62]
        providerName = 'postgres'

        geomColumn = self.parameterAsString(parameters, self.GEOMETRY_COLUMN, context)
        if not geomColumn:
            geomColumn = 'geom'

        options = {}
        if overwrite:
            options['overwrite'] = True
        if convertLowerCase:
            options['lowercaseFieldNames'] = True
            geomColumn = geomColumn.lower()
        if dropStringLength:
            options['dropStringConstraints'] = True
        if forceSinglePart:
            options['forceSinglePartGeometryType'] = True

        # Clear geometry column for non-geometry tables
        if source.wkbType() == QgsWkbTypes.NoGeometry:
            geomColumn = None

        uri = db.uri
        uri.setDataSource(schema, table, geomColumn, '', primaryKeyField)

        if encoding:
            options['fileEncoding'] = encoding

        exporter = QgsVectorLayerExporter(uri.uri(), providerName, source.fields(),
                                          source.wkbType(), source.sourceCrs(), overwrite, options)

        if exporter.errorCode() != QgsVectorLayerExporter.NoError:
            raise QgsProcessingException(
                self.tr('Error importing to PostGIS\n{0}').format(exporter.errorMessage()))

        features = source.getFeatures()
        total = 100.0 / source.featureCount() if source.featureCount() else 0
        for current, f in enumerate(features):
            if feedback.isCanceled():
                break

            if not exporter.addFeature(f, QgsFeatureSink.FastInsert):
                feedback.reportError(exporter.errorMessage())

            feedback.setProgress(int(current * total))

        exporter.flushBuffer()
        if exporter.errorCode() != QgsVectorLayerExporter.NoError:
            raise QgsProcessingException(
                self.tr('Error importing to PostGIS\n{0}').format(exporter.errorMessage()))

        if geomColumn and createIndex:
            db.create_spatial_index(table, schema, geomColumn)

        db.vacuum_analyze(table, schema)

        return {}

    def dbConnectionNames(self):


# -*- coding: utf-8 -*-
#
# This file is part of Harvesting Kit.
# Copyright (C) 2015 CERN.
#
# Harvesting Kit is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Harvesting Kit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Harvesting Kit; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Contains conversion package from CDS to INSPIRE MARCXML.

>>> from harvestingkit.bibrecord import BibRecordPackage
>>> from harvestingkit.inspire_cds_package.from_cds import CDS2Inspire
>>> bibrecs = BibRecordPackage("cds.xml")
>>> bibrecs.parse()
>>> inspire_xml = CDS2Inspire.convert(bibrecs.get_records())
"""

from __future__ import print_function

import os
import re

from tempfile import mkstemp

from ..bibrecord import (record_get_field_instances,
                         record_add_field,
                         record_get_field_values,
                         record_delete_field,
                         record_delete_fields,
                         record_replace_field,
                         field_get_subfield_instances,
                         create_field,
                         field_swap_subfields,
                         field_get_subfields)

from ..utils import (
    punctuate_authorname,
    convert_images,
    convert_date_to_iso,
    unzip,
    locate,
    download_file,
)
from .base import MARCXMLConversion

class CDS2Inspire(MARCXMLConversion):

    """Convert CDS to INSPIRE."""

    # By setting the class variable here, we run it only once a session
    kbs = MARCXMLConversion.load_config("cds", "inspire")

    def __init__(self, bibrec, strip_fields_list=None):
        """Create."""
        super(CDS2Inspire, self).__init__(bibrec, strip_fields_list)
        self.collections = set([])

    def get_record(self):
        """Override the base get_record."""
        self.update_system_numbers()
        self.add_systemnumber("CDS")
        self.fields_list = [
            "024", "041", "035", "037", "088", "100",
            "110", "111", "242", "245", "246", "260",
            "269", "300", "502", "650", "653", "693",
            "700", "710", "773", "856", "520", "500",
            "980"
        ]
        self.keep_only_fields()

        self.determine_collections()
        self.add_cms_link()
        self.update_languages()
        self.update_reportnumbers()
        self.update_date()
        self.update_pagenumber()
        self.update_authors()
        self.update_subject_categories("SzGeCERN", "INSPIRE", "categories_inspire")
        self.update_keywords()
        self.update_experiments()
        self.update_collaboration()
        self.update_journals()
        self.update_links_and_ffts()

        if 'THESIS' in self.collections:
            self.update_thesis_supervisors()
            self.update_thesis_information()

        if 'NOTE' in self.collections:
            self.add_notes()

        for collection in self.collections:
            record_add_field(self.record,
                             tag='980',
                             subfields=[('a', collection)])
        self.remove_controlfields()
        return self.record

    def determine_collections(self):
        """Try to determine which collections this record should belong to."""
        for value in record_get_field_values(self.record, '980', code='a'):
            if 'NOTE' in value.upper():
                self.collections.add('NOTE')
            if 'THESIS' in value.upper():
                self.collections.add('THESIS')
            if 'CONFERENCEPAPER' in value.upper():
                self.collections.add('ConferencePaper')
            if "HIDDEN" in value.upper():
                self.hidden = True

        if self.is_published():
            self.collections.add("PUBLISHED")
            self.collections.add("CITEABLE")

        if 'NOTE' not in self.collections:
            from itertools import product
            # TODO: Move this to a KB
            kb = ['ATLAS-CONF-', 'CMS-PAS-', 'ATL-', 'CMS-DP-',
                  'ALICE-INT-', 'LHCb-PUB-']
            values = record_get_field_values(self.record, "088", code='a')
            for val, rep in product(values, kb):
                if val.startswith(rep):
                    self.collections.add('NOTE')
                    break

        # 980 Arxiv tag
        if record_get_field_values(self.record, '035',
                                   filter_subfield_code="a",
                                   filter_subfield_value="arXiv"):
            self.collections.add("arXiv")

        # 980 HEP && CORE
        self.collections.add('HEP')
        self.collections.add('CORE')

        # 980 Conference Note
        if 'ConferencePaper' not in self.collections:
            for value in record_get_field_values(self.record,
                                                 tag='962',
                                                 code='n'):
                if value[-2:].isdigit():
                    self.collections.add('ConferencePaper')
                    break
        # Clear out any existing ones.
        record_delete_fields(self.record, "980")

    def is_published(self):
        """Check fields 980 and 773 to see if the record has already been published.

        :return: True is published, else False
        """
        field980 = record_get_field_instances(self.record, '980')
        field773 = record_get_field_instances(self.record, '773')
        for f980 in field980:
            if 'a' in field_get_subfields(f980):
                for f773 in field773:
                    if 'p' in field_get_subfields(f773):
                        return True
        return False

    def add_cms_link(self):
        """Special handling if record is a CMS NOTE."""
        intnote = record_get_field_values(self.record, '690',
                                          filter_subfield_code="a",
                                          filter_subfield_value='INTNOTE')
        if intnote:
            val_088 = record_get_field_values(self.record,
                                              tag='088',
                                              filter_subfield_code="a")
            for val in val_088:
                if 'CMS' in val:
                    url = ('http://weblib.cern.ch/abstract?CERN-CMS' +
                           val.split('CMS', 1)[-1])
                    record_add_field(self.record,
                                     tag='856',
                                     ind1='4',
                                     subfields=[('u', url)])

    def update_system_numbers(self):
        """035 Externals."""
        scn_035_fields = record_get_field_instances(self.record, '035')
        forbidden_values = ["cercer",
                            "inspire",
                            "xx",
                            "cern annual report",
                            "cmscms",
                            "wai01"]
        for field in scn_035_fields:
            subs = field_get_subfields(field)
            if '9' in subs:
                if 'a' not in subs:
                    continue
                for sub in subs['9']:
                    if sub.lower() in forbidden_values:
                        break
                else:
                    # No forbidden values (We did not "break")
                    suffixes = [s.lower() for s in subs['9']]
                    if 'spires' in suffixes:
                        new_subs = [('a', 'SPIRES-%s' % subs['a'][0])]
                        record_add_field(
                            self.record, '970', subfields=new_subs)
                        continue
            if 'a' in subs:
                for sub in subs['a']:
                    if sub.lower() in forbidden_values:
                        record_delete_field(self.record, tag="035",
                                            field_position_global=field[4])

    def update_reportnumbers(self):
        """Handle reportnumbers. """
        rep_088_fields = record_get_field_instances(self.record, '088')
        for field in rep_088_fields:
            subs = field_get_subfields(field)
            if '9' in subs:
                for val in subs['9']:
                    if val.startswith('P0') or val.startswith('CM-P0'):
                        sf = [('9', 'CERN'), ('b', val)]
                        record_add_field(self.record, '595', subfields=sf)
            for key, val in field[0]:
                if key in ['a', '9'] and not val.startswith('SIS-'):
                    record_add_field(
                        self.record, '037', subfields=[('a', val)])
        record_delete_fields(self.record, "088")

        # 037 Externals also...
        rep_037_fields = record_get_field_instances(self.record, '037')
        for field in rep_037_fields:
            subs = field_get_subfields(field)
            if 'a' in subs:
                for value in subs['a']:
                    if 'arXiv' in value:
                        new_subs = [('a', value), ('9', 'arXiv')]
                        for fld in record_get_field_instances(self.record,  '695'):
                            for key, val in field_get_subfield_instances(fld):
                                if key == 'a':
                                    new_subs.append(('c', val))
                                    break
                        nf = create_field(subfields=new_subs)
                        record_replace_field(self.record, '037', nf, field[4])
            for key, val in field[0]:
                if key in ['a', '9'] and val.startswith('SIS-'):
                    record_delete_field(
                        self.record, '037', field_position_global=field[4])

    def update_date(self):
        """269 Date normalization."""
        for field in record_get_field_instances(self.record, '269'):
            for idx, (key, value) in enumerate(field[0]):
                if key == "c":
                    field[0][idx] = ("c", convert_date_to_iso(value))
                    record_delete_fields(self.record, "260")

        if 'THESIS' not in self.collections:
            for field in record_get_field_instances(self.record, '260'):
                record_add_field(self.record, '269', subfields=field[0])
            record_delete_fields(self.record, '260')

    def update_pagenumber(self):
        """300 page number."""
        for field in record_get_field_instances(self.record, '300'):
            for idx, (key, value) in enumerate(field[0]):
                if key == 'a':
                    if "mult." not in value and value != " p":
                        field[0][idx] = ('a', re.sub(r'[^\d-]+', '', value))
                    else:
                        record_delete_field(self.record, '300',
                                            field_position_global=field[4])
                        break

    def update_authors(self):
        """100 & 700 punctuate author names."""
        author_names = record_get_field_instances(self.record, '100')
        author_names.extend(record_get_field_instances(self.record, '700'))
        for field in author_names:
            subs = field_get_subfields(field)
            if 'i' not in subs or 'XX' in subs['i']:
                if 'j' not in subs or 'YY' in subs['j']:
                    for idx, (key, value) in enumerate(field[0]):
                        if key == 'a':
                            field[0][idx] = ('a', punctuate_authorname(value))

    def update_thesis_supervisors(self):
        """700 -> 701 Thesis supervisors."""
        for field in record_get_field_instances(self.record, '700'):
            record_add_field(self.record, '701', subfields=field[0])
        record_delete_fields(self.record, '700')

    def update_thesis_information(self):
        """501 degree info - move subfields."""
        fields_501 = record_get_field_instances(self.record, '502')
        for idx, field in enumerate(fields_501):
            new_subs = []
            for key, value in field[0]:
                if key == 'a':
                    new_subs.append(('b', value))
                elif key == 'b':
                    new_subs.append(('c', value))
                elif key == 'c':
                    new_subs.append(('d', value))
                else:
                    new_subs.append((key, value))
            fields_501[idx] = field_swap_subfields(field, new_subs)

    def update_keywords(self):
        """653 Free Keywords."""
        for field in record_get_field_instances(self.record, '653', ind1='1'):
            subs = field_get_subfields(field)
            new_subs = []
            if 'a' in subs:
                for val in subs['a']:
                    new_subs.extend([('9', 'author'), ('a', val)])
            new_field = create_field(subfields=new_subs, ind1='1')
            record_replace_field(
                self.record, '653', new_field, field_position_global=field[4])

    def update_experiments(self):
        """Experiment mapping."""
        # 693 Remove if 'not applicable'
        for field in record_get_field_instances(self.record, '693'):
            subs = field_get_subfields(field)
            all_subs = subs.get('a', []) + subs.get('e', [])
            if 'not applicable' in [x.lower() for x in all_subs]:
                record_delete_field(self.record, '693',
                                    field_position_global=field[4])
            new_subs = []
            experiment_a = ""
            experiment_e = ""
            for (key, value) in subs.iteritems():
                if key == 'a':
                    experiment_a = value[0]
                    new_subs.append((key, value[0]))
                elif key == 'e':
                    experiment_e = value[0]
            experiment = "%s---%s" % (experiment_a.replace(" ", "-"),
                                      experiment_e)
            translated_experiments = self.get_config_item(experiment,
                                                          "experiments")
            new_subs.append(("e", translated_experiments))
            record_delete_field(self.record, tag="693",
                                field_position_global=field[4])
            record_add_field(self.record, "693", subfields=new_subs)

    def update_collaboration(self):
        """710 Collaboration."""
        for field in record_get_field_instances(self.record, '710'):
            subs = field_get_subfield_instances(field)
            for idx, (key, value) in enumerate(subs[:]):
                if key == '5':
                    subs.pop(idx)
                elif value.startswith('CERN. Geneva'):
                    subs.pop(idx)
            if len(subs) == 0:
                record_delete_field(self.record,
                                    tag='710',
                                    field_position_global=field[4])

    def update_journals(self):
        """773 journal translations."""
        for field in record_get_field_instances(self.record, '773'):
            subs = field_get_subfield_instances(field)
            new_subs = []
            for idx, (key, value) in enumerate(subs):
                if key == 'p':
                    journal_name = self.get_config_item(value, "journals", allow_substring=False)
                    journal_name = journal_name.replace('. ', '.').strip()
                    new_subs.append((key, journal_name))
                else:
                    new_subs.append((key, value))
            record_delete_field(self.record, tag="773",
                                field_position_global=field[4])
            record_add_field(self.record, "773", subfields=new_subs)

    def update_links_and_ffts(self):
        """FFT (856) Dealing with graphs."""
        figure_counter = 0
        for field in record_get_field_instances(self.record,
                                                tag='856',
                                                ind1='4'):
            subs = field_get_subfields(field)

            newsubs = []
            remove = False

            if 'z' in subs:
                is_figure = [s for s in subs['z'] if "figure" in s.lower()]
                if is_figure and 'u' in subs:
                    is_subformat = [
                        s for s in subs['u'] if "subformat" in s.lower()]
                    if not is_subformat:
                        url = subs['u'][0]
                        if url.endswith(".pdf"):
                            # We try to convert
                            fd, local_url = mkstemp(suffix=os.path.basename(url))
                            os.close(fd)
                            self.logger.info(
                                "Downloading %s into %s" % (url, local_url))
                            plotfile = ""
                            try:
                                plotfile = download_file(url=url,
                                                         download_to_file=local_url)
                            except Exception as e:
                                self.logger.exception(e)
                                remove = True
                            if plotfile:
                                converted = convert_images([plotfile])
                                if converted:
                                    url = converted.pop()
                                    msg = "Successfully converted %s to %s" \
                                          % (local_url, url)
                                    self.logger.info(msg)
                                else:
                                    msg = "Conversion failed on %s" \
                                          % (local_url,)
                                    self.logger.error(msg)
                                    url = None
                                    remove = True
                        if url:
                            newsubs.append(('a', url))
                            newsubs.append(('t', 'Plot'))
                            figure_counter += 1
                            if 'y' in subs:
                                newsubs.append(
                                    ('d', "%05d %s" % (figure_counter, subs['y'][0])))
                                newsubs.append(('n', subs['y'][0]))
                            else:
                                # Get basename without extension.
                                name = os.path.basename(
                                    os.path.splitext(subs['u'][0])[0])
                                newsubs.append(
                                    ('d', "%05d %s" % (figure_counter, name)))
                                newsubs.append(('n', name))

            if not newsubs and 'u' in subs:
                is_fulltext = [s for s in subs['u'] if ".pdf" in s]
                if is_fulltext:
                    newsubs = [('t', 'INSPIRE-PUBLIC'), ('a', subs['u'][0])]

            if not newsubs and 'u' in subs:
                remove = True


# Copyright (c) 2015 SUSE Linux GmbH.  All rights reserved.
#
# This file is part of kiwi.
#
# kiwi is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# kiwi is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with kiwi.  If not, see <http://www.gnu.org/licenses/>
#
import os
import struct
from collections import namedtuple

# In python2 bytes is string which is different from
# the bytes type in python3. The bytes type from the
# builtins generalizes this type to be bytes always
from builtins import bytes

# project
from kiwi.iso_tools.cdrtools import IsoToolsCdrTools
from kiwi.logger import log
from kiwi.defaults import Defaults
from kiwi.path import Path
from kiwi.command import Command
from kiwi.exceptions import (
    KiwiIsoLoaderError,
    KiwiIsoMetaDataError,
    KiwiCommandError
)

class Iso(object):
    """
    **Implements helper methods around the creation of ISO filesystems**

    :param str header_id: static identifier string for self written headers
    :param str header_end_name: file name to store the header_id to
    :param str header_end_file: full file path for the header_end_name file
    :param str boot_path: architecture specific boot path on the ISO
    """
    def __init__(self, source_dir):
        self.source_dir = source_dir
        self.header_id = '7984fc91-a43f-4e45-bf27-6d3aa08b24cf'
        self.header_end_name = 'header_end'
        self.header_end_file = self.source_dir + '/' + self.header_end_name
        self.boot_path = Defaults.get_iso_boot_path()

    @classmethod
    def create_hybrid(self, offset, mbrid, isofile, efi_mode=False):
        """
        Create hybrid ISO

        A hybrid ISO embeds both, an isolinux signature as well as a
        disk signature. kiwi always adds an msdos and a GPT table for
        the disk signatures

        :param str offset: hex offset
        :param str mbrid: boot record id
        :param str isofile: path to the ISO file
        :param bool efi_mode: sets the iso to support efi firmware or not
        """
        ignore_errors = [
            # we ignore this error message, for details see:
            # http://www.syslinux.org/archives/2015-March/023306.html
            'Warning: more than 1024 cylinders',
            'Not all BIOSes will be able to boot this device'
        ]
        isohybrid_parameters = [
            '--offset', format(offset),
            '--id', mbrid,
            '--type', '0x83',
        ]
        if efi_mode:
            isohybrid_parameters.append('--uefi')
        isohybrid_call = Command.run(
            ['isohybrid'] + isohybrid_parameters + [isofile]
        )
        # isohybrid warning messages on stderr should be treated
        # as fatal errors except for the ones we want to ignore
        # because unexpected after effects might happen if e.g a
        # gpt partition should be embedded but only a warning
        # appears if isohybrid can't find an efi loader. Thus we
        # are more strict and fail
        if isohybrid_call.error:
            error_list = isohybrid_call.error.split(os.linesep)
            error_fatal_list = []
            for error in error_list:
                ignore = False
                for ignore_error in ignore_errors:
                    if ignore_error in error:
                        ignore = True
                        break
                if not ignore and error:
                    error_fatal_list.append(error)
            if error_fatal_list:
                raise KiwiCommandError(
                    'isohybrid issue not ignored by kiwi: {0}'.format(
                        error_fatal_list
                    )
                )

    @classmethod
    def set_media_tag(self, isofile):
        """
        Include checksum tag in the ISO so it can be verified with
        the mediacheck program.

        :param str isofile: path to the ISO file
        """
        Command.run(
            [
                'tagmedia',
                '--md5',
                '--check',
                '--pad', '150',
                isofile
            ]
        )

    @classmethod
    def relocate_boot_catalog(self, isofile):
        """
        Move ISO boot catalog to the standardized place

        Check location of the boot catalog and move it to the place where
        all BIOS and firwmare implementations expects it

        :param str isofile: path to the ISO file
        """
        iso_metadata = Iso._read_iso_metadata(isofile)
        Iso._validate_iso_metadata(iso_metadata)
        with open(isofile, 'rb+') as iso:
            new_boot_catalog_sector = iso_metadata.path_table_sector - 1
            new_volume_descriptor = Iso._read_iso_sector(
                new_boot_catalog_sector - 1, iso
            )
            new_volume_id = Iso._sub_string(
                data=new_volume_descriptor, length=7
            )
            if bytes(b'CD001') not in new_volume_id:
                new_boot_catalog_sector = None
                ref_sector = iso_metadata.boot_catalog_sector
                for sector in range(0x12, 0x40):
                    new_volume_descriptor = Iso._read_iso_sector(sector, iso)
                    new_volume_id = Iso._sub_string(
                        data=new_volume_descriptor, length=7
                    )
                    if (bytes(b'TEA01') in new_volume_id or
                            sector + 1 == ref_sector):
                        new_boot_catalog_sector = sector + 1
                        break

            if (
                new_boot_catalog_sector and
                iso_metadata.boot_catalog_sector != new_boot_catalog_sector
            ):
                new_boot_catalog = Iso._read_iso_sector(
                    new_boot_catalog_sector, iso
                )
                empty_catalog = bytes(b'\x00') * 0x800
                if new_boot_catalog == empty_catalog:
                    eltorito_descriptor = Iso._embed_string_in_segment(
                        data=iso_metadata.eltorito_descriptor,
                        string=struct.pack('<I', new_boot_catalog_sector),
                        length=4,
                        start=0x47
                    )
                    Iso._write_iso_sector(
                        new_boot_catalog_sector, iso_metadata.boot_catalog, iso
                    )
                    Iso._write_iso_sector(
                        0x11, eltorito_descriptor, iso
                    )
                    log.debug(
                        'Relocated boot catalog from sector 0x%x to 0x%x',
                        iso_metadata.boot_catalog_sector,
                        new_boot_catalog_sector
                    )

    @classmethod
    def fix_boot_catalog(self, isofile):
        """
        Fixup inconsistencies in boot catalog

        Make sure all catalog entries are in correct order and provide
        complete metadata information e.g catalog name

        :param str isofile: path to the ISO file
        """
        iso_metadata = Iso._read_iso_metadata(isofile)
        Iso._validate_iso_metadata(iso_metadata)
        boot_catalog = iso_metadata.boot_catalog
        first_catalog_entry = Iso._sub_string(
            data=boot_catalog, length=32, start=32
        )
        first_catalog_entry = Iso._embed_string_in_segment(
            data=first_catalog_entry,
            string=struct.pack('B19s', 1, bytes(b'Legacy (isolinux)')),
            length=20,
            start=12
        )
        boot_catalog = Iso._embed_string_in_segment(
            data=boot_catalog,
            string=first_catalog_entry,
            length=32,
            start=32
        )
        second_catalog_entry = Iso._sub_string(
            data=boot_catalog, length=32, start=64
        )
        second_catalog_entry = Iso._embed_string_in_segment(
            data=second_catalog_entry,
            string=struct.pack('B19s', 1, bytes(b'UEFI (grub)')),
            length=20,
            start=12
        )
        second_catalog_entry_sector = second_catalog_entry[0]
        if second_catalog_entry_sector == 0x88:
            boot_catalog = Iso._embed_string_in_segment(
                data=boot_catalog,
                string=second_catalog_entry,
                length=32,
                start=96
            )
            second_catalog_entry = struct.pack(
                'BBH28s', 0x91, 0xef, 1, bytes(b'')
            )
            boot_catalog = Iso._embed_string_in_segment(
                data=boot_catalog,
                string=second_catalog_entry,
                length=32,
                start=64
            )
            with open(isofile, 'rb+') as iso:
                Iso._write_iso_sector(
                    iso_metadata.boot_catalog_sector, boot_catalog, iso
                )
            log.debug('Fixed iso catalog contents')

    def create_header_end_marker(self):
        """
        Prepare iso file to become a hybrid iso image.

        To do this the offest address of the end of the first iso
        block is required. To lookup this address a reference(marker)
        file named 'header_end' is created and will show up as last
        file in the block.
        """
        with open(self.header_end_file, 'w') as marker:
            marker.write('%s\n' % self.header_id)

    def create_header_end_block(self, isofile):
        """
        Find offset address of file containing the header_id and
        replace it by a list of 2k blocks in range 0 - offset + 1
        This is the required preparation to support hybrid ISO
        images, meaning to let isohybrid work correctly

        :param string isofile: path to the ISO file

        :raises KiwiIsoLoaderError: if the header_id file is not found
        :return: 512 byte blocks offset address

        :rtype: int
        """
        file_count = 0
        offset = 0
        found_id = False
        iso_tool = IsoToolsCdrTools(self.source_dir)
        with open(isofile, 'rb') as iso:
            for start in iso_tool.list_iso(isofile):
                if file_count >= 8:  # check only the first 8 files
                    break
                file_count += 1
                read_buffer = ''
                for index in range(0, -9, -1):  # go back up to 8 blocks
                    offset = start + index
                    iso.seek(offset << 11, 0)
                    read_buffer = iso.read(len(self.header_id))
                    if read_buffer == self.header_id.encode():
                        iso.seek(0, 0)
                        file_count = 8
                        found_id = True
                        break

            if found_id:
                log.debug('Found ISO header_end id at offset:')
                log.debug('--> 2k blocks: %d', offset)
                log.debug('--> 512 byte blocks(isohybrid): %d', offset * 4)
                log.debug('--> bytes(loop mount): %d', offset * 2048)
                with open(self.header_end_file, 'wb') as marker:
                    for index in range(offset + 1):
                        marker.write(iso.read(2048))
            else:
                raise KiwiIsoLoaderError(
                    'Header ID not found in iso file %s' % isofile
                )
            return offset * 4

    def setup_isolinux_boot_path(self):
        """
        Write the base boot path into the isolinux loader binary

        :raises KiwiIsoLoaderError: if loader/isolinux.bin is not found
        """
        loader_base_directory = self.boot_path + '/loader'
        loader_file = '/'.join(
            [self.source_dir, self.boot_path, 'loader/isolinux.bin']
        )
        if not os.path.exists(loader_file):
            raise KiwiIsoLoaderError(
                'No isolinux loader %s found'.format(loader_file)
            )
        try:
            Command.run(
                [
                    'isolinux-config', '--base', loader_base_directory,
                    loader_file
                ]
            )
        except Exception:
            # Setup of the base directory failed. This happens if
            # isolinux-config was not able to identify the isolinux
            # signature. As a workaround a compat directory /isolinux
            # is created which hardlinks all loader files