#  This file is part of ansi2html
#  Convert ANSI (terminal) colours and attributes to HTML
#  Copyright (C) 2012  Ralph Bean <rbean@redhat.com>
#  Copyright (C) 2013  Sebastian Pipping <sebastian@pipping.org>
#
#  Inspired by and developed off of the work by pixelbeat and blackjack.
#
#  This program is free software: you can redistribute it and/or
#  modify it under the terms of the GNU General Public License as
#  published by the Free Software Foundation, either version 3 of
#  the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see
#  <http://www.gnu.org/licenses/>.

import re
import sys
import optparse
import pkg_resources

try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

from ansi2html.style import get_styles
import six
from six.moves import map
from six.moves import zip

ANSI_FULL_RESET = 0
ANSI_INTENSITY_INCREASED = 1
ANSI_INTENSITY_REDUCED = 2
ANSI_INTENSITY_NORMAL = 22
ANSI_STYLE_ITALIC = 3
ANSI_STYLE_NORMAL = 23
ANSI_BLINK_SLOW = 5
ANSI_BLINK_FAST = 6
ANSI_BLINK_OFF = 25
ANSI_UNDERLINE_ON = 4
ANSI_UNDERLINE_OFF = 24
ANSI_CROSSED_OUT_ON = 9
ANSI_CROSSED_OUT_OFF = 29
ANSI_VISIBILITY_ON = 28
ANSI_VISIBILITY_OFF = 8
ANSI_FOREGROUND_CUSTOM_MIN = 30
ANSI_FOREGROUND_CUSTOM_MAX = 37
ANSI_FOREGROUND_256 = 38
ANSI_FOREGROUND_DEFAULT = 39
ANSI_BACKGROUND_CUSTOM_MIN = 40
ANSI_BACKGROUND_CUSTOM_MAX = 47
ANSI_BACKGROUND_256 = 48
ANSI_BACKGROUND_DEFAULT = 49
ANSI_NEGATIVE_ON = 7
ANSI_NEGATIVE_OFF = 27

_template = six.u("""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=%(output_encoding)s">

%(content)s

""")

class _State(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.intensity = ANSI_INTENSITY_NORMAL
        self.style = ANSI_STYLE_NORMAL
        self.blink = ANSI_BLINK_OFF
        self.underline = ANSI_UNDERLINE_OFF
        self.crossedout = ANSI_CROSSED_OUT_OFF
        self.visibility = ANSI_VISIBILITY_ON
        self.foreground = (ANSI_FOREGROUND_DEFAULT, None)
        self.background = (ANSI_BACKGROUND_DEFAULT, None)
        self.negative = ANSI_NEGATIVE_OFF

    def adjust(self, ansi_code, parameter=None):
        if ansi_code in (ANSI_INTENSITY_INCREASED, ANSI_INTENSITY_REDUCED, ANSI_INTENSITY_NORMAL):
            self.intensity = ansi_code
        elif ansi_code in (ANSI_STYLE_ITALIC, ANSI_STYLE_NORMAL):
            self.style = ansi_code
        elif ansi_code in (ANSI_BLINK_SLOW, ANSI_BLINK_FAST, ANSI_BLINK_OFF):
            self.blink = ansi_code
        elif ansi_code in (ANSI_UNDERLINE_ON, ANSI_UNDERLINE_OFF):
            self.underline = ansi_code
        elif ansi_code in (ANSI_CROSSED_OUT_ON, ANSI_CROSSED_OUT_OFF):
            self.crossedout = ansi_code
        elif ansi_code in (ANSI_VISIBILITY_ON, ANSI_VISIBILITY_OFF):
            self.visibility = ansi_code
        elif ANSI_FOREGROUND_CUSTOM_MIN <= ansi_code <= ANSI_FOREGROUND_CUSTOM_MAX:
            self.foreground = (ansi_code, None)
        elif ansi_code == ANSI_FOREGROUND_256:
            self.foreground = (ansi_code, parameter)
        elif ansi_code == ANSI_FOREGROUND_DEFAULT:
            self.foreground = (ansi_code, None)
        elif ANSI_BACKGROUND_CUSTOM_MIN <= ansi_code <= ANSI_BACKGROUND_CUSTOM_MAX:
            self.background = (ansi_code, None)
        elif ansi_code == ANSI_BACKGROUND_256:
            self.background = (ansi_code, parameter)
        elif ansi_code == ANSI_BACKGROUND_DEFAULT:
            self.background = (ansi_code, None)
        elif ansi_code in (ANSI_NEGATIVE_ON, ANSI_NEGATIVE_OFF):
            self.negative = ansi_code

    def to_css_classes(self):
        css_classes = []

        def append_unless_default(output, value, default):
            if value != default:
                css_class = 'ansi%d' % value
                output.append(css_class)

        def append_color_unless_default(output, color, default, negative, neg_css_class):
            value, parameter = color
            if value != default:
                prefix = 'inv' if negative else 'ansi'
                css_class_index = str(value) \
                        if (parameter is None) \
                        else '%d-%d' % (value, parameter)
                output.append(prefix + css_class_index)
            elif negative:
                output.append(neg_css_class)

        append_unless_default(css_classes, self.intensity, ANSI_INTENSITY_NORMAL)
        append_unless_default(css_classes, self.style, ANSI_STYLE_NORMAL)
        append_unless_default(css_classes, self.blink, ANSI_BLINK_OFF)
        append_unless_default(css_classes, self.underline, ANSI_UNDERLINE_OFF)
        append_unless_default(css_classes, self.crossedout, ANSI_CROSSED_OUT_OFF)
        append_unless_default(css_classes, self.visibility, ANSI_VISIBILITY_ON)

        flip_fore_and_background = (self.negative == ANSI_NEGATIVE_ON)
        append_color_unless_default(css_classes, self.foreground, ANSI_FOREGROUND_DEFAULT, flip_fore_and_background, 'inv_background')
        append_color_unless_default(css_classes, self.background, ANSI_BACKGROUND_DEFAULT, flip_fore_and_background, 'inv_foreground')

        return css_classes

def linkify(line):
    for match in re.findall(r'https?:\/\/\S+', line):
        line = line.replace(match, '%s' % (match, match))

    return line

def _needs_extra_newline(text):
    if not text or text.endswith('\n'):
        return False
    return True

class CursorMoveUp(object):
    pass

class Ansi2HTMLConverter(object):
    """ Convert Ansi color codes to CSS+HTML

    Example:
    >>> conv = Ansi2HTMLConverter()
    >>> ansi = " ".join(sys.stdin.readlines())
    >>> html = conv.convert(ansi)
    """

    def __init__(self,
                 inline=False,
                 dark_bg=True,
                 font_size='normal',
                 linkify=False,
                 escaped=True,
                 markup_lines=False,
                 output_encoding='utf-8',
                ):

        self.inline = inline
        self.dark_bg = dark_bg
        self.font_size = font_size
        self.linkify = linkify
        self.escaped = escaped
        self.markup_lines = markup_lines
        self.output_encoding = output_encoding
        self._attrs = None

        if inline:
            self.styles = dict([(item.klass.strip('.'), item) for item in get_styles(self.dark_bg)])

        self.ansi_codes_prog = re.compile('\033\\[' '([\\d;]*)' '([a-zA-z])')

    def apply_regex(self, ansi):
        parts = self._apply_regex(ansi)
        parts = self._collapse_cursor(parts)
        parts = list(parts)

        if self.linkify:
            parts = [linkify(part) for part in parts]

        combined = "".join(parts)

        if self.markup_lines:
            combined = "\n".join([
                """%s""" % (i, line)
                for i, line in enumerate(combined.split('\n'))
            ])

        return combined

    def _apply_regex(self, ansi):
        if self.escaped:
            specials = OrderedDict([
                ('&', '&'),
                ('<', '<'),
                ('>', '>'),
            ])
            for pattern, special in specials.items():
                ansi = ansi.replace(pattern, special)

        state = _State()
        inside_span = False
        last_end = 0  # the index of the last end of a code we've seen
        for match in self.ansi_codes_prog.finditer(ansi):
            yield ansi[last_end:match.start()]
            last_end = match.end()

            params, command = match.groups()

            if command not in 'mMA':
                continue

            # Special cursor-moving code.  The only supported one.
            if command == 'A':
                yield CursorMoveUp
                continue

            try:
                params = list(map(int, params.split(';')))
            except ValueError:
                params = [ANSI_FULL_RESET]

            # Find latest reset marker
            last_null_index = None
            skip_after_index = -1
            for i, v in enumerate(params):
                if i <= skip_after_index:
                    continue

                if v == ANSI_FULL_RESET:
                    last_null_index = i
                elif v in (ANSI_FOREGROUND_256, ANSI_BACKGROUND_256):
                    skip_after_index = i + 2

            # Process reset marker, drop everything before
            if last_null_index is not None:
                params = params[last_null_index + 1:]
                if inside_span:
                    inside_span = False
                    yield ''
                state.reset()

                if not params:
                    continue

            # Turn codes into CSS classes
            skip_after_index = -1
            for i, v in enumerate(params):
                if i <= skip_after_index:
                    continue

                if v in (ANSI_FOREGROUND_256, ANSI_BACKGROUND_256):
                    try:
                        parameter = params[i + 2]
                    except IndexError:
                        continue
                    skip_after_index = i + 2
                else:
                    parameter = None
                state.adjust(v, parameter=parameter)

            if inside_span:
                yield ''
                inside_span = False

            css_classes = state.to_css_classes()
            if not css_classes:
                continue

            if self.inline:
                style = [self.styles[klass].kw for klass in css_classes if
                         klass in self.styles]
                yield '' % "; ".join(style)
            else:
                yield '' % " ".join(css_classes)
            inside_span = True

        yield ansi[last_end:]
        if inside_span:
            yield ''
            inside_span = False

    def _collapse_cursor(self, parts):
        """ Act on any CursorMoveUp commands by deleting preceding tokens """

        final_parts = []
        for part in parts:

            # Throw out empty string tokens ("")
            if not part:
                continue

            # Go back, deleting every token in the last 'line'
            if part == CursorMoveUp:
                final_parts.pop()
                while '\n' not in final_parts[-1]:
                    final_parts.pop()

                continue

            # Otherwise, just pass this token forward
            final_parts.append(part)

        return final_parts

    def prepare(self, ansi='', ensure_trailing_newline=False):
        """ Load the contents of 'ansi' into this object """

        body = self.apply_regex(ansi)

        if ensure_trailing_newline and _needs_extra_newline(body):
            body += '\n'

        self._attrs = {
            'dark_bg': self.dark_bg,
            'font_size': self.font_size,
            'body': body,
        }

        return self._attrs

    def attrs(self):
        """ Prepare attributes for the template """
        if not self._attrs:
            raise Exception("Method .prepare not yet called.")
        return self._attrs

    def convert(self, ansi, full=True, ensure_trailing_newline=False):
        attrs = self.prepare(ansi, ensure_trailing_newline=ensure_trailing_newline)
        if not full:
            return attrs["body"]
        else:
            return _template % {
                'style' : "\n".join(map(str, get_styles(self.dark_bg))),
                'font_size' : self.font_size,
                'content' :  attrs["body"],
                'output_encoding' : self.output_encoding,
            }

    def produce_headers(self):
        return '\n' % {
            'style' : "\n".join(map(str, get_styles(self.dark_bg)))
        }

def main():
    """
    $ ls --color=always | ansi2html > directories.html
    $ sudo tail /var/log/messages | ccze -A | ansi2html > logs.html
    $ task burndown | ansi2html > burndown.html
    """

    version_str = pkg_resources.get_distribution('ansi2html').version
    parser = optparse.OptionParser(
        usage=main.__doc__,
        version="%%prog %s" % version_str)
    parser.add_option(
        "-p", "--partial", dest="partial",
        default=False, action="store_true",
        help="Process lines as them come in.  No headers are produced.")
    parser.add_option(
        "-i", "--inline", dest="inline",
        default=False, action="store_true",
        help="Inline style without headers or template.")
    parser.add_option(
        "-H", "--headers", dest="headers",
        default=False, action="store_true",
        help="Just produce the


# therm.py

"""
Module containing functions designed for thermogram time series analysis, with a focus on
peak signal detection during thermograms (Tmax).
"""

import pygaero._dchck as _check
import pandas as pd
import numpy as np
import peakutils

import matplotlib.pyplot as plt
import matplotlib.cm as cmx
from matplotlib import colors

__all__ = ['peak_find',
           'peakfind_df_ls',
           'plot_tmax',
           'plot_tmax_double',
           'flow_correction',
           'smooth',
           'get_cmap']

def peak_find(tseries_df, temp, ion_names, peak_threshold=0.05, min_dist=50, smth=False):
    """
    This function takes a pandas DataFrame containing desorption time series, along with a time series of Figaero
        heater temperatures. For each series (i.e., column in the DataFrame), the maximum value is found. For this,
        peakutils package is used. To ensure that a global maximum is found, parameters [peak_threshold] and [min_dist]
        may need to be optimized. However, the default values of 0.05 and 50 have been tested on desorption time series
        from several experiments, and no problems have been detected as of yet. Smaller min_dist may be needed to
        capture secondary TMax (TMax2) values.

    :param tseries_df: (DataFrame) pandas DataFrame with time series organized as the columns. Index should be
    :param temp: (float/int) Figaero desorption temperatures as recorded by EyeOn data, or other temperature logger.
    :param ion_names: (string) String names of ions to correspond to the time series in tseries_df (same order)
    :param peak_threshold: (float) Normalized peak threshold from 0. - 1.0.  Should be careful when using this in
            this DataFrame oriented function.  If some peaks are not found on some time series, it will not return
            the correct size dataframe and will throw an error because of passed-implied shape exception. Default value
            of 0.05 (or 5%) has been tested with figaero data from multiple experiments with no errors detected.
    :param min_dist: (int) The minimum distance of points between two peaks that will be identified. This is sensitive
            to the index resolution (e.g., time or temperature) of the input data in [tseries_df].
    :param smth: (bool) If true, time series are smoothed before finding the max values in the time series. NOT
            RECOMMENDED if the time series have already been smoothed before.
    :return: df_tmax: a pandas DataFrame of 5 columns with Tmax1, MaxSig1, Tmax2, MaxSig2, DubFlag (double peak flag)
    """
    # Check types of passed arguments to prevent errors during subsequent processing
    _check.check_dfs(values=[tseries_df])
    _check.check_num_equal(val1=tseries_df.shape[0], val2=len(temp))         # Number of datapoints must be equal
    _check.check_string(values=ion_names)
    _check.check_threshold(values=[peak_threshold], thresh=1.0000001, how='under')
    _check.check_threshold(values=[peak_threshold], thresh=0.0, how='over')
    _check.check_int(values=[min_dist])
    _check.check_bool(values=[smth])

    for i in range(0, len(tseries_df.columns)):
        # Process each ion's time series sequentially
        ion_tseries = tseries_df.values[:, i]

        if smth:
            ion_tseries = smooth(ion_tseries, window='hamming', window_len=11)

        # Find indices of peaks using peakutils.indexes (see code in libs/site-packages/peakutils).
        max_indices = peakutils.indexes(ion_tseries, thres=peak_threshold, min_dist=min_dist)
        # 2. The first two most prominent peaks are used to capture the major thermal behavior of the thermograms.
        dub_flag = 0
        npeaks = len(max_indices)
        # print('# of peaks for %s:' % tseries_df.columns[i], npeaks)                    # Debug line/optional output
        if npeaks == 0:
            # Optional output
            # print('No peaks above threshold found for ion %s. Assigning NaN to Tmax values.' % tseries_df.columns[i])
            if i == 0:
                TMax1 = []
                TMax2 = []
                MaxSig1 = []
                MaxSig2 = []
                DubFlag = []
                # give nan values to first ion since there are no peaks detected
            TMax1.append(np.nan)
            MaxSig1.append(np.nan)
            TMax2.append(np.nan)
            MaxSig2.append(np.nan)
            DubFlag.append(-1)
            # print('TMax1 for 0 peak ion now = ', TMax1[i])
        else:
            for j in max_indices:
                # print('j in max indices at count %.0f:' % dub_flag, j)        # debug line
                if i == 0:
                    if dub_flag == 0:
                        # Create Tmax/Tseries on very first pass (First element in max_indices for first ion)
                        TMax1 = [np.nan]
                        TMax2 = [np.nan]
                        MaxSig1 = [np.nan]
                        MaxSig2 = [np.nan]
                        DubFlag = [np.nan]
                        # Assign very first value
                        TMax1[i] = temp[j]
                        MaxSig1[i] = ion_tseries[j]
                        # print('first Tmax/SigMax assigned for ion', ion_names[i])
                        # print('Tmax =', TMax1, 'and MaxSig1 =', MaxSig1, '\n')
                        if npeaks == 1:
                            TMax2[i] = np.nan
                            TMax2[i] = np.nan
                            DubFlag[i] = 0
                    elif dub_flag == 1:
                        TMax2[i] = temp[j]
                        MaxSig2[i] = ion_tseries[j]
                        DubFlag[i] = 1
                    else:
                        pass
                    dub_flag += 1
                else:
                    if dub_flag == 0:
                        TMax1.append(temp[j])
                        MaxSig1.append(ion_tseries[j])
                        if npeaks == 1:
                            TMax2.append(np.nan)
                            MaxSig2.append(np.nan)
                            DubFlag.append(0)
                    elif dub_flag == 1:
                        TMax2.append(temp[j])
                        MaxSig2.append(ion_tseries[j])
                        DubFlag.append(1)
                    else:
                        pass
                    dub_flag += 1
    df_tmax = pd.DataFrame(data={'TMax1': TMax1,
                                 'MaxSig1': MaxSig1,
                                 'TMax2': TMax2,
                                 'MaxSig2': MaxSig2,
                                 'DubFlag': DubFlag},
                           index=tseries_df.columns.values)
    df_tmax.index.name = "Molecule"

    return df_tmax

def peakfind_df_ls(df_ls, pk_thresh=0.05, pk_win=50, min_temp=40.0, max_temp=190.0):
    """
    Function to apply function peak_find() to a list of pandas DataFrames, returning a list of DataFrames with
        TMax data. By default, peaks are only searched for within the temperature range 40.0 - 190.0 Celcius
    :param df_ls: (pandas DataFrames): List of pandas DataFrames to process using peak_find()
    :param pk_thresh: (float): Peak threshold between 0.0 - 1.0
    :param pk_win: (int): Value to be passed to parameter min_dist in peak_find(). See peak_find() documentation for
            more details
    :param min_temp: (float) Minimum temperature in celcius above which peak finding will take place
    :param max_temp: (float) Maximum temperature in celcius below which peak finding will take place.
    :return: df_tmax_ls: A list of pandas DataFrames with TMax data. See docstring for peak_find() for more information.
    """

    df_tmax_ls = []
    for df in df_ls:
        df_tmax = peak_find(tseries_df=df[(df.index.values < max_temp) & (df.index.values > min_temp)],
                            temp=df.index[(df.index.values < max_temp) & (df.index.values > min_temp)],
                            ion_names=df.columns,
                            peak_threshold=pk_thresh, min_dist=pk_win)
        df_tmax_ls.append(df_tmax)

    return df_tmax_ls

def plot_tmax(df, ions, tmax_temps, tmax_vals):
    """
    Function to plot the desorption time series for a set of specified ions. The index values of df (df.index.values)
        will be used for the x values during plotting. Tmax values are indicated by a circular red marker.
    :param df: (pandas DataFrame) pandas DataFrame containing the desorption time series
    :param ions: (string) List of string values (or np array) for the ions to plot
    :param tmax_vals: (float) Corresponding tmax values for the items listed in parameter [ions]
    :param tmax_temps: (float) Corresponding tmax temperatures for the items listed in parameter [ions]
    :return: Nothing returned. Plot popped to screen.
    """
    # Check data types to prevent errors during plotting
    _check.check_dfs(values=[df])
    _check.check_string(values=ions)
    _check.check_numeric(values=tmax_temps)
    _check.check_numeric(values=tmax_vals)

    n_series = len(ions)
    cmap = get_cmap(n=n_series, cm='hsv')

    for series_num, ion, tmax, maxsig in zip(range(0, n_series), ions, tmax_temps, tmax_vals):
        color = cmap(series_num)
        y = df.ix[:, ion].values
        plt.plot(df.index.values, y, linewidth=2, c=color, label=ion, zorder=0)
        plt.scatter(tmax, maxsig, marker='o', s=40, c='r', linewidths=1, zorder=1)

    plt.xlim((min(df.index.values)*.9, max(df.index.values)*1.1))
    plt.ylim((0, max(tmax_vals)*1.1))
    plt.legend(fontsize=8)
    plt.show()

    return 0

def plot_tmax_double(df, ions, tmax_temps, tmax_temps2, tmax_vals, tmax_vals2):
    """
    Function to plot the desorption time series for a set of specified ions. The index values of df (df.index.values)
        will be used for the x values during plotting. TMax values are indicated by a circular red marker. TMax2
        is also plotted for those time series that are bimodal (double peak in desorption thermogram)
    :param df: (pandas DataFrame) pandas DataFrame containing the desorption time series
    :param ions: (string) List of string values (or np array) for the ions to plot
    :param tmax_vals: (float) Corresponding tmax signal values for the items listed in parameter [ions]
    :param tmax_vals2: (float) Corresponding secondary peak tmax signal values for the items listed in parameter [ions]
    :param tmax_temps: (float) Corresponding tmax temperatures for the items listed in parameter [ions]
    :param tmax_temps2: (float) Corresponding secondary peak tmax temperatures for the items listed in parameter [ions]
    :return: Nothing returned. Plot popped to screen.
    """
    # Check data types to prevent errors during plotting
    _check.check_dfs(values=[df])
    _check.check_string(values=ions)
    _check.check_numeric(values=tmax_temps)
    _check.check_numeric(values=tmax_temps2)
    _check.check_numeric(values=tmax_vals)
    _check.check_numeric(values=tmax_vals2)

    n_series = len(ions)
    cmap = get_cmap(n=n_series, cm='hsv')

    for series_num, ion, tmax, tmax2, maxsig, maxsig2 in zip(range(0, n_series), ions, tmax_temps,
                                                             tmax_temps2, tmax_vals, tmax_vals2):
        color = cmap(series_num)
        y = df.ix[:, ion].values
        plt.plot(df.index.values, y, linewidth=2, c=color, label=ion, zorder=0)
        plt.scatter(tmax, maxsig, marker='o', s=50, c='r', zorder=1)          # Add TMax1
        plt.scatter(tmax2, maxsig2, marker='*', s=50, c='pink', zorder=1)    # Add TMax2

    plt.xlim((min(df.index.values)*.9, max(df.index.values)*1.1))
    plt.ylim((0, max(tmax_vals)*1.1))
    plt.legend(fontsize=8)
    plt.show()

    return 0

def flow_correction(thermograms, aero_samp_rates=[0.0], base_samp_rate=2.0):
    """
    A function to adjust desorption signals by the Figaero sample flow rate relative to the actual flow rate being
        pulled in by the CIMS. For example, if 4 LPM is used to sample aerosol, but the sample rate into the CIMS is
        2 LPM, then the thermograms need to be adjusted by *2/4. This is because the signal being sampled during
        desorptions is twice as concentrated as what would be sampled in the gas phase by a 2 LPM flow. This is useful
        and important if a direct comparison between gas and aerosol concentrations is to be made and if sensitivites
        obtained from gas-phase calibrations are to be applied to the aerosol signals.

        Note, thermograms are modified in-place, and overwritten.
    :param thermograms: (pandas DataFrames) Time series of aerosol desorption thermograms obtained from a Figaero
        ToF-CIMS
    :param aero_samp_rates: (float) Aerosol sample rates for the Figaero inlet. Typically this is higher than the
        base sample rate in order to reduce particle losses.
    :param base_samp_rate: (float) The base sample rate for the CIMS. Default is a nominal 2.0, which is popularly used.
        It is not suggested that this is changed unless a different sample rate has been verified.
    :return: (pandas DataFrame list) thermograms_corr: The desorption thermogram time series that have been adjusted
        for relative sample rates.
    """
    # Check data types to prevent subsequent errors
    _check.check_ls(ls=thermograms)
    _check.check_ls(ls=aero_samp_rates)
    _check.check_dfs(values=thermograms)
    _check.check_numeric(values=aero_samp_rates)
    _check.check_numeric(values=[base_samp_rate])

    for df, samp_rate in zip(thermograms, aero_samp_rates):
        for col in range(0, len(df.columns)):
            df.ix[:, col] = df.values[:, col] * (base_samp_rate / samp_rate)
    return 0

# Signal processing and plotting functions
def smooth(x, window_len=11, window='hanning'):
    """
    Note, this smoothing function was taken from the SciPy cookbook. Documentation, examples, and code to extend
     smoothing to 2 dimensional data can be found at https://scipy.github.io/old-wiki/pages/Cookbook/SignalSmooth
     "This method is based on the convolution of a scaled window with the signal. The signal is prepared by
     introducing reflected window-length copies of the signal at both ends so that boundary effect are minimized in
     the beginning and end part of the output signal."

    :param x: (float) Input signal to be smoothed
    :param window_len: (int, odd) the dimension of the smoothing window; should be an odd integer
    :param window: (string) The type of window from 'flat', 'hanning', 'hamming', 'barlett', 'blackmann'.  Flat window
        will produce a moving average smoothing.
    :return:
    """
    if x.ndim != 1:
            raise ValueError("smooth only accepts 1 dimension arrays.")
    if x.size < window_len:
            raise ValueError("Input vector needs to be bigger than window size.")
    if window_len < 3:
            return x
    # was 'if not window in'
    if window not in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
            raise ValueError("Window is one of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
    s = np.r_[2*x[0]-x[window_len-1::-1], x, 2*x[-1]-x[-1:-window_len:-1]]
    if window == 'flat':
            w = np.ones(window_len, 'd')
    else:
            w = eval('np.'+window+'(window_len)')
    y = np.convolve(w/w.sum(), s, mode='same')

    # return y of original length
    return y[window_len:-window_len+1]
    # return y

def get_cmap(n, cm='hsv'):
    """
    Returns a function that maps each index in 0, 1, ... N-1 to a distinct
    RGB color. (taken from stack overflow: (http://stackoverflow.com/questions/14720331/
    # how-to-generate-random-colors-in-matplotlib).
    :param n: # of colors in returned colormap
    :param cm: Choice of colormap to use.  Options include the following:
    :returns: Colormap of choice w/ N values
    cmaps = [('Perceptually Uniform Sequential',
                            ['viridis', 'inferno', 'plasma', 'magma']),
         ('Sequential',     ['Blues', 'BuGn', 'BuPu',
                             'GnBu', 'Greens', 'Greys', 'Oranges', 'OrRd',
                             'PuBu', 'PuBuGn', 'PuRd', 'Purples', 'RdPu',
                             'Reds', 'YlGn', 'YlGnBu', 'YlOrBr', 'YlOrRd']),
         ('Sequential (2)', ['afmhot', 'autumn', 'bone', 'cool',
                             'copper', 'gist_heat', 'gray', 'hot',
                             'pink', 'spring', 'summer', 'winter']),
         ('Diverging',      ['BrBG', 'bwr', 'coolwarm', 'PiYG', 'PRGn', 'PuOr',
                             'RdBu', 'RdGy', 'RdYlBu', 'RdYlGn', 'Spectral',
                             'seismic']),
         ('Qualitative',    ['Accent', 'Dark2', 'Paired', 'Pastel1',
                             'Pastel2', 'Set1', 'Set2', 'Set3']),
         ('Miscellaneous',  ['gist_earth', 'terrain', 'ocean', 'gist_stern',
                             'brg', 'CMRmap', 'cubehelix',
                             'gnuplot', 'gnuplot2', 'gist_ncar',
                             'nipy_spectral', 'jet', 'rainbow',


# Copyright (C) 2015-2017 East Asian Observatory
# All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful,but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,51 Franklin
# Street, Fifth Floor, Boston, MA  02110-1301, USA

from __future__ import absolute_import, division, print_function, \
    unicode_literals

from datetime import datetime

from hedwig.compat import first_value
from hedwig.error import ConsistencyError, DatabaseIntegrityError, Error
from hedwig.type.enum import MessageState, MessageThreadType
from hedwig.type.collection import ResultCollection
from hedwig.type.simple import Message, MessageRecipient
from .dummy_db import DBTestCase

class DBMessageTest(DBTestCase):
    def test_basic_message(self):
        # Check null results.
        messages = self.db.search_message()
        self.assertFalse(messages)

        # Create test message.
        person_1 = self.db.add_person('Person One')
        self.db.add_email(person_1, '1@a', primary=True, public=True)

        with self.assertRaisesRegex(Error, '^duplicate person_id'):
            message_id = self.db.add_message('test', 'test message',
                                             [person_1, person_1])

        with self.assertRaises(DatabaseIntegrityError):
            message_id = self.db.add_message('test', 'test message',
                                             [person_1, person_1],
                                             _test_skip_check=True)

        with self.assertRaises(DatabaseIntegrityError):
            message_id = self.db.add_message('test', 'test message',
                                             [1999999])

        message_id = self.db.add_message('test', 'test message', [person_1])
        self.assertIsInstance(message_id, int)

        # Test "search_message" method.
        messages = self.db.search_message()
        self.assertIsInstance(messages, ResultCollection)
        self.assertEqual(set(messages.keys()), set((message_id,)))
        message = messages[message_id]
        self.assertIsInstance(message, Message)

        self.assertEqual(message.id, message_id)
        self.assertEqual(message.subject, 'test')
        self.assertIsNone(message.body)
        self.assertIsNone(message.timestamp_send)
        self.assertIsNone(message.timestamp_sent)
        self.assertIsNone(message.identifier)
        self.assertIsNone(message.recipients)
        self.assertIsNone(message.thread_type)
        self.assertIsNone(message.thread_id)
        self.assertEqual(message.state, MessageState.UNSENT)

        # Test a search as if preparing to send messages.
        messages = self.db.search_message(
            state=MessageState.UNSENT,
            oldest_first=True, with_thread_identifiers=True,
            with_recipients=True, with_recipients_resolved=True,
            with_body=True)
        self.assertIsInstance(messages, ResultCollection)
        self.assertEqual(set(messages.keys()), set((message_id,)))
        message = messages[message_id]
        self.assertIsInstance(message, Message)
        self.assertEqual(message.id, message_id)
        self.assertEqual(message.subject, 'test')
        self.assertEqual(message.body, 'test message')
        self.assertIsNone(message.timestamp_send)
        self.assertIsNone(message.timestamp_sent)
        self.assertIsNone(message.identifier)
        self.assertIsNotNone(message.recipients)
        self.assertEqual(len(message.recipients), 1)
        recipient = list(message.recipients.values())[0]
        self.assertIsInstance(recipient, MessageRecipient)
        self.assertEqual(recipient.person_name, 'Person One')
        self.assertEqual(recipient.email_address, '1@a')
        self.assertTrue(recipient.email_public)

        self.db.update_message(
            message.id,
            state_prev=MessageState.UNSENT, state=MessageState.SENDING,
            state_is_system=True, timestamp_send=datetime.utcnow())

        messages = self.db.search_message()
        self.assertEqual(set(messages.keys()), set((message_id,)))
        message = messages[message_id]
        self.assertIsInstance(message.timestamp_send, datetime)
        self.assertEqual(message.state, MessageState.SENDING)

        self.assertFalse(self.db.search_message(state=MessageState.UNSENT))

        self.db.update_message(
            message_id,
            state_prev=MessageState.SENDING,
            state=MessageState.SENT,
            state_is_system=True,
            timestamp_sent=datetime.utcnow(),
            identifier='<1@localhost>')

        self.assertFalse(self.db.search_message(state=MessageState.UNSENT))

        messages = self.db.search_message()
        self.assertEqual(set(messages.keys()), set((message_id,)))
        message = messages[message_id]
        self.assertEqual(message.id, message_id)
        self.assertEqual(message.subject, 'test')
        self.assertIsNone(message.body)
        self.assertIsInstance(message.timestamp_send, datetime)
        self.assertIsInstance(message.timestamp_sent, datetime)
        self.assertEqual(message.identifier, '<1@localhost>')
        self.assertIsNone(message.recipients)
        self.assertEqual(message.state, MessageState.SENT)

        with self.assertRaisesRegex(ConsistencyError, '^message does not ex'):
            self.db.update_message(
                1999999, identifier='<2@localhost>')

        with self.assertRaisesRegex(ConsistencyError, '^no rows matched'):
            self.db.update_message(
                1999999, identifier='<2@localhost>', _test_skip_check=True)

        # Test "update_message" method.
        self.db.update_message(message_id=message_id,
                               state=MessageState.UNSENT)

        message = self.db.get_message(message_id)
        self.assertEqual(message.state, MessageState.UNSENT)

        self.db.update_message(message_id=message_id,
                               state=MessageState.DISCARD)

        messages = self.db.search_message()
        self.assertEqual(set(messages.keys()), set((message_id,)))
        message = messages[message_id]
        self.assertEqual(message.state, MessageState.DISCARD)

        self.assertFalse(self.db.search_message(state=MessageState.UNSENT))

        self.db.update_message(message_id=message_id,
                               state=MessageState.UNSENT)

        messages = self.db.search_message(state=MessageState.UNSENT)
        self.assertEqual(len(messages), 1)
        self.assertEqual(first_value(messages).id, message_id)

        with self.assertRaisesRegex(ConsistencyError, '^message does not'):
            self.db.update_message(
                message_id=1999999, state=MessageState.DISCARD)

        with self.assertRaisesRegex(ConsistencyError, '^no rows matched'):
            self.db.update_message(
                message_id=1999999, state=MessageState.DISCARD,
                _test_skip_check=True)

        with self.assertRaisesRegex(Error, '^invalid state'):
            self.db.update_message(message_id=message_id, state=999)

        with self.assertRaisesRegex(Error, '^no message updates specified'):
            self.db.update_message(message_id=message_id)

    def test_multiple_message(self):
        # Create some person records with multiple email addresses.
        person_1 = self.db.add_person('Person One')
        self.db.add_email(person_1, '1@a', primary=False, public=True)
        self.db.add_email(person_1, '1@b', primary=True, public=True)
        self.db.add_email(person_1, '1@c', primary=False, public=True)

        person_2 = self.db.add_person('Person Two')
        self.db.add_email(person_2, '2@a', primary=True, public=False)
        self.db.add_email(person_2, '2@b', primary=False, public=True)
        self.db.add_email(person_2, '2@c', primary=False, public=True)

        person_3 = self.db.add_person('Person Three')
        self.db.add_email(person_3, '3@a', primary=False, public=False)
        self.db.add_email(person_3, '3@b', primary=False, public=True)
        self.db.add_email(person_3, '3@c', primary=True, public=True)

        person_4 = self.db.add_person('Person Four')
        self.db.add_email(person_4, '4@a', primary=False, public=False)
        self.db.add_email(person_4, '4@b', primary=True, public=False)
        self.db.add_email(person_4, '4@c', primary=False, public=True)

        # Create two test messages.
        message_12 = self.db.add_message('test', 'test message',
                                         [person_1, person_2])
        message_34 = self.db.add_message('test', 'test message',
                                         [person_3, person_4])

        self.assertIsInstance(message_12, int)
        self.assertIsInstance(message_34, int)
        self.assertNotEqual(message_12, message_34)

        # Check that each message has the correct set of recipients.
        messages = self.db.search_message(
            state=MessageState.UNSENT,
            oldest_first=True, with_thread_identifiers=True,
            with_recipients=True, with_recipients_resolved=True,
            with_body=True)
        self.assertEqual(list(messages.keys()), [message_12, message_34])

        message = messages[message_12]
        self.assertEqual(message.id, message_12)
        self.assertEqual(set(message.recipients.values()), set((
            MessageRecipient(message_12, person_1, '1@b', 'Person One', True),
            MessageRecipient(message_12, person_2, '2@a', 'Person Two', False),
        )))

        message = messages[message_34]
        self.assertEqual(message.id, message_34)
        self.assertEqual(set(message.recipients.values()), set((
            MessageRecipient(message_34, person_3, '3@c', 'Person Three', True),
            MessageRecipient(message_34, person_4, '4@b', 'Person Four', False),
        )))

    def test_explicit_email(self):
        # Create some person records with multiple email addresses.
        person_1 = self.db.add_person('Person One')
        self.db.add_email(person_1, '1@a', primary=False, public=True)
        self.db.add_email(person_1, '1@b', primary=True, public=True)


# Copyright (C) 2015-2020  Chris Lalancette <clalancette@gmail.com>

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

'''
The class to support ISO9660 Directory Records.
'''

from __future__ import absolute_import

import bisect
import struct

from pycdlib import dates
from pycdlib import inode
from pycdlib import pycdlibexception
from pycdlib import rockridge
from pycdlib import utils

# For mypy annotations
if False:  # pylint: disable=using-constant-test
    from typing import BinaryIO, List, Optional, Tuple, Union  # NOQA pylint: disable=unused-import
    # NOTE: these imports have to be here to avoid circular deps
    from pycdlib import headervd  # NOQA pylint: disable=unused-import
    from pycdlib import path_table_record  # NOQA pylint: disable=unused-import

class XARecord(object):
    '''
    A class that represents an ISO9660 Extended Attribute record as defined
    in the Philips Yellow Book standard.
    '''
    __slots__ = ('_initialized', '_group_id', '_user_id', '_attributes',
                 '_filenum', '_pad_size')

    FMT = '=HHH2sB5s'

    def __init__(self):
        # type: () -> None
        self._pad_size = 0
        self._initialized = False

    def parse(self, xastr, len_fi):
        # type: (bytes, int) -> bool
        '''
        Parse an Extended Attribute Record out of a string.

        Parameters:
         xastr - The string to parse.
         len_fi - The length of the file identifier for this record.
        Returns:
         True if the data contains an XA record, False otherwise.
        '''
        if self._initialized:
            raise pycdlibexception.PyCdlibInternalError('This XARecord is already initialized')

        even_size = len_fi + (len_fi % 2)
        # In a "typical" XA record, the record immediately follows the DR
        # record (but comes before the Rock Ridge record, if this is a Rock
        # Ridge ISO).  However, we have seen ISOs (Windows 98 SE) that put some
        # padding between the end of the DR record and the XA record.  As far
        # as I can tell, that padding is the size of the file identifier,
        # but rounded up to the nearest even number.  We check both places for
        # the XA record.
        for offset in (0, even_size):
            parse_str = xastr[offset:]
            if len(parse_str) < struct.calcsize(self.FMT):
                return False

            (self._group_id, self._user_id, self._attributes, signature,
             self._filenum, unused) = struct.unpack_from(self.FMT, parse_str, 0)
            if signature != b'XA':
                continue

            if unused != b'\x00\x00\x00\x00\x00':
                raise pycdlibexception.PyCdlibInvalidISO('Unused fields should be 0')

            self._pad_size = offset
            break
        else:
            return False

        self._initialized = True

        return True

    def new(self):
        # type: () -> None
        '''
        Create a new Extended Attribute Record.

        Parameters:
         None.
        Returns:
         Nothing.
        '''
        if self._initialized:
            raise pycdlibexception.PyCdlibInternalError('This XARecord is already initialized')

        # FIXME: we should allow the user to set these
        self._group_id = 0
        self._user_id = 0
        self._attributes = 0
        self._filenum = 0
        self._initialized = True

    def record(self):
        # type: () -> bytes
        '''
        Record this Extended Attribute Record.

        Parameters:
         None.
        Returns:
         A string representing this Extended Attribute Record.
        '''
        if not self._initialized:
            raise pycdlibexception.PyCdlibInternalError('This XARecord is not initialized')

        return b'\x00' * self._pad_size + struct.pack(self.FMT, self._group_id,
                                                      self._user_id,
                                                      self._attributes,
                                                      b'XA', self._filenum,
                                                      b'\x00' * 5)

    @staticmethod
    def length():
        # type: () -> int
        '''
        A static method to return the size of an Extended Attribute Record.

        Parameters:
         None.
        Returns:
         The size of an Extended Attribute Record.
        '''
        return 14

class DirectoryRecord(object):
    '''
    A class that represents an ISO9660 directory record.
    '''
    __slots__ = ('initialized', 'new_extent_loc', 'ptr', 'extents_to_here',
                 'offset_to_here', 'data_continuation', 'vd', 'children',
                 'rr_children', 'inode', '_printable_name', 'date',
                 'index_in_parent', 'dr_len', 'xattr_len', 'file_flags',
                 'file_unit_size', 'interleave_gap_size', 'len_fi', 'isdir',
                 'orig_extent_loc', 'data_length', 'seqnum', 'is_root',
                 'parent', 'rock_ridge', 'xa_record', 'file_ident')

    FILE_FLAG_EXISTENCE_BIT = 0
    FILE_FLAG_DIRECTORY_BIT = 1
    FILE_FLAG_ASSOCIATED_FILE_BIT = 2
    FILE_FLAG_RECORD_BIT = 3
    FILE_FLAG_PROTECTION_BIT = 4
    FILE_FLAG_MULTI_EXTENT_BIT = 7

    FMT = '<BBLLLL7sBBBHHB'

    def __init__(self):
        # type: () -> None
        self.initialized = False
        self.new_extent_loc = -1
        self.ptr = None  # type: Optional[path_table_record.PathTableRecord]
        self.extents_to_here = 1
        self.offset_to_here = 0
        self.data_continuation = None  # type: Optional[DirectoryRecord]
        self.children = []  # type: List[DirectoryRecord]
        self.rr_children = []  # type: List[DirectoryRecord]
        self.index_in_parent = -1
        self.is_root = False
        self.isdir = False
        self.rock_ridge = None  # type: Optional[rockridge.RockRidge]
        self.xa_record = None  # type: Optional[XARecord]
        self.inode = None  # type: Optional[inode.Inode]

    def parse(self, vd, record, parent):
        # type: (headervd.PrimaryOrSupplementaryVD, bytes, Optional[DirectoryRecord]) -> str
        '''
        Parse a directory record out of a string.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         record - The string to parse for this record.
         parent - The parent of this record.
        Returns:
         The Rock Ridge version as a string if this Directory Record has Rock
         Ridge, '' otherwise.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        if len(record) > 255:
            # Since the length is supposed to be 8 bits, this should never
            # happen.
            raise pycdlibexception.PyCdlibInvalidISO('Directory record longer than 255 bytes!')

        # According to http://www.dubeyko.com/development/FileSystems/ISO9960/ISO9960.html,
        # the xattr_len is the number of bytes at the *beginning* of the file
        # extent.  Since this is only a byte, it is necessarily limited to 255
        # bytes.
        (self.dr_len, self.xattr_len, extent_location_le, extent_location_be,
         data_length_le, data_length_be_unused, dr_date, self.file_flags,
         self.file_unit_size, self.interleave_gap_size, seqnum_le, seqnum_be,
         self.len_fi) = struct.unpack_from(self.FMT, record[:33], 0)

        # In theory we should have a check here that checks to make sure that
        # the length of the record we were passed in matches the data record
        # length.  However, we have seen ISOs in the wild where this is
        # incorrect, so we elide the check here.

        if extent_location_le != utils.swab_32bit(extent_location_be):
            raise pycdlibexception.PyCdlibInvalidISO('Little-endian (%d) and big-endian (%d) extent location disagree' % (extent_location_le, utils.swab_32bit(extent_location_be)))
        self.orig_extent_loc = extent_location_le

        # Theoretically, we should check to make sure that the little endian
        # data length is the same as the big endian data length.  In practice,
        # though, we've seen ISOs where this is wrong.  Skip the check, and just
        # pick the little-endian as the 'actual' size, and hope for the best.

        self.data_length = data_length_le

        if seqnum_le != utils.swab_16bit(seqnum_be):
            raise pycdlibexception.PyCdlibInvalidISO('Little-endian and big-endian seqnum disagree')
        self.seqnum = seqnum_le

        self.date = dates.DirectoryRecordDate()
        self.date.parse(dr_date)

        # OK, we've unpacked what we can from the beginning of the string.  Now
        # we have to use the len_fi to get the rest.

        self.parent = parent
        self.vd = vd

        if self.parent is None:
            self.is_root = True

            # A root directory entry should always be exactly 34 bytes.
            # However, we have seen ISOs in the wild that get this wrong, so we
            # elide a check for it.

            self.file_ident = bytes(bytearray([record[33]]))

            # A root directory entry should always have 0 as the identifier.
            # However, we have seen ISOs in the wild that don't have this set
            # properly to 0.  In that case, we override what we parsed out from
            # the original with the correct value (\x00), and hope for the best.
            if self.file_ident != b'\x00':
                self.file_ident = b'\x00'
            self.isdir = True
        else:
            record_offset = 33
            self.file_ident = record[record_offset:record_offset + self.len_fi]
            record_offset += self.len_fi
            if self.file_flags & (1 << self.FILE_FLAG_DIRECTORY_BIT):
                self.isdir = True

            if self.len_fi % 2 == 0:
                record_offset += 1

            xa_rec = XARecord()
            if xa_rec.parse(record[record_offset:], self.len_fi):
                self.xa_record = xa_rec
                record_offset += len(self.xa_record.record())

            if len(record[record_offset:]) >= 2 and record[record_offset:record_offset + 2] in (b'SP', b'RR', b'CE', b'PX', b'ER', b'ES', b'PN', b'SL', b'NM', b'CL', b'PL', b'TF', b'SF', b'RE', b'AL'):
                self.rock_ridge = rockridge.RockRidge()

                is_first_dir_record_of_root = False

                if self.parent.is_root:
                    if self.file_ident == b'\x00':
                        is_first_dir_record_of_root = True
                        bytes_to_skip = 0
                    else:
                        if not self.parent.children:
                            raise pycdlibexception.PyCdlibInvalidISO('Parent has no dot child')
                        if self.parent.children[0].rock_ridge is None:
                            raise pycdlibexception.PyCdlibInvalidISO('Dot child does not have Rock Ridge; ISO is corrupt')
                        bytes_to_skip = self.parent.children[0].rock_ridge.bytes_to_skip
                else:
                    if self.parent.rock_ridge is None:
                        raise pycdlibexception.PyCdlibInvalidISO('Parent does not have Rock Ridge; ISO is corrupt')
                    bytes_to_skip = self.parent.rock_ridge.bytes_to_skip

                self.rock_ridge.parse(record[record_offset:],
                                      is_first_dir_record_of_root,
                                      bytes_to_skip,
                                      False)

        if self.xattr_len != 0:
            if self.file_flags & (1 << self.FILE_FLAG_RECORD_BIT):
                raise pycdlibexception.PyCdlibInvalidISO('Record Bit not allowed with Extended Attributes')
            if self.file_flags & (1 << self.FILE_FLAG_PROTECTION_BIT):
                raise pycdlibexception.PyCdlibInvalidISO('Protection Bit not allowed with Extended Attributes')

        if self.rock_ridge is None:
            ret = ''
        else:
            ret = self.rock_ridge.rr_version

        if self.is_root:
            self._printable_name = '/'.encode(vd.encoding)
        elif self.file_ident == b'\x00':
            self._printable_name = '.'.encode(vd.encoding)
        elif self.file_ident == b'\x01':
            self._printable_name = '..'.encode(vd.encoding)
        else:
            self._printable_name = self.file_ident

        self.initialized = True

        return ret

    def _rr_new(self, rr_version, rr_name, rr_symlink_target, rr_relocated_child,
                rr_relocated, rr_relocated_parent, file_mode):
        # type: (str, bytes, bytes, bool, bool, bool, int) -> None
        '''
        Internal method to add Rock Ridge to a Directory Record.

        Parameters:
         rr_version - A string containing the version of Rock Ridge to use for
                      this record.
         rr_name - The Rock Ridge name to associate with this directory record.
         rr_symlink_target - The target for the symlink, if this is a symlink
                             record (otherwise, None).
         rr_relocated_child - True if this is a directory record for a rock
                              ridge relocated child.
         rr_relocated - True if this is a directory record for a relocated
                        entry.
         rr_relocated_parent - True if this is a directory record for a rock
                               ridge relocated parent.
         file_mode - The Unix file mode for this Rock Ridge entry.
        Returns:
         Nothing.
        '''

        if self.parent is None:
            raise pycdlibexception.PyCdlibInternalError('Invalid call to create new Rock Ridge on root directory')

        self.rock_ridge = rockridge.RockRidge()
        is_first_dir_record_of_root = self.file_ident == b'\x00' and self.parent.is_root
        bytes_to_skip = 0
        if self.xa_record is not None:
            bytes_to_skip = XARecord.length()
        self.dr_len = self.rock_ridge.new(is_first_dir_record_of_root, rr_name,
                                          file_mode, rr_symlink_target,
                                          rr_version, rr_relocated_child,
                                          rr_relocated, rr_relocated_parent,
                                          bytes_to_skip, self.dr_len, {})

        # For files, we are done
        if not self.isdir:
            return

        # If this is a directory, we have to manipulate the file links
        # appropriately.
        if self.parent.is_root:
            if self.file_ident == b'\x00' or self.file_ident == b'\x01':
                # For the dot and dotdot children of the root, add one
                # directly to their Rock Ridge links.
                self.rock_ridge.add_to_file_links()
            else:
                # For all other children of the root, make sure to add one
                # to each of the dot and dotdot entries.
                if len(self.parent.children) < 2:
                    raise pycdlibexception.PyCdlibInvalidISO('Expected at least 2 children of the root directory record, saw %d' % (len(self.parent.children)))

                if self.parent.children[0].rock_ridge is None:
                    raise pycdlibexception.PyCdlibInvalidISO('Dot child of directory has no Rock Ridge; ISO is corrupt')
                self.parent.children[0].rock_ridge.add_to_file_links()

                if self.parent.children[1].rock_ridge is None:
                    raise pycdlibexception.PyCdlibInvalidISO('Dot-dot child of directory has no Rock Ridge; ISO is corrupt')
                self.parent.children[1].rock_ridge.add_to_file_links()
        else:
            if self.parent.rock_ridge is None:
                raise pycdlibexception.PyCdlibInvalidISO('Parent of the entry did not have Rock Ridge, ISO is corrupt')

            if self.file_ident == b'\x00':
                # If we are adding the dot directory, increment the parent
                # file links and our file links.
                self.parent.rock_ridge.add_to_file_links()
                self.rock_ridge.add_to_file_links()
            elif self.file_ident == b'\x01':
                # If we are adding the dotdot directory, copy the file links
                # from the dot directory of the grandparent.
                if self.parent.parent is None:
                    raise pycdlibexception.PyCdlibInternalError('Grandparent of the entry did not exist; this cannot be')
                if not self.parent.children:
                    raise pycdlibexception.PyCdlibInvalidISO('Grandparent of the entry did not have a dot entry; ISO is corrupt')
                if self.parent.parent.children[0].rock_ridge is None:
                    raise pycdlibexception.PyCdlibInvalidISO('Grandparent dotdot entry did not have Rock Ridge; ISO is corrupt')
                self.rock_ridge.copy_file_links(self.parent.parent.children[0].rock_ridge)
            else:
                # For all other entries, increment the parents file links
                # and the parents dot file links.
                self.parent.rock_ridge.add_to_file_links()

                if not self.parent.children:
                    raise pycdlibexception.PyCdlibInvalidISO('Parent of the entry did not have a dot entry; ISO is corrupt')
                if self.parent.children[0].rock_ridge is None:
                    raise pycdlibexception.PyCdlibInvalidISO('Dot child of the parent did not have a dot entry; ISO is corrupt')
                self.parent.children[0].rock_ridge.add_to_file_links()

    def _new(self, vd, name, parent, seqnum, isdir, length, xa):
        # type: (headervd.PrimaryOrSupplementaryVD, bytes, Optional[DirectoryRecord], int, bool, int, bool) -> None
        '''
        Internal method to create a new Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         name - The name for this directory record.
         parent - The parent of this directory record.
         seqnum - The sequence number to associate with this directory record.
         isdir - Whether this directory record represents a directory.
         length - The length of the data for this directory record.
         xa - True if this is an Extended Attribute record.
        Returns:
         Nothing.
        '''

        # Adding a new time should really be done when we are going to write
        # the ISO (in record()).  Ecma-119 9.1.5 says:
        #
        # 'This field shall indicate the date and the time of the day at which
        # the information in the Extent described by the Directory Record was
        # recorded.'
        #
        # We create it here just to have something in the field, but we'll
        # redo the whole thing when we are mastering.
        self.date = dates.DirectoryRecordDate()
        self.date.new()

        if length > 2**32 - 1:
            raise pycdlibexception.PyCdlibInvalidInput('Maximum supported file length is 2^32-1')

        self.data_length = length

        self.file_ident = name

        self.isdir = isdir

        self.seqnum = seqnum
        # For a new directory record entry, there is no original_extent_loc,
        # so we leave it at None.
        self.orig_extent_loc = None
        self.len_fi = len(self.file_ident)
        self.dr_len = struct.calcsize(self.FMT) + self.len_fi

        # From Ecma-119, 9.1.6, the file flag bits are:
        #
        # Bit 0 - Existence - 0 for existence known, 1 for hidden
        # Bit 1 - Directory - 0 for file, 1 for directory
        # Bit 2 - Associated File - 0 for not associated, 1 for associated
        # Bit 3 - Record - 0=structure not in xattr, 1=structure in xattr
        # Bit 4 - Protection - 0=no owner and group, 1=owner and group in xattr
        # Bit 5 - Reserved
        # Bit 6 - Reserved
        # Bit 7 - Multi-extent - 0=final directory record, 1=not final directory record
        self.file_flags = 0
        if self.isdir:
            self.file_flags |= (1 << self.FILE_FLAG_DIRECTORY_BIT)
        self.file_unit_size = 0  # FIXME: we don't support setting file unit size for now
        self.interleave_gap_size = 0  # FIXME: we don't support setting interleave gap size for now
        self.xattr_len = 0  # FIXME: we don't support xattrs for now

        self.parent = parent
        if parent is None:
            # If no parent, then this is the root
            self.is_root = True

        if xa:
            self.xa_record = XARecord()
            self.xa_record.new()
            self.dr_len += XARecord.length()

        self.dr_len += (self.dr_len % 2)

        if self.is_root:
            self._printable_name = '/'.encode(vd.encoding)
        elif self.file_ident == b'\x00':
            self._printable_name = '.'.encode(vd.encoding)
        elif self.file_ident == b'\x01':
            self._printable_name = '..'.encode(vd.encoding)
        else:
            self._printable_name = self.file_ident

        self.vd = vd

        self.initialized = True

    def new_symlink(self, vd, name, parent, rr_target, seqnum, rock_ridge,
                    rr_name, xa):
        # type: (headervd.PrimaryOrSupplementaryVD, bytes, DirectoryRecord, bytes, int, str, bytes, bool) -> None
        '''
        Create a new symlink Directory Record.  This implies that the new
        record will be Rock Ridge.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         name - The name for this directory record.
         parent - The parent of this directory record.
         rr_target - The symlink target for this directory record.
         seqnum - The sequence number for this directory record.
         rock_ridge - The version of Rock Ridge to use for this directory record.
         rr_name - The Rock Ridge name for this directory record.
         xa - True if this is an Extended Attribute record.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, name, parent, seqnum, False, 0, xa)
        if rock_ridge:
            self._rr_new(rock_ridge, rr_name, rr_target, False, False, False,
                         0o0120555)

    def new_file(self, vd, length, isoname, parent, seqnum, rock_ridge, rr_name,
                 xa, file_mode):
        # type: (headervd.PrimaryOrSupplementaryVD, int, bytes, DirectoryRecord, int, str, bytes, bool, int) -> None
        '''
        Create a new file Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         length - The length of the data.
         isoname - The name for this directory record.
         parent - The parent of this directory record.
         seqnum - The sequence number for this directory record.
         rock_ridge - Whether to make this a Rock Ridge directory record.
         rr_name - The Rock Ridge name for this directory record.
         xa - True if this is an Extended Attribute record.
         file_mode - The POSIX file mode for this entry.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, isoname, parent, seqnum, False, length, xa)
        if rock_ridge:
            self._rr_new(rock_ridge, rr_name, b'', False, False, False,
                         file_mode)

    def new_root(self, vd, seqnum, log_block_size):
        # type: (headervd.PrimaryOrSupplementaryVD, int, int) -> None
        '''
        Create a new root Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         seqnum - The sequence number for this directory record.
         log_block_size - The logical block size to use.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, b'\x00', None, seqnum, True, log_block_size, False)

    def new_dot(self, vd, parent, seqnum, rock_ridge, log_block_size, xa,
                file_mode):
        # type: (headervd.PrimaryOrSupplementaryVD, DirectoryRecord, int, str, int, bool, int) -> None
        '''
        Create a new 'dot' Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         parent - The parent of this directory record.
         seqnum - The sequence number for this directory record.
         rock_ridge - Whether to make this a Rock Ridge directory record.
         log_block_size - The logical block size to use.
         xa - True if this is an Extended Attribute record.
         file_mode - The POSIX file mode to set for this directory.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, b'\x00', parent, seqnum, True, log_block_size, xa)
        if rock_ridge:
            self._rr_new(rock_ridge, b'', b'', False, False, False, file_mode)

    def new_dotdot(self, vd, parent, seqnum, rock_ridge, log_block_size,
                   rr_relocated_parent, xa, file_mode):
        # type: (headervd.PrimaryOrSupplementaryVD, DirectoryRecord, int, str, int, bool, bool, int) -> None
        '''
        Create a new 'dotdot' Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         parent - The parent of this directory record.
         seqnum - The sequence number for this directory record.
         rock_ridge - Whether to make this a Rock Ridge directory record.
         log_block_size - The logical block size to use.
         rr_relocated_parent - True if this is a Rock Ridge relocated parent.
         xa - True if this is an Extended Attribute record.
         file_mode - The POSIX file mode to set for this directory.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, b'\x01', parent, seqnum, True, log_block_size, xa)
        if rock_ridge:
            self._rr_new(rock_ridge, b'', b'', False, False, rr_relocated_parent, file_mode)

    def new_dir(self, vd, name, parent, seqnum, rock_ridge, rr_name,
                log_block_size, rr_relocated_child, rr_relocated, xa, file_mode):
        # type: (headervd.PrimaryOrSupplementaryVD, bytes, DirectoryRecord, int, str, bytes, int, bool, bool, bool, int) -> None
        '''
        Create a new directory Directory Record.

        Parameters:
         vd - The Volume Descriptor this record is part of.
         name - The name for this directory record.
         parent - The parent of this directory record.
         seqnum - The sequence number for this directory record.
         rock_ridge - Whether to make this a Rock Ridge directory record.
         rr_name - The Rock Ridge name for this directory record.
         log_block_size - The logical block size to use.
         rr_relocated_child - True if this is a Rock Ridge relocated child.
         rr_relocated - True if this is a Rock Ridge relocated entry.
         xa - True if this is an Extended Attribute record.
         file_mode - The POSIX file mode to set for this directory.
        Returns:
         Nothing.
        '''
        if self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record already initialized')

        self._new(vd, name, parent, seqnum, True, log_block_size, xa)
        if rock_ridge:
            self._rr_new(rock_ridge, rr_name, b'', rr_relocated_child,
                         rr_relocated, False, file_mode)
            if rr_relocated_child and self.rock_ridge:
                # Relocated Rock Ridge entries are not exactly treated as
                # directories, so fix things up here.
                self.isdir = False
                self.file_flags = 0
                self.rock_ridge.add_to_file_links()

    def change_existence(self, is_hidden):
        # type: (bool) -> None
        '''
        Change the ISO9660 existence flag of this Directory Record.

        Parameters:
         is_hidden - True if this Directory Record should be hidden,
                     False otherwise.
        Returns:
         Nothing.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')

        if is_hidden:
            self.file_flags |= (1 << self.FILE_FLAG_EXISTENCE_BIT)
        else:
            self.file_flags &= ~(1 << self.FILE_FLAG_EXISTENCE_BIT)

    def _recalculate_extents_and_offsets(self, index, logical_block_size):
        # type: (int, int) -> Tuple[int, int]
        '''
        Internal method to recalculate the extents and offsets associated with
        children of this directory record.

        Parameters:
         index - The index at which to start the recalculation.
         logical_block_size - The block size to use for comparisons.
        Returns:
         A tuple where the first element is the total number of extents required
         by the children and where the second element is the offset into the
         last extent currently being used.
        '''
        if index == 0:
            dirrecord_offset = 0
            num_extents = 1
        else:
            dirrecord_offset = self.children[index - 1].offset_to_here
            num_extents = self.children[index - 1].extents_to_here

        for i in range(index, len(self.children)):
            c = self.children[i]
            dirrecord_len = c.dr_len
            if (dirrecord_offset + dirrecord_len) > logical_block_size:
                num_extents += 1
                dirrecord_offset = 0
            dirrecord_offset += dirrecord_len
            c.extents_to_here = num_extents
            c.offset_to_here = dirrecord_offset
            c.index_in_parent = i

        return num_extents, dirrecord_offset

    def _add_child(self, child, logical_block_size, allow_duplicate,
                   check_overflow):
        # type: (DirectoryRecord, int, bool, bool) -> bool
        '''
        An internal method to add a child to this object.  Note that this is
        called both during parsing and when adding a new object to the system,
        so it shouldn't have any functionality that is not appropriate for both.

        Parameters:
         child - The child directory record object to add.
         logical_block_size - The size of a logical block for this volume
                              descriptor.
         allow_duplicate - Whether to allow duplicate names, as there are
                           situations where duplicate children are allowed.
         check_overflow - Whether to check for overflow; if we are parsing, we
                          don't want to do this.
        Returns:
         True if adding this child caused the directory to overflow into another
         extent, False otherwise.
        '''
        if not self.isdir:
            raise pycdlibexception.PyCdlibInvalidInput('Trying to add a child to a record that is not a directory')

        # First ensure that this is not a duplicate.  For speed purposes, we
        # recognize that bisect_left will always choose an index to the *left*
        # of a duplicate child.  Thus, to check for duplicates we only need to
        # see if the child to be added is a duplicate with the entry that
        # bisect_left returned.
        index = bisect.bisect_left(self.children, child)
        if index != len(self.children) and self.children[index].file_ident == child.file_ident:
            if not self.children[index].is_associated_file() and not child.is_associated_file():
                if not (self.rock_ridge is not None and self.file_identifier() == b'RR_MOVED'):
                    if not allow_duplicate:
                        raise pycdlibexception.PyCdlibInvalidInput('Failed adding duplicate name to parent')

                    self.children[index].data_continuation = child
                    self.children[index].file_flags |= (1 << self.FILE_FLAG_MULTI_EXTENT_BIT)
                    index += 1
        self.children.insert(index, child)

        if child.rock_ridge is not None and not child.is_dot() and not child.is_dotdot():
            lo = 0
            hi = len(self.rr_children)
            while lo < hi:
                mid = (lo + hi) // 2
                rr = self.rr_children[mid].rock_ridge
                if rr is not None:
                    if rr.name() < child.rock_ridge.name():
                        lo = mid + 1
                    else:
                        hi = mid
                else:
                    raise pycdlibexception.PyCdlibInternalError('Expected all children to have Rock Ridge, but one did not')
            rr_index = lo

            self.rr_children.insert(rr_index, child)

        # We now have to check if we need to add another logical block.
        # We have to iterate over the entire list again, because where we
        # placed this last entry may rearrange the empty spaces in the blocks
        # that we've already allocated.
        num_extents, offset_unused = self._recalculate_extents_and_offsets(index,
                                                                           logical_block_size)

        overflowed = False
        if check_overflow and (num_extents * logical_block_size > self.data_length):
            overflowed = True
            # When we overflow our data length, we always add a full block.
            self.data_length += logical_block_size
            # We also have to make sure to update the length of the dot child,
            # as that should always reflect the length.
            self.children[0].data_length = self.data_length
            # We also have to update all of the dotdot entries.  If this is
            # the root directory record (no parent), we first update the root
            # dotdot entry.  In all cases, we update the dotdot entry of all
            # children that are directories.
            if self.parent is None:
                self.children[1].data_length = self.data_length

            for c in self.children:
                if not c.is_dir():
                    continue
                if len(c.children) > 1:
                    c.children[1].data_length = self.data_length

        return overflowed

    def add_child(self, child, logical_block_size, allow_duplicate=False):
        # type: (DirectoryRecord, int, bool) -> bool
        '''
        Add a new child to this directory record.

        Parameters:
         child - The child directory record object to add.
         logical_block_size - The size of a logical block for this volume
                              descriptor.
         allow_duplicate - Whether to allow duplicate names, as there are
                           situations where duplicate children are allowed.
        Returns:
         True if adding this child caused the directory to overflow into another
         extent, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')

        return self._add_child(child, logical_block_size, allow_duplicate, True)

    def track_child(self, child, logical_block_size, allow_duplicate=False):
        # type: (DirectoryRecord, int, bool) -> None
        '''
        Track an existing child of this directory record.

        Parameters:
         child - The child directory record object to add.
         logical_block_size - The size of a logical block for this volume
                              descriptor.
         allow_duplicate - Whether to allow duplicate names, as there are
                           situations where duplicate children are allowed.
        Returns:
         Nothing.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')

        self._add_child(child, logical_block_size, allow_duplicate, False)

    def remove_child(self, child, index, logical_block_size):
        # type: (DirectoryRecord, int, int) -> bool
        '''
        Remove a child from this Directory Record.

        Parameters:
         child - The child DirectoryRecord object to remove.
         index - The index of the child into this DirectoryRecord children list.
         logical_block_size - The size of a logical block on this volume
                              descriptor.
        Returns:
         True if removing this child caused an underflow, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')

        if index < 0:
            # This should never happen
            raise pycdlibexception.PyCdlibInternalError('Invalid child index to remove')

        # Unfortunately, Rock Ridge specifies that a CL 'directory' is replaced
        # by a *file*, not another directory.  Thus, we can't just depend on
        # whether this child is marked as a directory by the file flags during
        # parse time.  Instead, we check if this is either a true directory,
        # or a Rock Ridge CL entry, and in either case try to manipulate the
        # file links.
        if child.rock_ridge is not None:
            if child.isdir or child.rock_ridge.child_link_record_exists():
                if len(self.children) < 2:
                    raise pycdlibexception.PyCdlibInvalidISO('Expected a dot and dotdot entry, but missing; ISO is corrupt')
                if self.children[0].rock_ridge is None or self.children[1].rock_ridge is None:
                    raise pycdlibexception.PyCdlibInvalidISO('Missing Rock Ridge entry on dot or dotdot; ISO is corrupt')

                if self.parent is None:
                    self.children[0].rock_ridge.remove_from_file_links()
                    self.children[1].rock_ridge.remove_from_file_links()
                else:
                    if self.rock_ridge is None:
                        raise pycdlibexception.PyCdlibInvalidISO('Child has Rock Ridge, but parent does not; ISO is corrupt')
                    self.rock_ridge.remove_from_file_links()

                    self.children[0].rock_ridge.remove_from_file_links()

        del self.children[index]

        # We now have to check if we need to remove a logical block.
        # We have to iterate over the entire list again, because where we
        # removed this last entry may rearrange the empty spaces in the blocks
        # that we've already allocated.
        num_extents, dirrecord_offset = self._recalculate_extents_and_offsets(index,
                                                                              logical_block_size)

        underflow = False
        total_size = (num_extents - 1) * logical_block_size + dirrecord_offset
        if (self.data_length - total_size) > logical_block_size:
            self.data_length -= logical_block_size
            # We also have to make sure to update the length of the dot child,
            # as that should always reflect the length.
            self.children[0].data_length = self.data_length
            # We also have to update all of the dotdot entries.  If this is
            # the root directory record (no parent), we first update the root
            # dotdot entry.  In all cases, we update the dotdot entry of all
            # children that are directories.
            if self.parent is None:
                self.children[1].data_length = self.data_length

            for c in self.children:
                if not c.is_dir():
                    continue
                if len(c.children) > 1:
                    c.children[1].data_length = self.data_length
            underflow = True

        return underflow

    def is_dir(self):
        # type: () -> bool
        '''
        Determine whether this Directory Record is a directory.

        Parameters:
         None.
        Returns:
         True if this DirectoryRecord object is a directory, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self.isdir

    def is_file(self):
        # type: () -> bool
        '''
        Determine whether this Directory Record is a file.

        Parameters:
         None.
        Returns:
         True if this DirectoryRecord object is a file, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return not self.isdir

    def is_symlink(self):
        # type: () -> bool
        '''
        Determine whether this Directory Record is a Rock Ridge
        symlink.  If using this to distinguish between symlinks, files, and
        directories, it is important to call this API *first*; symlinks are
        also considered files.

        Parameters:
         None.
        Returns:
         True if this Directory Record object is a symlink, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self.rock_ridge is not None and self.rock_ridge.is_symlink()

    def is_dot(self):
        # type: () -> bool
        '''
        Determine whether this Directory Record is a 'dot' entry.

        Parameters:
         None.
        Returns:
         True if this DirectoryRecord object is a 'dot' entry, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self.file_ident == b'\x00'

    def is_dotdot(self):
        # type: () -> bool
        '''
        Determine whether this Directory Record is a 'dotdot' entry.

        Parameters:
         None.
        Returns:
         True if this DirectoryRecord object is a 'dotdot' entry, False otherwise.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self.file_ident == b'\x01'

    def directory_record_length(self):
        # type: () -> int
        '''
        Determine the length of this Directory Record.

        Parameters:
         None.
        Returns:
         The length of this Directory Record.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self.dr_len

    def _extent_location(self):
        '''
        An internal method to get the location of this Directory Record on the
        ISO.

        Parameters:
         None.
        Returns:
         Extent location of this Directory Record on the ISO.
        '''
        if self.new_extent_loc < 0:
            return self.orig_extent_loc
        return self.new_extent_loc

    def extent_location(self):
        # type: () -> int
        '''
        Get the location of this Directory Record on the ISO.

        Parameters:
         None.
        Returns:
         Extent location of this Directory Record on the ISO.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self._extent_location()

    def file_identifier(self):
        # type: () -> bytes
        '''
        Get the identifier of this Directory Record.

        Parameters:
         None.
        Returns:
         String representing the identifier of this Directory Record.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')
        return self._printable_name

    def record(self):
        # type: () -> bytes
        '''
        Generate the string representing this Directory Record.

        Parameters:
         None.
        Returns:
         String representing this Directory Record.
        '''
        if not self.initialized:
            raise pycdlibexception.PyCdlibInternalError('Directory Record not initialized')

        # Ecma-119 9.1.5 says the date should reflect the time when the
        # record was written, so we make a new date now and use that to
        # write out the record.
        self.date = dates.DirectoryRecordDate()
        self.date.new()

        padlen = struct.calcsize(self.FMT) + self.len_fi


# Copyright 2012 the rootpy developers
# distributed under the terms of the GNU General Public License
"""
This module handles conversion of ROOT's TFile and
contained TTrees into HDF5 format with PyTables
"""
from __future__ import absolute_import

import os
import sys
import warnings
import pkg_resources

import tables
TABLES_NEW_API = int(pkg_resources.parse_version(tables.__version__)[0]) >= 3
if TABLES_NEW_API:
    tables_open = tables.open_file
else:
    tables_open = tables.openFile

from root_numpy import tree2rec, RootNumpyUnconvertibleWarning
from numpy.lib import recfunctions

from .io import root_open, TemporaryFile
from . import log; log = log[__name__]
from .extern.progressbar import ProgressBar, Bar, ETA, Percentage
from .logger.utils import check_tty

__all__ = [
    'tree2hdf5',
    'root2hdf5',
]

def _drop_object_col(rec, warn=True):
    # ignore columns of type `object` since PyTables does not support these
    if rec.dtype.hasobject:
        object_fields = []
        fields = rec.dtype.fields
        for name in rec.dtype.names:
            if fields[name][0].kind == 'O':
                object_fields.append(name)
                if warn:
                    log.warning(
                        "ignoring unsupported object branch '{0}'".format(
                            name))
        # NumPy 1.7.1: TypeError: Cannot change data-type for object array.
        #return rec[non_object_fields]
        if object_fields:
            rec = recfunctions.rec_drop_fields(rec, object_fields)
    return rec

def tree2hdf5(tree, hfile, group=None,
              entries=-1, selection=None,
              show_progress=False):
    """
    Convert a TTree into a HDF5 table.

    Parameters
    ----------

    tree : ROOT.TTree
        A ROOT TTree.

    hfile : string or PyTables HDF5 File
        A PyTables HDF5 File handle or string path to an existing HDF5 file.

    group : string or PyTables Group instance, optional (default=None)
        Write the table at this location in the HDF5 file.

    entries : int, optional (default=-1)
        The number of entries to read at once while converting a ROOT TTree
        into an HDF5 table. By default read the entire TTree into memory (this
        may not be desired if your TTrees are large).

    selection : string, optional (default=None)
        A ROOT selection expression to be applied on the TTree before
        conversion.

    show_progress : bool, optional (default=False)
        If True, then display and update a progress bar on stdout as the TTree
        is converted.

    """
    show_progress = show_progress and check_tty(sys.stdout)
    if show_progress:
        widgets = [Percentage(), ' ', Bar(), ' ', ETA()]

    own_h5file = False
    if isinstance(hfile, basestring):
        hfile = tables_open(filename=hfile, mode="w", title="Data")
        own_h5file = True

    log.info("Converting tree '{0}' with {1:d} entries ...".format(
        tree.GetName(),
        tree.GetEntries()))

    if not group:
        group = hfile.root
    elif isinstance(group, basestring):
        group_where = '/' + os.path.dirname(group)
        group_name = os.path.basename(group)
        if TABLES_NEW_API:
            group = hfile.create_group(group_where, group_name,
                                       createparents=True)
        else:
            group = hfile.createGroup(group_where, group_name)

    if tree.GetName() in group:
        log.warning(
            "Tree '{0}' already exists "
            "in the output file".format(tree.GetName()))
        return

    total_entries = tree.GetEntries()
    pbar = None
    if show_progress and total_entries > 0:
        pbar = ProgressBar(widgets=widgets, maxval=total_entries)

    if entries <= 0:
        # read the entire tree
        if pbar is not None:
            pbar.start()
        recarray = tree2rec(tree, selection=selection)
        recarray = _drop_object_col(recarray)
        if TABLES_NEW_API:
            table = hfile.create_table(
                group, tree.GetName(),
                recarray, tree.GetTitle())
        else:
            table = hfile.createTable(
                group, tree.GetName(),
                recarray, tree.GetTitle())
        # flush data in the table
        table.flush()
        # flush all pending data
        hfile.flush()
    else:
        # read the tree in chunks
        start = 0
        while start < total_entries or start == 0:
            if start > 0:
                with warnings.catch_warnings():
                    warnings.simplefilter(
                        "ignore",
                        RootNumpyUnconvertibleWarning)
                    warnings.simplefilter(
                        "ignore",
                        tables.NaturalNameWarning)
                    recarray = tree2rec(
                        tree,
                        selection=selection,
                        start=start,
                        stop=start + entries)
                recarray = _drop_object_col(recarray, warn=False)
                table.append(recarray)
            else:
                recarray = tree2rec(
                    tree,
                    selection=selection,
                    start=start,
                    stop=start + entries)
                recarray = _drop_object_col(recarray)
                if pbar is not None:
                    # start after any output from root_numpy
                    pbar.start()
                if TABLES_NEW_API:
                    table = hfile.create_table(
                        group, tree.GetName(),
                        recarray, tree.GetTitle())
                else:
                    table = hfile.createTable(
                        group, tree.GetName(),
                        recarray, tree.GetTitle())
            start += entries
            if start <= total_entries and pbar is not None:
                pbar.update(start)
            # flush data in the table
            table.flush()
            # flush all pending data
            hfile.flush()

    if pbar is not None:
        pbar.finish()

    if own_h5file:
        hfile.close()

def root2hdf5(rfile, hfile, rpath='',
              entries=-1, userfunc=None,
              selection=None,
              show_progress=False,
              ignore_exception=False):
    """
    Convert all trees in a ROOT file into tables in an HDF5 file.

    Parameters
    ----------

    rfile : string or asrootpy'd ROOT File
        A ROOT File handle or string path to an existing ROOT file.

    hfile : string or PyTables HDF5 File
        A PyTables HDF5 File handle or string path to an existing HDF5 file.

    rpath : string, optional (default='')
        Top level path to begin traversal through the ROOT file. By default
        convert everything in and below the root directory.

    entries : int, optional (default=-1)
        The number of entries to read at once while converting a ROOT TTree
        into an HDF5 table. By default read the entire TTree into memory (this
        may not be desired if your TTrees are large).

    userfunc : callable, optional (default=None)
        A function that will be called on every tree and that must return a
        tree or list of trees that will be converted instead of the original
        tree.