from datetime import datetime, timedelta, date, time

import nose
from pandas.compat import lrange, zip

import numpy as np
from numpy.testing.decorators import slow

from pandas import Index, Series, DataFrame

from pandas.tseries.index import date_range, bdate_range
from pandas.tseries.offsets import DateOffset
from pandas.tseries.period import period_range, Period, PeriodIndex
from pandas.tseries.resample import DatetimeIndex

from pandas.util.testing import assert_series_equal, ensure_clean
import pandas.util.testing as tm

from pandas.tests.test_graphics import _skip_if_no_scipy_gaussian_kde

@tm.mplskip
class TestTSPlot(tm.TestCase):
    def setUp(self):
        freq = ['S', 'T', 'H', 'D', 'W', 'M', 'Q', 'A']
        idx = [period_range('12/31/1999', freq=x, periods=100) for x in freq]
        self.period_ser = [Series(np.random.randn(len(x)), x) for x in idx]
        self.period_df = [DataFrame(np.random.randn(len(x), 3), index=x,
                                    columns=['A', 'B', 'C'])
                          for x in idx]

        freq = ['S', 'T', 'H', 'D', 'W', 'M', 'Q-DEC', 'A', '1B30Min']
        idx = [date_range('12/31/1999', freq=x, periods=100) for x in freq]
        self.datetime_ser = [Series(np.random.randn(len(x)), x) for x in idx]
        self.datetime_df = [DataFrame(np.random.randn(len(x), 3), index=x,
                                      columns=['A', 'B', 'C'])
                            for x in idx]

    def tearDown(self):
        tm.close()

    @slow
    def test_ts_plot_with_tz(self):
        # GH2877
        index = date_range('1/1/2011', periods=2, freq='H',
                           tz='Europe/Brussels')
        ts = Series([188.5, 328.25], index=index)
        _check_plot_works(ts.plot)

    def test_fontsize_set_correctly(self):
        # For issue #8765
        import matplotlib.pyplot as plt  # noqa
        df = DataFrame(np.random.randn(10, 9), index=range(10))
        ax = df.plot(fontsize=2)
        for label in (ax.get_xticklabels() + ax.get_yticklabels()):
            self.assertEqual(label.get_fontsize(), 2)

    @slow
    def test_frame_inferred(self):
        # inferred freq
        import matplotlib.pyplot as plt  # noqa
        idx = date_range('1/1/1987', freq='MS', periods=100)
        idx = DatetimeIndex(idx.values, freq=None)

        df = DataFrame(np.random.randn(len(idx), 3), index=idx)
        _check_plot_works(df.plot)

        # axes freq
        idx = idx[0:40].union(idx[45:99])
        df2 = DataFrame(np.random.randn(len(idx), 3), index=idx)
        _check_plot_works(df2.plot)

        # N > 1
        idx = date_range('2008-1-1 00:15:00', freq='15T', periods=10)
        idx = DatetimeIndex(idx.values, freq=None)
        df = DataFrame(np.random.randn(len(idx), 3), index=idx)
        _check_plot_works(df.plot)

    def test_nonnumeric_exclude(self):
        import matplotlib.pyplot as plt

        idx = date_range('1/1/1987', freq='A', periods=3)
        df = DataFrame({'A': ["x", "y", "z"], 'B': [1, 2, 3]}, idx)

        ax = df.plot()  # it works
        self.assertEqual(len(ax.get_lines()), 1)  # B was plotted
        plt.close(plt.gcf())

        self.assertRaises(TypeError, df['A'].plot)

    @slow
    def test_tsplot(self):
        from pandas.tseries.plotting import tsplot
        import matplotlib.pyplot as plt

        ax = plt.gca()
        ts = tm.makeTimeSeries()

        f = lambda *args, **kwds: tsplot(s, plt.Axes.plot, *args, **kwds)

        for s in self.period_ser:
            _check_plot_works(f, s.index.freq, ax=ax, series=s)

        for s in self.datetime_ser:
            _check_plot_works(f, s.index.freq.rule_code, ax=ax, series=s)

        for s in self.period_ser:
            _check_plot_works(s.plot, ax=ax)

        for s in self.datetime_ser:
            _check_plot_works(s.plot, ax=ax)

        ax = ts.plot(style='k')
        self.assertEqual((0., 0., 0.), ax.get_lines()[0].get_color())

    def test_both_style_and_color(self):
        import matplotlib.pyplot as plt  # noqa

        ts = tm.makeTimeSeries()
        self.assertRaises(ValueError, ts.plot, style='b-', color='#000099')

        s = ts.reset_index(drop=True)
        self.assertRaises(ValueError, s.plot, style='b-', color='#000099')

    @slow
    def test_high_freq(self):
        freaks = ['ms', 'us']
        for freq in freaks:
            rng = date_range('1/1/2012', periods=100000, freq=freq)
            ser = Series(np.random.randn(len(rng)), rng)
            _check_plot_works(ser.plot)

    def test_get_datevalue(self):
        from pandas.tseries.converter import get_datevalue
        self.assertIsNone(get_datevalue(None, 'D'))
        self.assertEqual(get_datevalue(1987, 'A'), 1987)
        self.assertEqual(get_datevalue(Period(1987, 'A'), 'M'),
                         Period('1987-12', 'M').ordinal)
        self.assertEqual(get_datevalue('1/1/1987', 'D'),
                         Period('1987-1-1', 'D').ordinal)

    @slow
    def test_ts_plot_format_coord(self):
        def check_format_of_first_point(ax, expected_string):
            first_line = ax.get_lines()[0]
            first_x = first_line.get_xdata()[0].ordinal
            first_y = first_line.get_ydata()[0]
            try:
                self.assertEqual(expected_string,
                                 ax.format_coord(first_x, first_y))
            except (ValueError):
                raise nose.SkipTest("skipping test because issue forming "
                                    "test comparison GH7664")

        annual = Series(1, index=date_range('2014-01-01', periods=3,
                                            freq='A-DEC'))
        check_format_of_first_point(annual.plot(), 't = 2014  y = 1.000000')

        # note this is added to the annual plot already in existence, and
        # changes its freq field
        daily = Series(1, index=date_range('2014-01-01', periods=3, freq='D'))
        check_format_of_first_point(daily.plot(),
                                    't = 2014-01-01  y = 1.000000')
        tm.close()

        # tsplot
        import matplotlib.pyplot as plt
        from pandas.tseries.plotting import tsplot
        tsplot(annual, plt.Axes.plot)
        check_format_of_first_point(plt.gca(), 't = 2014  y = 1.000000')
        tsplot(daily, plt.Axes.plot)
        check_format_of_first_point(plt.gca(), 't = 2014-01-01  y = 1.000000')

    @slow
    def test_line_plot_period_series(self):
        for s in self.period_ser:
            _check_plot_works(s.plot, s.index.freq)

    @slow
    def test_line_plot_datetime_series(self):
        for s in self.datetime_ser:
            _check_plot_works(s.plot, s.index.freq.rule_code)

    @slow
    def test_line_plot_period_frame(self):
        for df in self.period_df:
            _check_plot_works(df.plot, df.index.freq)

    @slow
    def test_line_plot_datetime_frame(self):
        for df in self.datetime_df:
            freq = df.index.to_period(df.index.freq.rule_code).freq
            _check_plot_works(df.plot, freq)

    @slow
    def test_line_plot_inferred_freq(self):
        for ser in self.datetime_ser:
            ser = Series(ser.values, Index(np.asarray(ser.index)))
            _check_plot_works(ser.plot, ser.index.inferred_freq)

            ser = ser[[0, 3, 5, 6]]
            _check_plot_works(ser.plot)

    def test_fake_inferred_business(self):
        import matplotlib.pyplot as plt
        fig = plt.gcf()
        plt.clf()
        fig.add_subplot(111)
        rng = date_range('2001-1-1', '2001-1-10')
        ts = Series(lrange(len(rng)), rng)
        ts = ts[:3].append(ts[5:])
        ax = ts.plot()
        self.assertFalse(hasattr(ax, 'freq'))

    @slow
    def test_plot_offset_freq(self):
        ser = tm.makeTimeSeries()
        _check_plot_works(ser.plot)

        dr = date_range(ser.index[0], freq='BQS', periods=10)
        ser = Series(np.random.randn(len(dr)), dr)
        _check_plot_works(ser.plot)

    @slow
    def test_plot_multiple_inferred_freq(self):
        dr = Index([datetime(2000, 1, 1), datetime(2000, 1, 6), datetime(
            2000, 1, 11)])
        ser = Series(np.random.randn(len(dr)), dr)
        _check_plot_works(ser.plot)

    @slow
    def test_uhf(self):
        import pandas.tseries.converter as conv
        import matplotlib.pyplot as plt
        fig = plt.gcf()
        plt.clf()
        fig.add_subplot(111)

        idx = date_range('2012-6-22 21:59:51.960928', freq='L', periods=500)
        df = DataFrame(np.random.randn(len(idx), 2), idx)

        ax = df.plot()
        axis = ax.get_xaxis()

        tlocs = axis.get_ticklocs()
        tlabels = axis.get_ticklabels()
        for loc, label in zip(tlocs, tlabels):
            xp = conv._from_ordinal(loc).strftime('%H:%M:%S.%f')
            rs = str(label.get_text())
            if len(rs):
                self.assertEqual(xp, rs)

    @slow
    def test_irreg_hf(self):
        import matplotlib.pyplot as plt
        fig = plt.gcf()
        plt.clf()
        fig.add_subplot(111)

        idx = date_range('2012-6-22 21:59:51', freq='S', periods=100)
        df = DataFrame(np.random.randn(len(idx), 2), idx)

        irreg = df.ix[[0, 1, 3, 4]]
        ax = irreg.plot()
        diffs = Series(ax.get_lines()[0].get_xydata()[:, 0]).diff()

        sec = 1. / 24 / 60 / 60
        self.assertTrue((np.fabs(diffs[1:] - [sec, sec * 2, sec]) < 1e-8).all(
        ))

        plt.clf()
        fig.add_subplot(111)
        df2 = df.copy()
        df2.index = df.index.asobject
        ax = df2.plot()
        diffs = Series(ax.get_lines()[0].get_xydata()[:, 0]).diff()
        self.assertTrue((np.fabs(diffs[1:] - sec) < 1e-8).all())

    def test_irregular_datetime64_repr_bug(self):
        import matplotlib.pyplot as plt
        ser = tm.makeTimeSeries()
        ser = ser[[0, 1, 2, 7]]

        fig = plt.gcf()
        plt.clf()
        ax = fig.add_subplot(211)
        ret = ser.plot()
        self.assertIsNotNone(ret)

        for rs, xp in zip(ax.get_lines()[0].get_xdata(), ser.index):
            self.assertEqual(rs, xp)

    def test_business_freq(self):
        import matplotlib.pyplot as plt  # noqa
        bts = tm.makePeriodSeries()
        ax = bts.plot()
        self.assertEqual(ax.get_lines()[0].get_xydata()[0, 0],
                         bts.index[0].ordinal)
        idx = ax.get_lines()[0].get_xdata()
        self.assertEqual(PeriodIndex(data=idx).freqstr, 'B')

    @slow
    def test_business_freq_convert(self):
        n = tm.N
        tm.N = 300
        bts = tm.makeTimeSeries().asfreq('BM')
        tm.N = n
        ts = bts.to_period('M')
        ax = bts.plot()
        self.assertEqual(ax.get_lines()[0].get_xydata()[0, 0],
                         ts.index[0].ordinal)
        idx = ax.get_lines()[0].get_xdata()
        self.assertEqual(PeriodIndex(data=idx).freqstr, 'M')

    def test_nonzero_base(self):
        # GH2571
        idx = (date_range('2012-12-20', periods=24, freq='H') + timedelta(
            minutes=30))
        df = DataFrame(np.arange(24), index=idx)
        ax = df.plot()
        rs = ax.get_lines()[0].get_xdata()
        self.assertFalse(Index(rs).is_normalized)

    def test_dataframe(self):
        bts = DataFrame({'a': tm.makeTimeSeries()})
        ax = bts.plot()
        idx = ax.get_lines()[0].get_xdata()
        tm.assert_numpy_array_equal(bts.index.to_period(), PeriodIndex(idx))

    @slow
    def test_axis_limits(self):
        import matplotlib.pyplot as plt

        def _test(ax):
            xlim = ax.get_xlim()
            ax.set_xlim(xlim[0] - 5, xlim[1] + 10)
            ax.get_figure().canvas.draw()
            result = ax.get_xlim()
            self.assertEqual(result[0], xlim[0] - 5)
            self.assertEqual(result[1], xlim[1] + 10)

            # string
            expected = (Period('1/1/2000', ax.freq),
                        Period('4/1/2000', ax.freq))
            ax.set_xlim('1/1/2000', '4/1/2000')
            ax.get_figure().canvas.draw()
            result = ax.get_xlim()
            self.assertEqual(int(result[0]), expected[0].ordinal)
            self.assertEqual(int(result[1]), expected[1].ordinal)

            # datetim
            expected = (Period('1/1/2000', ax.freq),
                        Period('4/1/2000', ax.freq))
            ax.set_xlim(datetime(2000, 1, 1), datetime(2000, 4, 1))
            ax.get_figure().canvas.draw()
            result = ax.get_xlim()
            self.assertEqual(int(result[0]), expected[0].ordinal)
            self.assertEqual(int(result[1]), expected[1].ordinal)
            fig = ax.get_figure()
            plt.close(fig)

        ser = tm.makeTimeSeries()
        ax = ser.plot()
        _test(ax)

        df = DataFrame({'a': ser, 'b': ser + 1})
        ax = df.plot()
        _test(ax)

        df = DataFrame({'a': ser, 'b': ser + 1})
        axes = df.plot(subplots=True)

        for ax in axes:
            _test(ax)

    def test_get_finder(self):
        import pandas.tseries.converter as conv

        self.assertEqual(conv.get_finder('B'), conv._daily_finder)
        self.assertEqual(conv.get_finder('D'), conv._daily_finder)
        self.assertEqual(conv.get_finder('M'), conv._monthly_finder)
        self.assertEqual(conv.get_finder('Q'), conv._quarterly_finder)
        self.assertEqual(conv.get_finder('A'), conv._annual_finder)
        self.assertEqual(conv.get_finder('W'), conv._daily_finder)

    @slow
    def test_finder_daily(self):
        import matplotlib.pyplot as plt
        xp = Period('1999-1-1', freq='B').ordinal
        day_lst = [10, 40, 252, 400, 950, 2750, 10000]
        for n in day_lst:
            rng = bdate_range('1999-1-1', periods=n)
            ser = Series(np.random.randn(len(rng)), rng)
            ax = ser.plot()
            xaxis = ax.get_xaxis()
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(xp, rs)
            vmin, vmax = ax.get_xlim()
            ax.set_xlim(vmin + 0.9, vmax)
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(xp, rs)
            plt.close(ax.get_figure())

    @slow
    def test_finder_quarterly(self):
        import matplotlib.pyplot as plt
        xp = Period('1988Q1').ordinal
        yrs = [3.5, 11]
        for n in yrs:
            rng = period_range('1987Q2', periods=int(n * 4), freq='Q')
            ser = Series(np.random.randn(len(rng)), rng)
            ax = ser.plot()
            xaxis = ax.get_xaxis()
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(rs, xp)
            (vmin, vmax) = ax.get_xlim()
            ax.set_xlim(vmin + 0.9, vmax)
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(xp, rs)
            plt.close(ax.get_figure())

    @slow
    def test_finder_monthly(self):
        import matplotlib.pyplot as plt
        xp = Period('Jan 1988').ordinal
        yrs = [1.15, 2.5, 4, 11]
        for n in yrs:
            rng = period_range('1987Q2', periods=int(n * 12), freq='M')
            ser = Series(np.random.randn(len(rng)), rng)
            ax = ser.plot()
            xaxis = ax.get_xaxis()
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(rs, xp)
            vmin, vmax = ax.get_xlim()
            ax.set_xlim(vmin + 0.9, vmax)
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(xp, rs)
            plt.close(ax.get_figure())

    def test_finder_monthly_long(self):
        rng = period_range('1988Q1', periods=24 * 12, freq='M')
        ser = Series(np.random.randn(len(rng)), rng)
        ax = ser.plot()
        xaxis = ax.get_xaxis()
        rs = xaxis.get_majorticklocs()[0]
        xp = Period('1989Q1', 'M').ordinal
        self.assertEqual(rs, xp)

    @slow
    def test_finder_annual(self):
        import matplotlib.pyplot as plt
        xp = [1987, 1988, 1990, 1990, 1995, 2020, 2070, 2170]
        for i, nyears in enumerate([5, 10, 19, 49, 99, 199, 599, 1001]):
            rng = period_range('1987', periods=nyears, freq='A')
            ser = Series(np.random.randn(len(rng)), rng)
            ax = ser.plot()
            xaxis = ax.get_xaxis()
            rs = xaxis.get_majorticklocs()[0]
            self.assertEqual(rs, Period(xp[i], freq='A').ordinal)
            plt.close(ax.get_figure())

    @slow
    def test_finder_minutely(self):
        nminutes = 50 * 24 * 60
        rng = date_range('1/1/1999', freq='Min', periods=nminutes)
        ser = Series(np.random.randn(len(rng)), rng)
        ax = ser.plot()
        xaxis = ax.get_xaxis()
        rs = xaxis.get_majorticklocs()[0]
        xp = Period('1/1/1999', freq='Min').ordinal
        self.assertEqual(rs, xp)

    def test_finder_hourly(self):
        nhours = 23
        rng = date_range('1/1/1999', freq='H', periods=nhours)
        ser = Series(np.random.randn(len(rng)), rng)
        ax = ser.plot()
        xaxis = ax.get_xaxis()
        rs = xaxis.get_majorticklocs()[0]
        xp = Period('1/1/1999', freq='H').ordinal
        self.assertEqual(rs, xp)

    @slow
    def test_gaps(self):
        import matplotlib.pyplot as plt

        ts = tm.makeTimeSeries()
        ts[5:25] = np.nan
        ax = ts.plot()
        lines = ax.get_lines()
        tm._skip_if_mpl_1_5()
        self.assertEqual(len(lines), 1)
        l = lines[0]
        data = l.get_xydata()
        tm.assertIsInstance(data, np.ma.core.MaskedArray)
        mask = data.mask
        self.assertTrue(mask[5:25, 1].all())
        plt.close(ax.get_figure())

        # irregular
        ts = tm.makeTimeSeries()
        ts = ts[[0, 1, 2, 5, 7, 9, 12, 15, 20]]
        ts[2:5] = np.nan
        ax = ts.plot()
        lines = ax.get_lines()
        self.assertEqual(len(lines), 1)
        l = lines[0]
        data = l.get_xydata()
        tm.assertIsInstance(data, np.ma.core.MaskedArray)
        mask = data.mask
        self.assertTrue(mask[2:5, 1].all())
        plt.close(ax.get_figure())

        # non-ts
        idx = [0, 1, 2, 5, 7, 9, 12, 15, 20]
        ser = Series(np.random.randn(len(idx)), idx)
        ser[2:5] = np.nan
        ax = ser.plot()
        lines = ax.get_lines()
        self.assertEqual(len(lines), 1)
        l = lines[0]
        data = l.get_xydata()
        tm.assertIsInstance(data, np.ma.core.MaskedArray)
        mask = data.mask
        self.assertTrue(mask[2:5, 1].all())

    @slow
    def test_gap_upsample(self):
        low = tm.makeTimeSeries()
        low[5:25] = np.nan
        ax = low.plot()

        idxh = date_range(low.index[0], low.index[-1], freq='12h')
        s = Series(np.random.randn(len(idxh)), idxh)
        s.plot(secondary_y=True)
        lines = ax.get_lines()
        self.assertEqual(len(lines), 1)
        self.assertEqual(len(ax.right_ax.get_lines()), 1)
        l = lines[0]
        data = l.get_xydata()

        tm._skip_if_mpl_1_5()

        tm.assertIsInstance(data, np.ma.core.MaskedArray)
        mask = data.mask
        self.assertTrue(mask[5:25, 1].all())

    @slow
    def test_secondary_y(self):
        import matplotlib.pyplot as plt

        ser = Series(np.random.randn(10))
        ser2 = Series(np.random.randn(10))
        ax = ser.plot(secondary_y=True)
        self.assertTrue(hasattr(ax, 'left_ax'))
        self.assertFalse(hasattr(ax, 'right_ax'))
        fig = ax.get_figure()
        axes = fig.get_axes()
        l = ax.get_lines()[0]
        xp = Series(l.get_ydata(), l.get_xdata())
        assert_series_equal(ser, xp)
        self.assertEqual(ax.get_yaxis().get_ticks_position(), 'right')
        self.assertFalse(axes[0].get_yaxis().get_visible())
        plt.close(fig)

        ax2 = ser2.plot()
        self.assertEqual(ax2.get_yaxis().get_ticks_position(), 'default')
        plt.close(ax2.get_figure())

        ax = ser2.plot()
        ax2 = ser.plot(secondary_y=True)
        self.assertTrue(ax.get_yaxis().get_visible())
        self.assertFalse(hasattr(ax, 'left_ax'))
        self.assertTrue(hasattr(ax, 'right_ax'))
        self.assertTrue(hasattr(ax2, 'left_ax'))
        self.assertFalse(hasattr(ax2, 'right_ax'))

    @slow
    def test_secondary_y_ts(self):
        import matplotlib.pyplot as plt
        idx = date_range('1/1/2000', periods=10)
        ser = Series(np.random.randn(10), idx)
        ser2 = Series(np.random.randn(10), idx)
        ax = ser.plot(secondary_y=True)
        self.assertTrue(hasattr(ax, 'left_ax'))
        self.assertFalse(hasattr(ax, 'right_ax'))
        fig = ax.get_figure()
        axes = fig.get_axes()
        l = ax.get_lines()[0]
        xp = Series(l.get_ydata(), l.get_xdata()).to_timestamp()
        assert_series_equal(ser, xp)
        self.assertEqual(ax.get_yaxis().get_ticks_position(), 'right')
        self.assertFalse(axes[0].get_yaxis().get_visible())
        plt.close(fig)

        ax2 = ser2.plot()
        self.assertEqual(ax2.get_yaxis().get_ticks_position(), 'default')
        plt.close(ax2.get_figure())

        ax = ser2.plot()
        ax2 = ser.plot(secondary_y=True)
        self.assertTrue(ax.get_yaxis().get_visible())

    @slow
    def test_secondary_kde(self):
        tm._skip_if_no_scipy()
        _skip_if_no_scipy_gaussian_kde()

        import matplotlib.pyplot as plt  # noqa
        ser = Series(np.random.randn(10))
        ax = ser.plot(secondary_y=True, kind='density')
        self.assertTrue(hasattr(ax, 'left_ax'))
        self.assertFalse(hasattr(ax, 'right_ax'))
        fig = ax.get_figure()
        axes = fig.get_axes()
        self.assertEqual(axes[1].get_yaxis().get_ticks_position(), 'right')

    @slow
    def test_secondary_bar(self):
        ser = Series(np.random.randn(10))
        ax = ser.plot(secondary_y=True, kind='bar')
        fig = ax.get_figure()
        axes = fig.get_axes()
        self.assertEqual(axes[1].get_yaxis().get_ticks_position(), 'right')

    @slow
    def test_secondary_frame(self):
        df = DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
        axes = df.plot(secondary_y=['a', 'c'], subplots=True)
        self.assertEqual(axes[0].get_yaxis().get_ticks_position(), 'right')
        self.assertEqual(axes[1].get_yaxis().get_ticks_position(), 'default')
        self.assertEqual(axes[2].get_yaxis().get_ticks_position(), 'right')

    @slow
    def test_secondary_bar_frame(self):
        df = DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
        axes = df.plot(kind='bar', secondary_y=['a', 'c'], subplots=True)
        self.assertEqual(axes[0].get_yaxis().get_ticks_position(), 'right')
        self.assertEqual(axes[1].get_yaxis().get_ticks_position(), 'default')
        self.assertEqual(axes[2].get_yaxis().get_ticks_position(), 'right')

    def test_mixed_freq_regular_first(self):
        import matplotlib.pyplot as plt  # noqa
        s1 = tm.makeTimeSeries()
        s2 = s1[[0, 5, 10, 11, 12, 13, 14, 15]]

        # it works!
        s1.plot()

        ax2 = s2.plot(style='g')
        lines = ax2.get_lines()
        idx1 = PeriodIndex(lines[0].get_xdata())
        idx2 = PeriodIndex(lines[1].get_xdata())
        self.assertTrue(idx1.equals(s1.index.to_period('B')))
        self.assertTrue(idx2.equals(s2.index.to_period('B')))
        left, right = ax2.get_xlim()
        pidx = s1.index.to_period()
        self.assertEqual(left, pidx[0].ordinal)
        self.assertEqual(right, pidx[-1].ordinal)

    @slow
    def test_mixed_freq_irregular_first(self):
        import matplotlib.pyplot as plt  # noqa
        s1 = tm.makeTimeSeries()
        s2 = s1[[0, 5, 10, 11, 12, 13, 14, 15]]
        s2.plot(style='g')
        ax = s1.plot()
        self.assertFalse(hasattr(ax, 'freq'))
        lines = ax.get_lines()
        x1 = lines[0].get_xdata()
        tm.assert_numpy_array_equal(x1, s2.index.asobject.values)
        x2 = lines[1].get_xdata()
        tm.assert_numpy_array_equal(x2, s1.index.asobject.values)

    def test_mixed_freq_regular_first_df(self):
        # GH 9852
        import matplotlib.pyplot as plt  # noqa
        s1 = tm.makeTimeSeries().to_frame()
        s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15], :]
        ax = s1.plot()
        ax2 = s2.plot(style='g', ax=ax)
        lines = ax2.get_lines()
        idx1 = PeriodIndex(lines[0].get_xdata())
        idx2 = PeriodIndex(lines[1].get_xdata())
        self.assertTrue(idx1.equals(s1.index.to_period('B')))
        self.assertTrue(idx2.equals(s2.index.to_period('B')))
        left, right = ax2.get_xlim()
        pidx = s1.index.to_period()
        self.assertEqual(left, pidx[0].ordinal)
        self.assertEqual(right, pidx[-1].ordinal)

    @slow
    def test_mixed_freq_irregular_first_df(self):
        # GH 9852
        import matplotlib.pyplot as plt  # noqa
        s1 = tm.makeTimeSeries().to_frame()
        s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15], :]
        ax = s2.plot(style='g')
        ax = s1.plot(ax=ax)
        self.assertFalse(hasattr(ax, 'freq'))
        lines = ax.get_lines()
        x1 = lines[0].get_xdata()
        tm.assert_numpy_array_equal(x1, s2.index.asobject.values)
        x2 = lines[1].get_xdata()
        tm.assert_numpy_array_equal(x2, s1.index.asobject.values)

    def test_mixed_freq_hf_first(self):
        idxh = date_range('1/1/1999', periods=365, freq='D')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        high.plot()
        ax = low.plot()
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, 'D')

    @slow
    def test_mixed_freq_alignment(self):
        ts_ind = date_range('2012-01-01 13:00', '2012-01-02', freq='H')
        ts_data = np.random.randn(12)

        ts = Series(ts_data, index=ts_ind)
        ts2 = ts.asfreq('T').interpolate()

        ax = ts.plot()
        ts2.plot(style='r')

        self.assertEqual(ax.lines[0].get_xdata()[0],
                         ax.lines[1].get_xdata()[0])

    @slow
    def test_mixed_freq_lf_first(self):
        import matplotlib.pyplot as plt

        idxh = date_range('1/1/1999', periods=365, freq='D')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        low.plot(legend=True)
        ax = high.plot(legend=True)
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, 'D')
        leg = ax.get_legend()
        self.assertEqual(len(leg.texts), 2)
        plt.close(ax.get_figure())

        idxh = date_range('1/1/1999', periods=240, freq='T')
        idxl = date_range('1/1/1999', periods=4, freq='H')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        low.plot()
        ax = high.plot()
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, 'T')

    def test_mixed_freq_irreg_period(self):
        ts = tm.makeTimeSeries()
        irreg = ts[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 29]]
        rng = period_range('1/3/2000', periods=30, freq='B')
        ps = Series(np.random.randn(len(rng)), rng)
        irreg.plot()
        ps.plot()

    @slow
    def test_to_weekly_resampling(self):
        idxh = date_range('1/1/1999', periods=52, freq='W')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        high.plot()
        ax = low.plot()
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, idxh.freq)

        # tsplot
        from pandas.tseries.plotting import tsplot
        import matplotlib.pyplot as plt

        tsplot(high, plt.Axes.plot)
        lines = tsplot(low, plt.Axes.plot)
        for l in lines:
            self.assertTrue(PeriodIndex(data=l.get_xdata()).freq, idxh.freq)

    @slow
    def test_from_weekly_resampling(self):
        idxh = date_range('1/1/1999', periods=52, freq='W')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        low.plot()
        ax = high.plot()

        expected_h = idxh.to_period().asi8
        expected_l = np.array([1514, 1519, 1523, 1527, 1531, 1536, 1540, 1544,
                               1549, 1553, 1558, 1562])
        for l in ax.get_lines():
            self.assertTrue(PeriodIndex(data=l.get_xdata()).freq, idxh.freq)
            xdata = l.get_xdata(orig=False)
            if len(xdata) == 12:  # idxl lines
                self.assert_numpy_array_equal(xdata, expected_l)
            else:
                self.assert_numpy_array_equal(xdata, expected_h)
        tm.close()

        # tsplot
        from pandas.tseries.plotting import tsplot
        import matplotlib.pyplot as plt

        tsplot(low, plt.Axes.plot)
        lines = tsplot(high, plt.Axes.plot)
        for l in lines:
            self.assertTrue(PeriodIndex(data=l.get_xdata()).freq, idxh.freq)
            xdata = l.get_xdata(orig=False)
            if len(xdata) == 12:  # idxl lines
                self.assert_numpy_array_equal(xdata, expected_l)
            else:
                self.assert_numpy_array_equal(xdata, expected_h)

    @slow
    def test_from_resampling_area_line_mixed(self):
        idxh = date_range('1/1/1999', periods=52, freq='W')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = DataFrame(np.random.rand(len(idxh), 3),
                         index=idxh, columns=[0, 1, 2])
        low = DataFrame(np.random.rand(len(idxl), 3),
                        index=idxl, columns=[0, 1, 2])

        # low to high
        for kind1, kind2 in [('line', 'area'), ('area', 'line')]:
            ax = low.plot(kind=kind1, stacked=True)
            ax = high.plot(kind=kind2, stacked=True, ax=ax)

            # check low dataframe result
            expected_x = np.array([1514, 1519, 1523, 1527, 1531, 1536, 1540,
                                   1544, 1549, 1553, 1558, 1562])
            expected_y = np.zeros(len(expected_x))
            for i in range(3):
                l = ax.lines[i]
                self.assertEqual(PeriodIndex(l.get_xdata()).freq, idxh.freq)
                self.assert_numpy_array_equal(
                    l.get_xdata(orig=False), expected_x)
                # check stacked values are correct
                expected_y += low[i].values
                self.assert_numpy_array_equal(
                    l.get_ydata(orig=False), expected_y)

            # check high dataframe result
            expected_x = idxh.to_period().asi8
            expected_y = np.zeros(len(expected_x))
            for i in range(3):
                l = ax.lines[3 + i]
                self.assertEqual(PeriodIndex(
                    data=l.get_xdata()).freq, idxh.freq)
                self.assert_numpy_array_equal(
                    l.get_xdata(orig=False), expected_x)
                expected_y += high[i].values
                self.assert_numpy_array_equal(
                    l.get_ydata(orig=False), expected_y)

        # high to low
        for kind1, kind2 in [('line', 'area'), ('area', 'line')]:
            ax = high.plot(kind=kind1, stacked=True)
            ax = low.plot(kind=kind2, stacked=True, ax=ax)

            # check high dataframe result
            expected_x = idxh.to_period().asi8
            expected_y = np.zeros(len(expected_x))
            for i in range(3):
                l = ax.lines[i]
                self.assertEqual(PeriodIndex(
                    data=l.get_xdata()).freq, idxh.freq)
                self.assert_numpy_array_equal(
                    l.get_xdata(orig=False), expected_x)
                expected_y += high[i].values
                self.assert_numpy_array_equal(
                    l.get_ydata(orig=False), expected_y)

            # check low dataframe result
            expected_x = np.array([1514, 1519, 1523, 1527, 1531, 1536, 1540,
                                   1544, 1549, 1553, 1558, 1562])
            expected_y = np.zeros(len(expected_x))
            for i in range(3):
                l = ax.lines[3 + i]
                self.assertEqual(PeriodIndex(
                    data=l.get_xdata()).freq, idxh.freq)
                self.assert_numpy_array_equal(
                    l.get_xdata(orig=False), expected_x)
                expected_y += low[i].values
                self.assert_numpy_array_equal(
                    l.get_ydata(orig=False), expected_y)

    @slow
    def test_mixed_freq_second_millisecond(self):
        # GH 7772, GH 7760
        idxh = date_range('2014-07-01 09:00', freq='S', periods=50)
        idxl = date_range('2014-07-01 09:00', freq='100L', periods=500)
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        # high to low
        high.plot()
        ax = low.plot()
        self.assertEqual(len(ax.get_lines()), 2)
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, 'L')
        tm.close()

        # low to high
        low.plot()
        ax = high.plot()
        self.assertEqual(len(ax.get_lines()), 2)
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(data=l.get_xdata()).freq, 'L')

    @slow
    def test_irreg_dtypes(self):
        # date
        idx = [date(2000, 1, 1), date(2000, 1, 5), date(2000, 1, 20)]
        df = DataFrame(np.random.randn(len(idx), 3), Index(idx, dtype=object))
        _check_plot_works(df.plot)

        # np.datetime64
        idx = date_range('1/1/2000', periods=10)
        idx = idx[[0, 2, 5, 9]].asobject
        df = DataFrame(np.random.randn(len(idx), 3), idx)
        _check_plot_works(df.plot)

    @slow
    def test_time(self):
        t = datetime(1, 1, 1, 3, 30, 0)
        deltas = np.random.randint(1, 20, 3).cumsum()
        ts = np.array([(t + timedelta(minutes=int(x))).time() for x in deltas])
        df = DataFrame({'a': np.random.randn(len(ts)),
                        'b': np.random.randn(len(ts))},
                       index=ts)
        ax = df.plot()

        # verify tick labels
        ticks = ax.get_xticks()
        labels = ax.get_xticklabels()
        for t, l in zip(ticks, labels):
            m, s = divmod(int(t), 60)
            h, m = divmod(m, 60)
            xp = l.get_text()
            if len(xp) > 0:
                rs = time(h, m, s).strftime('%H:%M:%S')
                self.assertEqual(xp, rs)

        # change xlim
        ax.set_xlim('1:30', '5:00')

        # check tick labels again
        ticks = ax.get_xticks()
        labels = ax.get_xticklabels()
        for t, l in zip(ticks, labels):
            m, s = divmod(int(t), 60)
            h, m = divmod(m, 60)
            xp = l.get_text()
            if len(xp) > 0:
                rs = time(h, m, s).strftime('%H:%M:%S')
                self.assertEqual(xp, rs)

    @slow
    def test_time_musec(self):
        t = datetime(1, 1, 1, 3, 30, 0)
        deltas = np.random.randint(1, 20, 3).cumsum()
        ts = np.array([(t + timedelta(microseconds=int(x))).time()
                       for x in deltas])
        df = DataFrame({'a': np.random.randn(len(ts)),
                        'b': np.random.randn(len(ts))},
                       index=ts)
        ax = df.plot()

        # verify tick labels
        ticks = ax.get_xticks()
        labels = ax.get_xticklabels()
        for t, l in zip(ticks, labels):
            m, s = divmod(int(t), 60)

            # TODO: unused?
            # us = int((t - int(t)) * 1e6)

            h, m = divmod(m, 60)
            xp = l.get_text()
            if len(xp) > 0:
                rs = time(h, m, s).strftime('%H:%M:%S.%f')
                self.assertEqual(xp, rs)

    @slow
    def test_secondary_upsample(self):
        idxh = date_range('1/1/1999', periods=365, freq='D')
        idxl = date_range('1/1/1999', periods=12, freq='M')
        high = Series(np.random.randn(len(idxh)), idxh)
        low = Series(np.random.randn(len(idxl)), idxl)
        low.plot()
        ax = high.plot(secondary_y=True)
        for l in ax.get_lines():
            self.assertEqual(PeriodIndex(l.get_xdata()).freq, 'D')
        self.assertTrue(hasattr(ax, 'left_ax'))
        self.assertFalse(hasattr(ax, 'right_ax'))
        for l in ax.left_ax.get_lines():
            self.assertEqual(PeriodIndex(l.get_xdata()).freq, 'D')

    @slow
    def test_secondary_legend(self):
        import matplotlib.pyplot as plt
        fig = plt.gcf()
        plt.clf()


from flask import Flask, request
from flask_restful import Resource, Api
from flask_cors import CORS
import flask_login
import os
import logging
import model
import key_helper

app = Flask(__name__)
CORS(app)

login_manager = flask_login.LoginManager()
login_manager.init_app(app)

api = Api(app)

db_host = os.getenv('DB_HOST', 'localhost')
db_port = os.getenv('DB_PORT', '5432')
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://postgres:password@' + db_host + ':' + db_port + '/openbeta'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

model.db.init_app(app)

app.config['db'] = model.db
app.config['model'] = model

if not os.getenv('NO_LOGGING'):
    logging.basicConfig()
    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

class FeatureSetEndpoint(Resource):
    """FeatureSet are collection of routes, boundaries, cliffs
       While not as human-friendly as /route and /boundary this endpoint can be used by apps
       to get all matching features in one single api call
    """
    @flask_login.login_required
    def get(self):
        if 'latlng' in request.args and 'r' in request.args:
            latlng = request.args['latlng']
            r = request.args['r']
            featureset = model.search_within_radius_in_meters(location=latlng, radius=r, route=True, boundary=True)
            return featureset_output_formatter(featureset)
        else:
            return featureset_output_formatter()

    @flask_login.login_required
    def post(self):
        json_data = request.get_json(force=True)
        if json_data['type'].upper() == 'FEATURECOLLECTION':
            features = json_data['features']
            for item in features:
                if item['geometry']['type'] == 'Polygon':
                    boundary = model.Boundary(item)
                    model.db.session.add(boundary)
                else:
                    route = model.Route(item)
                    model.db.session.add(route)
                model.db.session.commit()

api.add_resource(FeatureSetEndpoint, '/featureset')

class RecentActivitiesEndpoint(Resource):
    """Return a featureset of most recent activity
    """
    @flask_login.login_required
    def get(self):
        if 'count' in request.args:
            count = request.args['count']
            featureset = model.recent_activities(count=count, route=True, boundary=True)
            return featureset_output_formatter(featureset)
        else:
            return featureset_output_formatter()

api.add_resource(RecentActivitiesEndpoint, '/recent')

class Route(Resource):

    @flask_login.login_required
    def get(self):
        if 'latlng' in request.args and 'r' in request.args:
            latlng = request.args['latlng']
            r = request.args['r']
            featureset = model.search_within_radius_in_meters(location=latlng, radius=r)
            return featureset.route
        elif 'boundary_id' in request.args:
            features = model.search_within_boundary_by_id(request.args['boundary_id'])
            return features;
        else:
            return {
                "type": "FeatureCollection",
                "features": {}
            }

    @flask_login.login_required
    def post(self):
        json_data = request.get_json(force=True)

        if json_data['type'] == 'FeatureCollection':
            features = json_data['features']
            for item in features:
                route = model.Route(item)
                model.db.session.add(route)
                model.db.session.commit()

        elif json_data['type'] == 'Feature':
            route = model.Route(json_data['feature'])
            model.db.session.add(route)
            model.db.session.commit()
        return json_data

api.add_resource(Route, '/routes')

class Boundary(Resource):

    @flask_login.login_required
    def get(self):
        if 'id' in request.args:
            boundary_id = parse_id("area", request.args['id'])
            if boundary_id is not None:
                try:
                    result = model.get_boundary_by_id(boundary_id)
                    if result is None:
                        return "Empty", 204
                    return result
                except (TypeError, Exception) as e:
                    print e
                    return "Internal exception", 400
        return "Invalid id", 400

    @flask_login.login_required
    def post(self):
        data = request.form
        if len(data) > 0:
            for key in data:
                print "key: %s , value: %s" % (key, data[key])

        json_data = request.get_json(force=True)
        if json_data['type'].upper() == 'FEATURECOLLECTION':
            features = json_data['features']
            for item in features:
                boundary = model.Boundary(item)
                model.db.session.add(boundary)
                model.db.session.commit()
            return 'OK'
        elif json_data['type'].upper() == 'FEATURE':
            boundary = model.Boundary(json_data)
            model.db.session.add(boundary)
            model.db.session.commit()
            return boundary.to_json(), 200

    @flask_login.login_required
    def put(self):
        feature = request.get_json(force=True)
        if feature['id']:
            boundary_id = parse_id("area", feature['id'])
            if boundary_id is not None:
                model.db.session.query(model.Boundary).filter(model.Boundary.boundary_id == boundary_id).first();
                new_boundary = model.Boundary(feature)
                new_boundary.boundary_id = boundary_id
                model.db.session.merge(new_boundary);
                model.db.session.commit()
                return new_boundary.to_json(), 200
        return "Error", 400

api.add_resource(Boundary, '/boundaries')

@login_manager.request_loader
def load_user_from_request(request):

    api_key = request.args.get('api_key')
    try:
        key_helper.userKeySigner.unsign(api_key)
    except Exception:
        return None
    if api_key:
        user = model.APIUser.query.filter_by(api_key=api_key, active=True).first()
        if user:
            return user
    return None

def init_db_day0():
    model.db.drop_all()
    model.db.create_all()
    model.setup_temporal_tables()

def featureset_output_formatter(data=None, output='geojson'):
    if output == 'geojson':
        if data is None:
            data = model.FeatureSet(route={}, boundary={})

        return {
            "kind": "FeatureSet",
            "route": data.route,
            "boundary": data.boundary
        }
    else: # TODO support other format such as yaml, simplified json (geojson-like but more compact)
        return {
        }

def parse_id(type, geojson_id):


#
# Copyright 2008-2014 Zuza Software Foundation
#
# This file is part of the Translate Toolkit.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

from contextlib import contextmanager

from lxml import etree

from translate.storage.placeables import StringElem, xliff
from translate.storage.xml_extract import misc, xpath_breadcrumb

class Translatable:
    """A node corresponds to a translatable element. A node may have children,
    which correspond to placeables.
    """

    def __init__(self, placeable_name, xpath, dom_node, source, is_inline=False):
        self.placeable_name = placeable_name
        self.source = source
        self.xpath = xpath
        self.is_inline = is_inline
        self.dom_node = dom_node

    @property
    def placeables(self):
        return [
            placeable
            for placeable in self.source
            if isinstance(placeable, Translatable)
        ]

    @property
    def has_translatable_text(self):
        """Check if it contains any chunk of text with more than whitespace.

        If not, then there's nothing to translate.
        """
        for chunk in self.source:
            if isinstance(chunk, str) and chunk.strip() != "":
                return True
        return False

def reduce_unit_tree(f, unit_node, *state):
    return misc.reduce_tree(
        f, unit_node, unit_node, lambda unit_node: unit_node.placeables, *state
    )

class ParseState:
    """Maintain constants and variables used during the walking of a DOM tree
    (via the function apply).
    """

    def __init__(self, no_translate_content_elements, inline_elements={}, nsmap={}):
        self.no_translate_content_elements = no_translate_content_elements
        self.inline_elements = inline_elements
        self.is_inline = False
        self.xpath_breadcrumb = xpath_breadcrumb.XPathBreadcrumb()
        self.placeable_name = "<top-level>"
        self.nsmap = nsmap

def _process_placeable(dom_node, state):
    """Process current placeable.

    This returns all nested translatable content for this placeable as a single
    Translatable object, or just returns an empty Translatable object for this
    placeable if there is no nested translatable content.
    """
    placeable = find_translatable_dom_nodes(dom_node, state, process_translatable)

    if len(placeable) == 0:
        # There are no recognized child tags and thus no Translatable object is
        # returned. So create a Translatable with the name "placeable".
        return Translatable("placeable", state.xpath_breadcrumb.xpath, dom_node, [])
    elif len(placeable) == 1:
        # The ideal situation: we got exactly one Translatable back when
        # processing this tree.
        return placeable[0]
    else:
        raise Exception(
            "BUG: find_translatable_dom_nodes should never return "
            "more than a single Translatable object"
        )

def process_translatable(dom_node, state):
    """Process a translatable DOM node.

    Any translatable content present in a child node is treated as a placeable.
    """
    source = [str(dom_node.text or "")]

    # Append Translatable objects and unicode strings for the translatable
    # content for all the children.
    for child in dom_node:
        source.append(_process_placeable(child, state))
        source.append(str(child.tail or ""))

    translatable = Translatable(
        state.placeable_name,
        state.xpath_breadcrumb.xpath,
        dom_node,
        source,
        state.is_inline,
    )
    return [translatable]

def _has_idml_translatable_content(dom_node):
    has_translatable_content = True
    if dom_node.tag == "ParagraphStyleRange":
        has_translatable_content = False

        for content_node in dom_node.findall(".//Content"):
            # Iterate over all the Content tags in this ParagraphStyleRange tag.
            if content_node.text is not None and content_node.text.strip():
                return True

            for child in content_node.iterdescendants():
                # The Content node can just have a child before any nested
                # text, and therefore its text is None, so we have to check if
                # it has children, and if any of its children has text.
                if (
                    not isinstance(child, etree._ProcessingInstruction)
                    and child.text is not None
                    and child.text.strip()
                ):
                    return True

                if child.tail is not None and child.tail.strip():
                    return True

    return has_translatable_content

def _retrieve_idml_placeables(dom_node, state):
    source = []
    for child in dom_node:
        if not isinstance(child, etree._Element):
            continue

        if isinstance(child, etree._ProcessingInstruction):
            # TODO this probably won't be using the right xpath.
            source.append(
                Translatable(
                    "placeable", state.xpath_breadcrumb.xpath, child, [], False
                )
            )

            if child.tail is not None and child.tail.strip():
                source.append(str(child.tail))

            continue

        namespace, tag = misc.parse_tag(child.tag)

        with parse_status_set(namespace, tag, state):
            # Ensure we extract all the tags below ParagraphStyleRange as
            # placeables, independently of them being translatable or not.
            # state.is_inline = True

            nested_stuff = []

            if child.text is not None and child.text.strip():
                nested_stuff = [str(child.text)]

            nested_stuff.extend(_retrieve_idml_placeables(child, state))

            source.append(
                Translatable(
                    "placeable",
                    state.xpath_breadcrumb.xpath,
                    child,
                    nested_stuff,
                    state.is_inline,
                )
            )

            if child.tail is not None and child.tail.strip():
                source.append(str(child.tail))

    return source

def process_idml_translatable(dom_node, state):
    if _has_idml_translatable_content(dom_node):
        source = _retrieve_idml_placeables(dom_node, state)

        translatable = Translatable(
            state.placeable_name,
            state.xpath_breadcrumb.xpath,
            dom_node,
            source,
            state.is_inline,
        )
        return [translatable]

    return []

def _process_children(dom_node, state, process_func):
    """Process an untranslatable DOM node.

    Since the node is untranslatable it just returns any translatable content
    present in its child nodes.
    """
    children = [
        find_translatable_dom_nodes(child, state, process_func) for child in dom_node
    ]

    # Flatten a list of lists into a list of elements
    children = [child for child_list in children for child in child_list]

    if len(children) > 1:
        _namespace, tag = misc.parse_tag(dom_node.tag)
        intermediate_translatable = Translatable(
            tag, state.xpath_breadcrumb.xpath, dom_node, children
        )
        return [intermediate_translatable]
    else:
        return children

def compact_tag(nsmap, namespace, tag):
    if namespace in nsmap:
        return f"{nsmap[namespace]}:{tag}"
    else:
        return f"{{{namespace}}}{tag}"

@contextmanager
def parse_status_set(namespace, tag, state):
    # Set XPath breadcrumb item for the current node.
    xpath_item = compact_tag(state.nsmap, namespace, tag)
    state.xpath_breadcrumb.start_tag(xpath_item)

    # Set the placeable name for the current node.
    old_placeable_name = state.placeable_name
    state.placeable_name = tag

    # Set the inline status for the current node.
    old_inline = state.is_inline
    state.is_inline = (namespace, tag) in state.inline_elements

    yield state

    # Reset inline status, placeable name and XPath breadcrumb to the
    # previous values.
    state.is_inline = old_inline
    state.placeable_name = old_placeable_name
    state.xpath_breadcrumb.end_tag()

def find_translatable_dom_nodes(dom_node, state, process_func=process_translatable):
    # For now, we only want to deal with XML elements.
    # And we want to avoid processing instructions, which
    # are XML elements (in the inheritance hierarchy).
    if not isinstance(dom_node, etree._Element) or isinstance(
        dom_node, etree._ProcessingInstruction
    ):
        return []

    namespace, tag = misc.parse_tag(dom_node.tag)

    with parse_status_set(namespace, tag, state):
        if (namespace, tag) not in state.no_translate_content_elements:
            return process_func(dom_node, state)
        else:
            return _process_children(dom_node, state, process_func)

class IdMaker:
    def __init__(self):
        self._max_id = 0
        self._obj_id_map = {}

    def get_id(self, obj):
        if not self.has_id(obj):
            self._obj_id_map[obj] = self._max_id
            self._max_id += 1
        return self._obj_id_map[obj]

    def has_id(self, obj):
        return obj in self._obj_id_map

def _to_placeables(parent_translatable, translatable, id_maker):
    """Convert the translatable object to a list of strings and XLIFF
    placeables.
    """
    result = []
    for chunk in translatable.source:
        if isinstance(chunk, str):
            result.append(chunk)
        else:


from . import MemoryPolicy
import numpy as np
import copy
from collections import deque
from ...ngmeth_utils.srtheo_utils import srtheo_voc
from ...ngstrat import get_strategy
from ...ngmeth import negentropy_final

class Inventions(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'inventions' in mem.keys()
		mem['inventions'] = {'known_meanings':{},'last_known_meanings':[],'counts':{},'nb_interactions':0,'nb_inventions':0,'invented_meanings':{}}

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context):
		time_stamp = mem['inventions']['nb_interactions'] + 1
		mem['inventions']['nb_interactions'] = time_stamp
		if ms not in mem['inventions']['last_known_meanings']:
			if role == 'speaker':
				mem['inventions']['nb_inventions'] += 1
				if ms not in mem['inventions']['invented_meanings'].keys():
					mem['inventions']['invented_meanings'][ms] = [(time_stamp,w)]
				else:
					mem['inventions']['invented_meanings'][ms].append((time_stamp,w))
		if ms not in mem['inventions']['known_meanings'].keys():
			mem['inventions']['known_meanings'][ms] = (time_stamp,w)
		mem['inventions']['last_known_meanings'] = copy.copy(voc.get_known_meanings())
		if ms not in mem['inventions']['counts'].keys():
			mem['inventions']['counts'][ms] = 1
		else:
			mem['inventions']['counts'][ms] += 1

class SuccessMatrixMP(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'success_matrix' in list(mem.keys())
		mem['success_matrix'] = np.zeros((self.nb_boxes,self.nb_boxes,2))

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context):
		x,y = self.get_coords(context)
		if bool_succ:
			mem['success_matrix'][x,y,0] += 1
		else:
			mem['success_matrix'][x,y,1] += 1

class PastInterMP(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'past_interactions' in list(mem.keys())
		mem['past_interactions'] = []

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context):
		past_int = mem['past_interactions']
		d = abs(context[0]-context[1])
		c = (context[0]+context[1])/2.
		mem['past_interactions'] = past_int[-self.past_window:]+[(d, c, bool_succ)]

class PastInterAll(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'past_interactions_all' in list(mem.keys())
		mem['past_interactions_all'] = []

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context):
		mem['past_interactions_all'].append({'role':role,'ms':ms,'mh':mh,'w':w,'bool_succ':bool_succ,'context':context})

class OldVoc(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'old_voc' in list(mem.keys())
		mem['old_voc'] = copy.deepcopy(voc)

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context):
		mem['old_voc'] = copy.deepcopy(voc)

class OtherVoc(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'other_voc' in list(mem.keys())
		mem['other_voc'] = copy.deepcopy(voc)

class LastResultMP(MemoryPolicy):

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if bool_succ:
			mem["result"]=1
		else:
			mem["result"]=0

	def init_memory(self,mem,voc,cfg=None):

		assert not 'result' in list(mem.keys())
		mem["result"]=1

class ProbaSuccessIncrease(MemoryPolicy):#!! only a cache

	#def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
	#	m_rm_list = list(  set([ms]) | set(voc.get_known_meanings(w=w)) ) not removing here because voc is already updated, hence lists of ms and ws to be removed are not always known

	def init_memory(self,mem,voc,cfg=None):
		assert not 'proba_of_success_increase' in list(mem.keys())
		mem['proba_of_success_increase'] = dict()

class SuccessCountPerMMP(MemoryPolicy):

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if role=='speaker':
			m1=ms
		else:
			m1=ms#mh justification: getting information from the other's vocabulary for the hearer
		if bool_succ:
			try:
				mem["success_m"][m1] += 1
			except KeyError:
				mem["success_m"][m1] = 1
		else:
			try:
				mem["fail_m"][m1] += 1
			except KeyError:
				mem["fail_m"][m1] = 1

	def init_memory(self,mem,voc,cfg=None):
		assert not 'success_m' in list(mem.keys())
		assert not 'fail_m' in list(mem.keys())
		if voc is not None and hasattr(voc,'_content'):
			mem["success_m"] = np.zeros(voc.get_M())#[0]*voc._M
			mem["fail_m"] = np.zeros(voc.get_M())#[0]*voc._M
		else:
			mem["success_m"] = dict()
			mem["fail_m"] = dict()

class StratMP(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'strat' in list(mem.keys())
		mem['strat'] = get_strategy(**cfg['strat_cfg'])

class TimeDecreaseSuccessCountPerMMP(SuccessCountPerMMP):

	def __init__(self,mem_type,time_scale=100):
		MemoryPolicy.__init__(self,mem_type=mem_type)
		self.time_scale = time_scale
		if self.time_scale == 0:
			self.factor = 0.
		else:
			self.factor = np.exp(-1./self.time_scale)

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if role=='speaker':
			m1=ms
		else:
			m1=ms#mh justification: getting information from the other's vocabulary for the hearer
		if m1 in list(mem["success_m"].keys()):
			for a in [mem["success_m"][m1],mem["fail_m"][m1]]:
				a *= self.factor

		if bool_succ:
			try:
				mem["success_m"][m1] += 1
			except KeyError:
				mem["success_m"][m1] = 1
		else:
			try:
				mem["fail_m"][m1] += 1
			except KeyError:
				mem["fail_m"][m1] = 1

class SuccessCountPerMWMP(MemoryPolicy):
	def init_memory(self,mem,voc,cfg=None):

		assert not 'success_mw' in list(mem.keys())
		assert not 'fail_mw' in list(mem.keys())
		mem["success_mw"] = np.zeros((voc._M, voc._W))#not adapted to growing m and w spaces, and init not 'voc is None'-proof
		mem["fail_mw"] = np.zeros((voc._M, voc._W))

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if role=='speaker':
			m1 = ms
		else:
			m1 =ms#mh justification: getting information from the other's vocabulary for the hearer
		if bool_succ:
			mem["success_mw"][m1, w]+=1.
		else:
			mem["fail_mw"][m1, w]+=1.

class SuccessCountMP(MemoryPolicy):

	def init_memory(self,mem,voc,cfg=None):
		assert not 'success' in mem.keys()
		assert not 'fail' in mem.keys()
		mem['success'] = 0
		mem['fail'] = 0

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if bool_succ:
			mem['success'] += 1
		else:
			mem['fail'] += 1

class TimeWeightedSuccessCountPerMWMP(SuccessCountPerMWMP):

	def __init__(self,mem_type,time_scale=100):
		MemoryPolicy.__init__(self,mem_type=mem_type)
		self.time_scale = time_scale
		self.increment = 1.
		if time_scale == 0.:
			self.factor = 0.
		else:
			self.factor = np.exp(1./self.time_scale)

	def update_memory(self,ms,w,mh,voc,mem,role,bool_succ,context=[]):
		if role == 'speaker':
			m1 = ms
		else:
			m1 = ms#mh justification: getting information from the other's vocabulary for the hearer
		if bool_succ:
			mem["success_mw"][m1, w] += self.increment
		else:
			mem["fail_mw"][m1, w] += self.increment
		self.increment *= self.factor

class TimeDecreaseSuccessCountPerMWMP(TimeWeightedSuccessCountPerMWMP):

	def __init__(self,mem_type,time_scale=100,epsilon=0.01):
		MemoryPolicy.__init__(self,mem_type=mem_type)
		self.time_scale = time_scale


# -*- coding: utf-8 -*-
# Part of the psychopy.iohub library.
# Copyright (C) 2012-2016 iSolver Software Solutions
# Distributed under the terms of the GNU General Public License (GPL).
from __future__ import division, absolute_import
"""
Python Interface for ioSync. ioSync uses a Teensy 3 / 3.1 microcontroller
to provide a low cost solution for digital and analog inputs, as well as digital
outputs.

Be sure to use Teensiduno to compile and upload the ioSync sketch to the Teensy 3
which will be used. Otherwise the pySync interface will not function.

ioSync uses a majority of the Teensy 3 pins, including the ones on the bottom
of the Teensy 3 accessible via the solder pads.

ioSync Teensy 3 Pin Assignment
===============================

The ioSync uses the T3 pins as follows:

Teensy 3 board LED:
~~~~~~~~~~~~~~~~~~~

    LED: 13   // Not available as LED if Using SPI.

Non USB UART (Serial) RT and TX:
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    UART_RX: RX1
    UART_TX: TX1

Digital Outputs:
~~~~~~~~~~~~~~~~

    DO_0 2
    DO_1 3
    DO_2 4
    DO_3 5
    DO_4 25 // on bottom of T3
    DO_5 26 // on bottom of T3
    DO_6 27 // on bottom of T3
    DO_7 28   // on bottom of T3

Digital Inputs:
~~~~~~~~~~~~~~~

    DI_0: 6
    DI_1: 7
    DI_2: 8
    DI_3: 9
    DI_4: 29 // on bottom of T3
    DI_5: 30 // on bottom of T3
    DI_6: 31 // on bottom of T3
    DI_7: 32 // on bottom of T3

DI_0 to DI_7 can be read as an 8bit byte. There is a 9th digial input pin that
can be used as an independent input bit:

    DIN_8: 33

Analog Inputs:
~~~~~~~~~~~~~~

    AI_0: 14
    AI_1: 15
    AI_2: 16
    AI_3: 17
    AI_4: A10 // on bottom of T3
    AI_5: A11 // on bottom of T3
    AI_6: A12 // on bottom of T3
    AI_7: A13 // on bottom of T3

Current settings for the analog input channels are:
    * 16 bit
    * 16 sample averaging
    * 1000 Hz Sampling Rate
    * An external reference must be provided.

SPI Pins:
~~~~~~~~~~

    SPI_SS:     CS0     // digital pin 10 on T3, Device Select
    SPI_MOSI:   DOUT    // digital pin 11 on T3, SPI Data Output
    SPI_MISO:   DIN     // digital pin 12 on T3, SPI Data Input
    SPI_SCK:    SCK     // digital pin 13 on 3, Clock

I2C Pins:
~~~~~~~~~~

    I2C_SCL: SCL      // digital Pin 19 on T3
    I2C_SDA: SDA      // digital Pin 18 on T3

UART:
~~~~~

    RX: RX1         // digital pin 0 on T3
    TX: TX1         // digital pin 1 in T3

PWM Outputs:
~~~~~~~~~~~~

    PMW_0: 20   // digital pin 20 on T3
    PMW_1: 21   // digital pin 21 on T3
    PMW_2: 22   // digital pin 22 on T3
    PMW_3: 23   // digital pin 23 on T3

"""
from collections import OrderedDict
import struct

import serial
import numpy as np

from ... import Computer
from ....errors import print2err
from ....util import NumPyRingBuffer as RingBuffer

from .t3keymap import char2t3code, mod2t3code

getTime = Computer.getTime

_t3key_format = struct.Struct('!HHB')
_keycode2bytes = dict()

for ctckey, ctcshort in char2t3code.items():
    _keycode2bytes[ctckey] = bytearray(_t3key_format.pack(ctcshort, 0, 0))

class T3Event(object):
    DIGITAL_INPUT_EVENT = 1
    ANALOG_INPUT_EVENT = 2
    THRESHOLD_EVENT = 3

    def __init__(self, event_type, event_time_bytes, remaining_bytes):
        self._type = event_type
        self.usec_device_time = (event_time_bytes[4] << 40) + (event_time_bytes[5] << 32) \
            + (event_time_bytes[0] << 24) + (event_time_bytes[1] << 16) \
            + (event_time_bytes[2] << 8) + event_time_bytes[3]
        self.device_time = self.usec_device_time / 1000000.0
        self.local_time = T3Request.sync_state.remote2LocalTime(
            self.device_time)
        self._parseRemainingBytes(remaining_bytes)

    def getTypeInt(self):
        return self._type

    def asdict(self):
        return dict(event_type=self.getTypeInt(),
                    device_time=self.device_time,
                    device_usec_time=self.device_usec_time,
                    local_time=self.local_time)

    def _parseRemainingBytes(self, remaining_bytes):
        raise AttributeError('T3Event._parseRemainingBytes must be extended')

class DigitalInputEvent(T3Event):

    def __init__(self, event_type, event_time_bytes, din_value):
        T3Event.__init__(self, event_type, event_time_bytes, din_value)

    def _parseRemainingBytes(self, din_value):
        self._value = din_value[0]

    def getDigitalInputByte(self):
        return self._value

    def asdict(self):
        rdict = T3Event.asdict(self)
        rdict['value'] = self.getDigitalInputByte()
        return rdict

class AnalogInputEvent(T3Event):

    def __init__(self, event_type, event_time_bytes, analog_data):
        T3Event.__init__(self, event_type, event_time_bytes, analog_data)

    def _parseRemainingBytes(self, analog_data):
        self.ain_channels = []
        for i in range(0, len(analog_data), 2):
            aval = (analog_data[i] << 8) + analog_data[i + 1]
            self.ain_channels.append(aval)

    def asdict(self):
        rdict = T3Event.asdict(self)
        rdict['channels'] = self.ain_channels
        return rdict

class ThresholdEvent(T3Event):

    def __init__(self, event_type, event_time_bytes, threshold_values):
        T3Event.__init__(self, event_type, event_time_bytes, threshold_values)

    def _parseRemainingBytes(self, threshold_values):
        self.threshold_state_changed = []
        for i in range(0, len(threshold_values), 2):
            aval = (threshold_values[i] << 8) + threshold_values[i + 1]
            signed = (aval & 127) - (aval & 128)
            self.threshold_state_changed.append(signed)

    def asdict(self):
        rdict = T3Event.asdict(self)
        rdict['threshold_state_changed'] = self.threshold_state_changed
        return rdict

EVENT_TYPE_2_CLASS = dict()
EVENT_TYPE_2_CLASS[T3Event.DIGITAL_INPUT_EVENT] = DigitalInputEvent
EVENT_TYPE_2_CLASS[T3Event.ANALOG_INPUT_EVENT] = AnalogInputEvent
EVENT_TYPE_2_CLASS[T3Event.THRESHOLD_EVENT] = ThresholdEvent

################################

class T3Request(object):
    NULL_REQUEST = 0
    GET_USEC_TIME = 1
    SET_DIGITAL_OUT_PIN = 2
    SET_DIGITAL_OUT_STATE = 3
    GET_DIGITAL_IN_STATE = 4
    GET_AIN_CHANNELS = 5
    SET_T3_INPUTS_STREAMING_STATE = 6
    SYNC_TIME_BASE = 7
    RESET_STATE = 8
    GENERATE_KEYBOARD_EVENT = 9
    SET_ANALOG_THRESHOLDS = 10
    REQ_COUNTER_START = 11
    _request_counter = REQ_COUNTER_START
    sync_state = None

    def __init__(self, request_type, user_byte_array=None):
        self._id = T3Request._request_counter
        T3Request._request_counter += 1
        if T3Request._request_counter > 255:
            T3Request._request_counter = T3Request.REQ_COUNTER_START

        self._type = request_type
        self._tx_data = bytearray([request_type, self._id, 0])
        if user_byte_array:
            self._tx_data.extend(bytearray([i for i in user_byte_array]))
        self._tx_byte_count = len(self._tx_data)
        self._tx_data[2] = self._tx_byte_count
        self._rx_data = None
        self._rx_byte_count = 0

        if T3Request.sync_state is None:
            T3Request.sync_state = TimeSyncState(5)

        self.tx_time = None
        self.rx_time = None
        self.usec_device_time = None
        self.device_time = None
        self.iohub_time = None

    def getTypeInt(self):
        return self._type

    def getID(self):
        return self._id

    def getTxByteArray(self):
        return self._tx_data

    def getTxByteCount(self):
        return self._tx_byte_count

    def getRxByteArray(self):
        return self._rx_data

    def getRxByteCount(self):
        return self._rx_byte_count

    def setRxByteArray(self, d):
        self._rx_data = d
        self._rx_byte_count = len(d)
        if self._rx_byte_count < 8:
            raise AttributeError(
                'Request Rx Byte Size to short: {0}'.format(
                    self._rx_data))
        self.usec_device_time = (
            d[6] << 40) + (d[7] << 32) + (d[2] << 24) + (d[3] << 16) + (d[4] << 8) + d[5]
        self.device_time = self.usec_device_time / 1000000.0

    @classmethod
    def _readRequestReply(cls, t3, request_id):
        request = t3.getActiveRequests().pop(request_id, None)
        if request is None:
            return None
        request.rx_time = getTime()
        rx_data = [request_id, ord(t3.getSerialPort().read(1))]
        if rx_data[1] > 0:
            rx_data.extend([ord(c)
                            for c in t3.getSerialPort().read(rx_data[1] - 2)])
        request.setRxByteArray(rx_data)
        syncstate = cls.sync_state
        request.iohub_time = None
        if syncstate.getOffset() is not None:
            request.iohub_time = float(
                syncstate.remote2LocalTime(
                    request.device_time))
        return request

    def asdict(self):
        return dict(id=self.getID(),
                    request_type=self.getTypeInt(),
                    device_time=self.device_time,
                    usec_device_time=self.usec_device_time,
                    iohub_time=self.iohub_time,
                    tx_time=self.tx_time,
                    rx_time=self.rx_time
                    )

class EnableT3InputStreaming(T3Request):

    def __init__(self, enable_digital, enable_analog, enable_thresh):
        T3Request.__init__(self, T3Request.SET_T3_INPUTS_STREAMING_STATE, [
                           enable_digital, enable_analog, enable_thresh])

class GetT3UsecRequest(T3Request):

    def __init__(self):
        T3Request.__init__(self, T3Request.GET_USEC_TIME)

class ResetStateRequest(T3Request):

    def __init__(self):
        T3Request.__init__(self, T3Request.RESET_STATE)

class SyncTimebaseRequest(T3Request):
    sync_point_counter = 0
    sync_run_data = np.zeros((3, 3), dtype=np.float64)

    def __init__(self):
        T3Request.__init__(self, T3Request.SYNC_TIME_BASE)

    def syncWithT3Time(self):
        if self.sync_point_counter == 3:
            # calc sync run min, update state, and clear sync_run_data
            min_rtt_point_index = self.sync_run_data[:, 2].argmin()
            L2, R2, RTT2 = self.sync_run_data[min_rtt_point_index, :]
            if len(self.sync_state.RTTs) > 0:
                self.sync_state.offsets.append(R2 - L2)
            self.sync_state.L_times.append(L2)
            self.sync_state.R_times.append(R2)
            SyncTimebaseRequest.sync_point_counter = 0
        else:
            i = SyncTimebaseRequest.sync_point_counter
            self.sync_run_data[i, 0] = (
                (self.rx_time + self.tx_time) / 2.0)  # L
            self.sync_run_data[i, 1] = self.device_time  # R
            self.sync_run_data[i, 2] = (self.rx_time - self.tx_time)  # rtt

            SyncTimebaseRequest.sync_point_counter += 1

class GetT3DigitalInputStateRequest(T3Request):

    def __init__(self):
        T3Request.__init__(self, T3Request.GET_DIGITAL_IN_STATE)

class GetT3AnalogInputStateRequest(T3Request):

    def __init__(self):
        T3Request.__init__(self, T3Request.GET_AIN_CHANNELS)

class GenerateKeyboardEventRequest(T3Request):
    """
    Requests the Teensy to generate a key press event and then a release event
    press_duration*100 msec later. key_symbol must be one of the str keys found
    in the iohub.devices.mcu.iosync.t3keymap.char2t3code dict.
    """

    def __init__(self, key_symbol, modifiers=[], press_duration=0.2):
        ba5 = _keycode2bytes.get(key_symbol, None)
        if ba5 is None:
            estr = "GenerateKeyboardEventRequest: {} is not a valid key_symbol.".format(key_symbol)
            raise ValueError(estr)

        if press_duration < 0 or press_duration > 255:
            estr = "GenerateKeyboardEventRequest: {} is not a valid press_duration.".format(press_duration)
            raise ValueError(estr)

        mods = 0
        for m in modifiers:


from __future__ import division

import numpy as np
from matplotlib.pyplot import *
from matplotlib.colors import LogNorm
import numexpr

def plothist(data, title=None, bins=50, normed=False):
    if 'inline' not in get_backend():
        olf = gcf()
        figure('histograms')
    h, b = np.histogram(data, bins=bins, normed=normed)
    step(b, h.tolist()+[h[-1]], where='post', label=title)
    if 'inline' not in get_backend():
        #focus to the figure previously in focus
        figure(olf.number)

def plot2dhist(datax, datay, title=None, bins=50, normed=False, cbmax=None, cbmin=None, logscale=False):
    h, bx, by = np.histogram2d(datax, datay, bins=bins, normed=normed)
    if logscale:
        imshow(
            h, extent=(bx[0], bx[-2], by[0], by[-2]),
            aspect='auto', interpolation='none',
            norm = LogNorm(vmin=cbmin, vmax=cbmax)
            )
    else:
        imshow(
            h, extent=(bx[0], bx[-2], by[0], by[-2]),
            vmin=cbmin, vmax=cbmax,
            aspect='auto', interpolation='none'
            )

def digitized2dhist(datax, datay, bins=50, zbins=5, logscale=True):
    h, bx, by = np.histogram2d(datax, datay, bins=bins)
    #test iterability
    try:
        [e for e in zbins]
        return -np.rot90(np.digitize(h.ravel(), zbins).reshape(h.shape))
    except TypeError:
        if logscale:
            return -np.rot90(np.digitize(
                h.ravel(),
                np.logspace(0, np.log10(h.max()), zbins)
                ).reshape(h.shape))
        else:
            return np.digitize(
                h.ravel(),
                np.linspace(1, h.max(), zbins)
                ).reshape(h.shape)

def decimate_hist(h, col=1, thr=10):
    """keep only the bins having more than thr. Column 0 is the bin"""
    ret = [h[0]]
    for l in h[1:]:
        if ret[-1][col] > thr:
            ret.append(l)
        else:
            ret[-1][1:] += l[1:]
    return np.array(ret)

def meanMap1D(x, values, bins=50):
    number, b = np.histogram(x, bins)
    total, b = np.histogram(x, bins=b, weights=values)
    selection = np.where(number>0)
    return b[selection], total[selection]/number[selection], number[selection]

def meanMap(x, y, values, bins=50):
    number, bx, by = np.histogram2d(x, y, bins)
    total, bx, by = np.histogram2d(x, y, bins=[bx, by], weights=values)
    return np.where(number==0, -1, total/number), bx[:-1], by[:-1]

def timecorrel(data):
    """time correlation of an array of size [time, particles"""
    k = np.copy(data)
    if k.dtype.kind != 'c':
        k -= data.mean()
    p = numexpr.evaluate(
        'k * complex(k0.real, -k0.imag)',
        {'k':k, 'k0':k[0]}
        ).mean(axis=1)
    n = numexpr.evaluate('abs(k).real**2').mean()
    return numexpr.evaluate('p/n')

def time_correlation(data, av=10):
    """read the particle-wise scalar from a time serie of files and compute the time correlation"""
    if av==0:
        c = np.zeros_like(data.sum(axis=1))
        for t0 in range(len(data)-1):
            c[:len(data)-t0] += timecorrel(data[t0:])
        for dt in range(len(c)):
            c[dt] /= len(c)-dt
        return c
    else:
        return np.mean([timecorrel(data[t0:len(data)+t0-av]) for t0 in range(av)], axis=0)

def correlate_qlm(qlm, average=True):
    qlm_cor = np.real(qlm*np.conj(qlm[0]))[:,:,[0]+2*np.arange(1,qlm.shape[-1])].sum(axis=-1)
    if average:
            return (qlm_cor/qlm_cor[0]).mean(axis=-1)
    else:
            return qlm_cor/qlm_cor[0]

def time_correlation_qlm(qlm, av=10):
    if av==0:
        c = np.zeros(len(qlm), float)
        for t0 in range(len(qlm)-1):
            c[:len(qlm)-t0] += correlate_qlm(qlm[t0:])
        for dt in range(len(c)):
            c[dt] /= len(c)-dt
        return c
    else:
        return np.mean([correlate_qlm(qlm[t0:len(qlm)+t0-av]) for t0 in range(av)], axis=0)

def space_correlation(positions, values, bins):
    """spatial correlation of a scalar observable"""
    v = values - values.mean()
    total = np.zeros(len(bins)-1, values.dtype)
    rdf = np.zeros_like(total)
    for p, u in zip(positions, np.conjugate(v)):
        distsq = np.sum((positions-p)**2, axis=-1)
        h = np.histogram(distsq, bins=bins, weights=v*u)[0]
        n = np.histogram(distsq, bins=bins)[0]
        rdf += n
        nonzero = n>0
        total[nonzero] += h[nonzero]/n[nonzero]
    return total/(v*np.conj(v)).sum(), rdf/bins[1:]**2

class StructureFactor2D:
    """Discreet FFT to make a structure factor"""
    def __init__(self, shape):
        self.im2S = ImageStructureFactor(shape)
        self.im2S.set_window()
        self.im = np.zeros(shape)
        self.S = np.zeros(len(self.im2S.qs))
        self.Ns = []

    def __call__(self, pos, accumulate=True):
        if len(pos)==0:
            self.Ns.append(0)
            return
        #draw a white pixel at the position of each particle
        self.im.fill(0)
        for x, y, z in pos:
            self.im[x,y,z] = 1
        #remove offset
        self.im -= self.im.mean()
        #send to im2S
        S = self.im2S(self.im)
        if accumulate:
            self.S += S*len(pos)
            self.Ns.append(len(pos))
        else:
            return S

    def get_S(self):
        #radial average (division)
        S = np.copy(self.S)
        S[self.nbq>0] /= self.nbq[self.nbq>0]
        S[0] = np.var(self.Ns) * len(self.Ns)
        return S

class StructureFactor3D:
    def __init__(self, shape):
        self.im = np.zeros(shape)
        #mask of wavenumbers
        self.wavenumbers = list(map(np.fft.fftfreq, shape))
        self.dists = numexpr.evaluate('sqrt(qx**2+qy**2+qz**2)', {
            'qx':self.wavenumbers[0][:,None,None],
            'qy':self.wavenumbers[1][None,:,None],
            'qz':self.wavenumbers[1][None,None,:shape[-1]//2+1]
            })
        self.spectrum = np.zeros_like(self.dists)
        #bin the wavenumbers
        self.nbq, self.qs = np.histogram(self.dists.ravel(), self.wavenumbers[0][:len(self.wavenumbers[0])//2])
        self.S = np.zeros(self.nbq.shape)
        self.ws = list(map(np.hamming, shape))
        self.Ns = []

    def __call__(self, pos, periodic=False, accumulate=True):
        if len(pos)==0:
            self.Ns.append(0)
            return
        #draw a white pixel at the position of each particle
        self.im.fill(0)
        for x, y, z in pos:
            self.im[x,y,z] = 1
        #remove offset
        self.im -= self.im.mean()
        if not periodic:
            #windowing
            for d, w in enumerate(self.ws):
                self.im *= w[tuple([None]*d + [slice(None)] + [None]*(2-d))]
        #do the (half)Fourier transform
        self.spectrum = numexpr.evaluate(
            'abs(f).real**2',
            {'f': np.fft.rfftn(self.im)}
            )

        #radial average (sum)
        S = np.histogram(self.dists.ravel(), self.qs, weights=self.spectrum.ravel())[0]/self.spectrum.mean()
        if accumulate:
            self.S += S*len(pos)
            self.Ns.append(len(pos))
        else:
            return S

    def get_S(self):
        #radial average (division)
        S = np.copy(self.S)
        S[self.nbq>0] /= self.nbq[self.nbq>0]
        S[0] = np.var(self.Ns) * len(self.Ns)
        return S

class ImageStructureFactor:
    """A class to compute radially averaged structure factor of a 2D image"""
    def __init__(self, shape):
        assert len(shape) == 2, "only 2D images implemented"
        L = max(shape)


# vim: set fileencoding=utf-8 :
"""
 mediatum - a multimedia content repository

 Copyright (C) 2007 Arne Seifert <seiferta@in.tum.de>
 Copyright (C) 2007 Matthias Kramm <kramm@in.tum.de>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
import re
import sys

pattern_bracket = re.compile("\\([^)(]*\\)")
pattern_string = re.compile('"([^"]*)"')
pattern_and = re.compile("\\b[aA][nN][dD]\\b")
pattern_or = re.compile("\\b[oO][rR]\\b")
pattern_not = re.compile("\\s*\\b[nN][oO][tT]\\b\\s*")
pattern_marker = re.compile("@c<([0-9]*)>c@")
pattern_stringmarker = re.compile("@s<([0-9]*)>s@")
pattern_space = re.compile("[ \t\n\r]")

class ParseException:

    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return "parse exception: " + self.msg + "\n"

class AndCondition:

    def __init__(self, a, b):
        self.a = a
        self.b = b

    def __str__(self):
        return "(" + ustr(self.a) + ") AND (" + ustr(self.b) + ")"

class OrCondition:

    def __init__(self, a, b):
        self.a = a
        self.b = b

    def __str__(self):
        return "(" + ustr(self.a) + ") OR (" + ustr(self.b) + ")"

class NotCondition:

    def __init__(self, a):
        self.a = a

    def __str__(self):
        return "NOT (" + ustr(self.a) + ")"

class TrueCondition:

    def __init__(self):
        pass

    def __str__(self):
        return "TRUE"

class FalseCondition:

    def __init__(self):
        pass

    def __str__(self):
        return "FALSE"

class BoolParser:

    def extendClauses(self, s, l):
        scanner = pattern_marker.scanner(s)
        while True:
            match = scanner.search()
            if not match:
                break
            c = match.group()
            clause = l[int(match.group(1))]
            s = s.replace(c, clause)
        return s

    def extendStrings(self, s, l):
        scanner = pattern_stringmarker.scanner(s)
        while True:
            match = scanner.search()
            if not match:
                break
            c = match.group()
            string = l[int(match.group(1))]
            s = s.replace(c, string)
        return s

    def replaceStrings(self, s):
        l = []
        while True:
            m = pattern_string.search(s)
            if m:
                clause = m.group(1)
                s = s[0:m.start()] + "@s<" + ustr(len(l)) + ">s@" + s[m.end():]
                l += [clause]
            else:
                break
        return s, l

    def parse2(self, s, l=None, stringlist=None, onlybrackets=0):
        if l is None:
            l = []
        if stringlist is None:
            s, stringlist = self.replaceStrings(s)

        s = s.strip()

        while True:
            # remove outer brackets ()
            while s[0] == '(' and s[-1] == ')' and ('(' not in s[1:-1]) and (')' not in s[1:-1]):
                s = s[1:-1].strip()

            m = pattern_bracket.search(s)
            if m:
                clause = m.group()
                s = s[0:m.start()] + "@c<" + ustr(len(l)) + ">c@" + s[m.end():]
                l += [self.extendClauses(clause, l)]
            else:
                break

        # handle OR
        m = pattern_or.search(s)
        if m:
            if m.start() <= 0:
                raise ParseException("left side of OR missing while parsing \"" + s + "\"")
            if m.end() >= len(s) - 1:
                raise ParseException("right side of OR missing while parsing \"" + s + "\"")

            left = self.parse2(self.extendClauses(s[0:m.start()], l), l, stringlist)
            right = self.parse2(self.extendClauses(s[m.end():], l), l, stringlist)
            return self.getOrClass()(left, right)

        # handle AND
        m = pattern_and.search(s)

        if m:
            if m.start() <= 0:
                raise ParseException("left side of AND missing while parsing \"" + s + "\"")

            if m.end() >= len(s) - 1:
                raise ParseException("right side of AND missing while parsing \"" + s + "\"")

            left = self.parse2(self.extendClauses(s[0:m.start()], l), l, stringlist)
            right = self.parse2(self.extendClauses(s[m.end():], l), l, stringlist)

            return self.getAndClass()(left, right)

        s = s.strip()

        if s.lower().startswith("not "):
            inverse = self.parse2(self.extendClauses(s[4:], l), l, stringlist)
            return self.getNotClass()(inverse)

        term = self.extendClauses(s, l).strip()
        if '(' in term and not onlybrackets:
            return self.parse2(term, l, stringlist, 1)
        else:
            return self.parseSimpleCondition(self.extendStrings(term, stringlist))

    def parse(self, s):
        s = pattern_space.sub(" ", s).strip()
        if len(s) == 0:
            return self.default()
        return self.parse2(s)

    def parseSimpleCondition(self, s):
        s2 = s.lower()
        if s2 == "false":
            return FalseCondition()
        if s2 == "true":
            return TrueCondition()
        raise ParseException("syntax error: " + s)

    def default(self):
        return TrueCondition()

    def getAndClass(self):
        return AndCondition

    def getOrClass(self):
        return OrCondition

    def getNotClass(self):
        return NotCondition

if __name__ == "__main__":
    b = BoolParser()
    print ustr(b.parse("not true or not false"))
    print ustr(b.parse("true and (true and (true and (true)))"))
    print ustr(b.parse("((((true) and false) and true) and false) and true"))
    print ustr(b.parse("true or (true or (true or (true)))"))
    print ustr(b.parse("((((true) or false) or (true or false)) or (false or true)) or true"))
    print ustr(b.parse("((true and false and ((false))))"))

    class StringCondition:

        def __init__(self, s):
            self.s = s

        def __str__(self):
            return "'" + self.s + "'"

    class StringParser(BoolParser):

        def parseSimpleCondition(self, s):
            s2 = s.lower()
            if s2 == "false":
                return FalseCondition()


"""SCons.Tool.qt

Tool-specific initialization for Qt.

There normally shouldn't be any need to import this module directly.
It will usually be imported through the generic SCons.Tool.Tool()
selection method.

"""

#
# Copyright (c) 2001 - 2016 The SCons Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "src/engine/SCons/Tool/qt.py rel_2.5.0:3543:937e55cd78f7 2016/04/09 11:29:54 bdbaddog"

import os.path
import re

import SCons.Action
import SCons.Builder
import SCons.Defaults
import SCons.Scanner
import SCons.Tool
import SCons.Util

class ToolQtWarning(SCons.Warnings.Warning):
    pass

class GeneratedMocFileNotIncluded(ToolQtWarning):
    pass

class QtdirNotFound(ToolQtWarning):
    pass

SCons.Warnings.enableWarningClass(ToolQtWarning)

header_extensions = [".h", ".hxx", ".hpp", ".hh"]
if SCons.Util.case_sensitive_suffixes('.h', '.H'):
    header_extensions.append('.H')
cplusplus = __import__('c++', globals(), locals(), [])
cxx_suffixes = cplusplus.CXXSuffixes

def checkMocIncluded(target, source, env):
    moc = target[0]
    cpp = source[0]
    # looks like cpp.includes is cleared before the build stage :-(
    # not really sure about the path transformations (moc.cwd? cpp.cwd?) :-/
    path = SCons.Defaults.CScan.path(env, moc.cwd)
    includes = SCons.Defaults.CScan(cpp, env, path)
    if not moc in includes:
        SCons.Warnings.warn(
            GeneratedMocFileNotIncluded,
            "Generated moc file '%s' is not included by '%s'" %
            (str(moc), str(cpp)))

def find_file(filename, paths, node_factory):
    for dir in paths:
        node = node_factory(filename, dir)
        if node.rexists():
            return node
    return None

class _Automoc(object):
    """
    Callable class, which works as an emitter for Programs, SharedLibraries and
    StaticLibraries.
    """

    def __init__(self, objBuilderName):
        self.objBuilderName = objBuilderName

    def __call__(self, target, source, env):
        """
        Smart autoscan function. Gets the list of objects for the Program
        or Lib. Adds objects and builders for the special qt files.
        """
        try:
            if int(env.subst('$QT_AUTOSCAN')) == 0:
                return target, source
        except ValueError:
            pass
        try:
            debug = int(env.subst('$QT_DEBUG'))
        except ValueError:
            debug = 0

        # some shortcuts used in the scanner
        splitext = SCons.Util.splitext
        objBuilder = getattr(env, self.objBuilderName)

        # some regular expressions:
        # Q_OBJECT detection
        q_object_search = re.compile(r'[^A-Za-z0-9]Q_OBJECT[^A-Za-z0-9]')
        # cxx and c comment 'eater'
        #comment = re.compile(r'(//.*)|(/\*(([^*])|(\*[^/]))*\*/)')
        # CW: something must be wrong with the regexp. See also bug #998222
        #     CURRENTLY THERE IS NO TEST CASE FOR THAT

        # The following is kind of hacky to get builders working properly (FIXME)
        objBuilderEnv = objBuilder.env
        objBuilder.env = env
        mocBuilderEnv = env.Moc.env
        env.Moc.env = env

        # make a deep copy for the result; MocH objects will be appended
        out_sources = source[:]

        for obj in source:
            if not obj.has_builder():
                # binary obj file provided
                if debug:
                    print "scons: qt: '%s' seems to be a binary. Discarded." % str(obj)
                continue
            cpp = obj.sources[0]
            if not splitext(str(cpp))[1] in cxx_suffixes:
                if debug:
                    print "scons: qt: '%s' is no cxx file. Discarded." % str(cpp)
                # c or fortran source
                continue
            #cpp_contents = comment.sub('', cpp.get_text_contents())
            cpp_contents = cpp.get_text_contents()
            h=None
            for h_ext in header_extensions:
                # try to find the header file in the corresponding source
                # directory
                hname = splitext(cpp.name)[0] + h_ext
                h = find_file(hname, (cpp.get_dir(),), env.File)
                if h:
                    if debug:
                        print "scons: qt: Scanning '%s' (header of '%s')" % (str(h), str(cpp))
                    #h_contents = comment.sub('', h.get_text_contents())
                    h_contents = h.get_text_contents()
                    break
            if not h and debug:
                print "scons: qt: no header for '%s'." % (str(cpp))
            if h and q_object_search.search(h_contents):
                # h file with the Q_OBJECT macro found -> add moc_cpp
                moc_cpp = env.Moc(h)
                moc_o = objBuilder(moc_cpp)
                out_sources.append(moc_o)
                #moc_cpp.target_scanner = SCons.Defaults.CScan
                if debug:
                    print "scons: qt: found Q_OBJECT macro in '%s', moc'ing to '%s'" % (str(h), str(moc_cpp))
            if cpp and q_object_search.search(cpp_contents):
                # cpp file with Q_OBJECT macro found -> add moc
                # (to be included in cpp)
                moc = env.Moc(cpp)
                env.Ignore(moc, moc)
                if debug:
                    print "scons: qt: found Q_OBJECT macro in '%s', moc'ing to '%s'" % (str(cpp), str(moc))
                #moc.source_scanner = SCons.Defaults.CScan
        # restore the original env attributes (FIXME)
        objBuilder.env = objBuilderEnv
        env.Moc.env = mocBuilderEnv

        return (target, out_sources)

AutomocShared = _Automoc('SharedObject')
AutomocStatic = _Automoc('StaticObject')

def _detect(env):
    """Not really safe, but fast method to detect the QT library"""
    QTDIR = None
    if not QTDIR:
        QTDIR = env.get('QTDIR',None)
    if not QTDIR:
        QTDIR = os.environ.get('QTDIR',None)
    if not QTDIR:
        moc = env.WhereIs('moc')
        if moc:
            QTDIR = os.path.dirname(os.path.dirname(moc))
            SCons.Warnings.warn(
                QtdirNotFound,
                "Could not detect qt, using moc executable as a hint (QTDIR=%s)" % QTDIR)
        else:
            QTDIR = None
            SCons.Warnings.warn(
                QtdirNotFound,
                "Could not detect qt, using empty QTDIR")
    return QTDIR

def uicEmitter(target, source, env):
    adjustixes = SCons.Util.adjustixes
    bs = SCons.Util.splitext(str(source[0].name))[0]
    bs = os.path.join(str(target[0].get_dir()),bs)
    # first target (header) is automatically added by builder
    if len(target) < 2:
        # second target is implementation
        target.append(adjustixes(bs,
                                 env.subst('$QT_UICIMPLPREFIX'),
                                 env.subst('$QT_UICIMPLSUFFIX')))
    if len(target) < 3:
        # third target is moc file
        target.append(adjustixes(bs,
                                 env.subst('$QT_MOCHPREFIX'),
                                 env.subst('$QT_MOCHSUFFIX')))
    return target, source

def uicScannerFunc(node, env, path):
    lookout = []
    lookout.extend(env['CPPPATH'])
    lookout.append(str(node.rfile().dir))
    includes = re.findall("<include.*?>(.*?)</include>", node.get_text_contents())
    result = []
    for incFile in includes:
        dep = env.FindFile(incFile,lookout)
        if dep:
            result.append(dep)
    return result

uicScanner = SCons.Scanner.Base(uicScannerFunc,
                                name = "UicScanner",
                                node_class = SCons.Node.FS.File,
                                node_factory = SCons.Node.FS.File,
                                recursive = 0)

def generate(env):
    """Add Builders and construction variables for qt to an Environment."""
    CLVar = SCons.Util.CLVar
    Action = SCons.Action.Action
    Builder = SCons.Builder.Builder

    env.SetDefault(QTDIR  = _detect(env),
                   QT_BINPATH = os.path.join('$QTDIR', 'bin'),
                   QT_CPPPATH = os.path.join('$QTDIR', 'include'),
                   QT_LIBPATH = os.path.join('$QTDIR', 'lib'),
                   QT_MOC = os.path.join('$QT_BINPATH','moc'),