<commit_before>package org.mwc.debrief.track_shift.zig_detector.target;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import junit.framework.TestCase;

import org.apache.commons.math3.stat.regression.SimpleRegression;
import org.eclipse.core.runtime.ILog;
import org.eclipse.core.runtime.ILogListener;
import org.eclipse.core.runtime.IStatus;
import org.mwc.debrief.track_shift.zig_detector.moving_average.TimeRestrictedMovingAverage;
import org.mwc.debrief.track_shift.zig_detector.ownship.LegOfData;
import org.osgi.framework.Bundle;

import flanagan.math.Minimisation;
import flanagan.math.MinimisationFunction;

public class ZigDetector
{
  private static class ArrayWalker
  {
    private final int _start;
    private final int _end;
    private int _current;
    private final int _step;

    private Boolean _lastAbove = null;
    private int _lastOpposite;
    private int _sameSideCount;
    private String _name;
    private double _lastError = 0d;

    public ArrayWalker(final int start, final int end, final String name)
    {
      _start = start;
      _end = end;
      _name = name;

      _current = start;
      _step = _end > _start ? 1 : -1;
    }

    @Override
    public String toString()
    {
      return _name;
    }

    final public boolean justStarted()
    {
      return _lastAbove == null;
    }

    final public void setLastOpposite(int i)
    {
      _lastOpposite = i;
      _lastError = 0d;
    }

    final public int getLastOpposite()
    {
      return _lastOpposite;
    }

    final public void setSameSideCount(int i)
    {
      _sameSideCount = i;
    }

    final public int getSameSideCount()
    {
      return _sameSideCount;
    }

    final public void setLastAbove(boolean thisAbove)
    {
      _lastAbove = thisAbove;
    }

    final public boolean getLastAbove()
    {
      return _lastAbove;
    }

    public boolean hasNext()
    {
      return _current != _end + _step;
    }

    public Integer next()
    {
      // check we haven't passed the end
      if (!hasNext())
      {
        throw new IllegalArgumentException(
            "Can't call next() once we've passed the end");
      }

      final Integer res = _current;
      _current += _step;
      return res;
    }

    public double getLastError()
    {
      return _lastError;
    }

    public void setLastError(double error2)
    {
      _lastError = error2;
    }
  }

  private static interface EventHappened
  {
    public void eventAt(long time, double score, double threshold);
  }

  static class FlanaganArctan implements MinimisationFunction
  {
    private static double calcForecast(final double[] params,
        final double elapsedSecs)
    {
      final double B = params[0];
      final double P = params[1];
      final double Q = params[2];
      final double dX = Math.cos(Math.toRadians(B)) + Q * elapsedSecs;
      final double dY = Math.sin(Math.toRadians(B)) + P * elapsedSecs;
      double res = Math.toDegrees(Math.atan2(dY, dX));
      if (res < 0)
      {
        res += 360;
      }
      return res;
    }

    final private List<Long> _times;

    final private List<Double> _bearings;

    public FlanaganArctan(final List<Long> beforeTimes,
        final List<Double> beforeBearings)
    {
      _times = beforeTimes;
      _bearings = beforeBearings;
    }

    // evaluation function
    @Override
    public double function(final double[] params)
    {
      double runningSum = 0;

      // ok, loop through the data
      for (int i = 0; i < _times.size(); i++)
      {
        final double thisForecast = calcForecast(params, _times.get(i));
        final double thisMeasured = _bearings.get(i);
        double thisError = thisForecast - thisMeasured;
        if (thisError > 180)
        {
          thisError -= 360;
        }
        else if (thisError < -180)
        {
          thisError += 360;
        }
        final double sqError = Math.pow(thisError, 2);

        runningSum += sqError;
      }
      // normalise by dividing by the number cuts - 3
      // (the -3 is relates to the number of variables being considered)
      final double mean = runningSum / (_times.size() - 3);

      return mean;

//       final double rms = Math.sqrt(mean);
//       return rms;
    }

  }

  static class FlanaganArctan_Legacy implements MinimisationFunction
  {
    private static double calcForecast(final double B, final double P,
        final double Q, final double elapsedSecs)
    {
      final double dX = Math.cos(Math.toRadians(B)) + Q * elapsedSecs;
      final double dY = Math.sin(Math.toRadians(B)) + P * elapsedSecs;
      return Math.toDegrees(Math.atan2(dY, dX));
    }

    final private Long[] _times;

    final private Double[] _bearings;

    public FlanaganArctan_Legacy(final List<Long> beforeTimes,
        final List<Double> beforeBearings)
    {
      _times = beforeTimes.toArray(new Long[]
      {});
      _bearings = beforeBearings.toArray(new Double[]
      {});
    }

    // evaluation function
    @Override
    public double function(final double[] point)
    {
      final double B = point[0];
      final double P = point[1];
      final double Q = point[2];

      double runningSum = 0;
      final Long firstTime = _times[0];

      // ok, loop through the data
      for (int i = 0; i < _times.length; i++)
      {
        final long elapsedMillis = _times[i] - firstTime;
        final double elapsedSecs = elapsedMillis / 1000d;
        final double thisForecast = calcForecast(B, P, Q, elapsedSecs);
        final double thisMeasured = _bearings[i];
        double thisError = thisForecast - thisMeasured;
        if (thisError > 180)
        {
          thisError -= 360;
        }
        else if (thisError < -180)
        {
          thisError += 360;
        }
        final double sqError = Math.pow(thisError, 2);
        runningSum += sqError;
      }
      final double mean = runningSum / _times.length;

      final double rms = Math.sqrt(mean);
      return rms;
    }

  }

  protected static class ScoredTime implements Comparable<ScoredTime>
  {
    private final long _time;
    private final double _score;

    public ScoredTime(final long time, final double score)
    {
      _time = time;
      _score = score;
    }

    @Override
    public int compareTo(final ScoredTime o)
    {
      final Long myTime = _time;
      final Long hisTime = o._time;
      return myTime.compareTo(hisTime);
    }
  }

  public static class TestMe extends TestCase
  {
    @SuppressWarnings("unused")
    private ILegStorer getLegStorer()
    {
      final ILegStorer legStorer = new ILegStorer()
      {

        @Override
        public void storeLeg(final String scenarioName, final long tStart,
            final long tEnd, final double rms)
        {
          // System.out.println("store it: " + new Date(tStart) + ", "
          // + new Date(tEnd));
        }
      };
      return legStorer;
    }

    @SuppressWarnings("unused")
    private ILog getLogger()
    {
      final ILog logger = new ILog()
      {

        @Override
        public void addLogListener(final ILogListener listener)
        {
          // not required, class just for testing
        }

        @Override
        public Bundle getBundle()
        {
          return null;
        }

        @Override
        public void log(final IStatus status)
        {
          // not required, class just for testing
        }

        @Override
        public void removeLogListener(final ILogListener listener)
        {
          // not required, class just for testing
        }
      };
      return logger;
    }

    @SuppressWarnings("unused")
    private IZigStorer getZigStorer()
    {
      final IZigStorer zigStorer = new IZigStorer()
      {

        @Override
        public void finish()
        {
          // TODO Auto-generated method stub

        }

        @Override
        public void storeZig(final String scenarioName, final long tStart,
            final long tEnd, final double rms)
        {
          // TODO Auto-generated method stub

        }
      };
      return zigStorer;
    }

    public void testCalcStart()
    {
      final List<Long> list = new ArrayList<Long>(Arrays.asList(new Long[]
      {0L, 40L, 100L, 140L, 180L, 220L, 260L, 280L}));
      assertEquals("correct time", 3, calculateNewStart(list, 1, 100));
      assertEquals("correct time", 5, calculateNewStart(list, 2, 100));
      assertEquals("correct time", 1, calculateNewStart(list, 0, 20));

      // and a reverse list
      Collections.reverse(list);
      assertEquals("correct time", 4, calculateNewStart(list, 1, 100));
      assertEquals("correct time", 5, calculateNewStart(list, 2, 100));
      assertEquals("correct time", 2, calculateNewStart(list, 0, 30));

    }

    public void testCleanValues()
    {
      List<Double> list = new ArrayList<Double>();
      list.add(355d);
      list.add(4d);
      list.add(2d);
      list.add(349d);
      list.add(2d);
      list = prepareBearings(list);

      assertEquals("correct length", 5, list.size());
      assertEquals("fixed bearings", 364d, list.get(1), 0.001);
      assertEquals("fixed bearings", 362d, list.get(2), 0.001);
      assertEquals("fixed bearings", 349d, list.get(3), 0.001);
      assertEquals("fixed bearings", 362d, list.get(4), 0.001);

      // and the other way around
      list.clear();

      list.add(55d);
      list.add(354d);
      list.add(339d);
      list.add(9d);
      list.add(2d);
      list = prepareBearings(list);

      assertEquals("correct length", 5, list.size());
      assertEquals("fixed bearings", -6d, list.get(1), 0.001);
      assertEquals("fixed bearings", -21d, list.get(2), 0.001);
      assertEquals("fixed bearings", 9d, list.get(3), 0.001);
      assertEquals("fixed bearings", 2d, list.get(4), 0.001);

    }

    public void testMultiSlice() throws ParseException
    {

      final double[] times =
          new double[]
          {0, 17.99999981, 33.99999978, 47.99999991, 62.00000003, 82.00000031,
              98.00000028, 109.9999999, 125.9999999, 140, 157.9999998, 172,
              186.0000001, 200.0000002, 210.0000001, 227.9999999, 243.9999998,
              258, 272.0000001, 294.0000002, 312, 332.0000003, 358.0000001,
              383.9999999, 410.0000003, 436.0000001, 472.0000003, 504.0000003,
              531.9999999, 563.9999998, 592.0000001, 624, 657.9999998,
              692.0000002, 727.9999998, 764.0000001, 810.0000001, 840.0000002,
              875.9999998, 907.9999998, 944, 982.0000001, 1020, 1056, 1092,
              1126, 1172, 1214, 1258, 1296, 1334, 1372, 1404, 1426, 1456, 1486,
              1514, 1540, 1562, 1590, 1624, 1652, 1678, 1710, 1744, 1774, 1808,
              1846, 1880, 1912, 1942, 1982, 2014, 2050, 2082, 2118, 2154, 2186,
              2220, 2246, 2276, 2298, 2324, 2354, 2386, 2414, 2446, 2476, 2510,
              2548, 2580, 2618, 2650, 2686, 2720, 2758, 2790, 2822, 2852, 2886,
              2922, 2952, 2980, 3012, 3038, 3066, 3090, 3114, 3140, 3172, 3202,
              3234, 3274, 3310, 3344, 3382, 3418, 3448, 3478, 3508, 3542, 3570,
              3606, 3642, 3676, 3710, 3746, 3778, 3808, 3846, 3884, 3922, 3954,
              3990, 4026, 4060, 4098, 4140, 4174, 4214, 4254, 4288, 4324, 4360,
              4398, 4440, 4484, 4512, 4542, 4574, 4610, 4644, 4668, 4690, 4716,
              4738, 4760, 4782, 4802, 4822, 4842, 4852, 4864, 4880, 4900, 4916,
              4932, 4952, 4972, 4992, 5014, 5040, 5068, 5090, 5112, 5138, 5158,
              5188, 5210, 5238, 5264, 5294, 5324, 5358, 5390, 5416, 5444, 5476,
              5514, 5556, 5592, 5634, 5680, 5720, 5756, 5800, 5842, 5882, 5918,
              5958, 6000, 6046, 6090, 6128, 6186, 6222, 6264, 6300, 6334, 6370,
              6408, 6448, 6488, 6534, 6576, 6620, 6664, 6712};
      final Double[] bearings =
          new Double[]
          {-147d, -143.6, -140.3, -137.3, -134d, -130d, -125.9, -122.6, -117.9,
              -114.9, -111.3, -107.3, -103.6, -99.6, -95.6, -91.3, -87.9,
              -84.2, -80.6, -76.9, -72.6, -68.9, -65.6, -60.6, -56.6, -52.9,
              -50.2, -47.2, -45.5, -43.9, -42.2, -40.2, -38.9, -37.9, -36.9,
              -36.5, -36.2, -35.5, -34.9, -34.2, -33.5, -32.9, -32.2, -31.9,
              -31.2, -30.9, -29.9, -28.5, -27.9, -27.2, -25.5, -24.9, -23.9,
              -23.9, -26.2, -28.5, -29.5, -31.5, -32.5, -31.5, -31.5, -31.2,
              -31.2, -30.9, -30.5, -29.9, -28.9, -28.2, -27.5, -27.9, -26.5,
              -25.5, -24.9, -24.2, -23.5, -22.9, -21.9, -20.9, -19.9, -19.5,
              -18.9, -18.2, -17.5, -16.8, -15.5, -15.2, -14.5, -13.8, -13.2,
              -12.2, -11.5, -10.2, -9.8, -8.5, -7.5, -6.5, -5.5, -4.8, -3.5,
              -3.2, -2.2, -1.5, -0.8, -0.2, 1.2, 1.5, 2.2, 4.2, 4.8, 5.8, 6.8,
              7.8, 8.8, 10.2, 10.5, 11.5, 12.5, 13.2, 14.5, 14.8, 16.5, 17.5,
              18.5, 19.9, 21.2, 21.9, 22.9, 23.2, 24.5, 25.5, 26.5, 26.9, 27.5,
              28.2, 28.5, 29.2, 29.5, 29.9, 30.2, 30.9, 31.5, 31.9, 32.5, 32.5,
              32.9, 32.9, 32.9, 32.9, 33.2, 33.2, 32.9, 31.9, 30.2, 27.5, 23.9,
              20.2, 14.8, 8.8, 3.2, -3.8, -9.5, -14.8, -19.5, -24.9, -31.2,
              -36.2, -40.2, -46.5, -51.5, -55.6, -60.6, -66.6, -72.9, -77.9,
              -82.9, -87.9, -91.9, -96.9, -101.6, -104.9, -107.9, -111.6,
              -114.9, -117.6, -120.3, -122.3, -123.3, -123.3, -123.3, -123.3,
              -122.3, -120.9, -118.9, -118.3, -116.9, -116.3, -115.3, -114.3,
              -113.3, -111.9, -110.9, -110.3, -109.3, -108.6, -107.6, -106.6,
              -105.6, -104.9, -104.3, -103.6, -102.3, -100.9, -99.9, -97.9,
              -96.6, -95.3, -92.9, -91.9};

      final Long[] normalTimes = new Long[times.length];
      final Long[] rawTimes = new Long[times.length];
      // final java.text.DateFormat sdf = new SimpleDateFormat("HHmmss");
      long startTime = 0;
      for (int i = 0; i < times.length; i++)
      {
        final long thisTime = (long) times[i];

        rawTimes[i] = thisTime;

        if (i == 0)
        {
          startTime = thisTime;
        }

        normalTimes[i] = (thisTime - startTime);
      }

      // start to collate the adta
      final List<Long> rawList1 = Arrays.asList(rawTimes);

      final List<Double> tBearings1 = Arrays.asList(bearings);

      // System.out
      // .println("last time:" + new Date(tList1.get(tList1.size() - 1)));

      // get the last 40 elements
      // final int start = tList1.size() - 50;
      // final int end = tList1.size() - 1;
      //
      // List<Long> tList = tList1.subList(start, end);
      // List<Double> tBearings = tBearings1.subList(start, end);

      final List<Double> tBearings = tBearings1;

      // final TPeriod flattest = findLowestRateIn(tList, tBearings);
      // assertNotNull("found data", flattest);
      // assertEquals(0, flattest.start);
      // assertEquals(6, flattest.end);

      // System.out.println("from:" + new Date(times[0]) + " // to:"
      // + new Date(times[times.length - 1]) + " // " + times.length + " entries");

      final ZigDetector detector = new ZigDetector();
      double zigRatio = 1000d;
      final double optimiseTolerance = 0.0000000004;

      // ILog logger = getLogger();
      // ILegStorer legStorer = getLegStorer();
      // IZigStorer zigStorer = getZigStorer();
      // detector.sliceThis(logger, "some name", "scenario", times[0],
      // times[times.length - 1], legStorer, zigStorer, zigRatio,
      // optimiseTolerance, tList, tBearings);

      // reverse the arrays
      // Collections.reverse(tList);
      // Collections.reverse(tBearings);
      //
      final long timeWindow = 240000;
      zigRatio = 15d;
      final EventHappened happened = new EventHappened()
      {
        @Override
        public void eventAt(final long time, final double score,
            final double threshold)
        {
          // System.out.println("event at " + new Date(time) + " score:" + score);
        }
      };

      final List<TPeriod> legs = new ArrayList<TPeriod>();

      detector.runThrough2(optimiseTolerance, rawList1, tBearings, happened,
          zigRatio, timeWindow, legs);

      listSlices(legs, rawList1);

      listSlicesForPlotting(legs, rawList1);

      // final Double[] bearings =
      // new Double[]
      // {180d, 180.3, 180.7, 181d, 181.4, 181.7, 182.1, 182.5, 182.8, 183.2,
      // 183.6, 184.1, 184.5, 184.9, 185.3, 185.8, 186.3, 186.7, 187.2,
      // 187.7, 188.2, 188.8, 189.3, 189.8, 190.4, 191d, 191.6, 192.2,
      // 192.8, 193.4, 194.1, 194.8, 195.5, 196.2, 196.9, 197.6, 198.4,
      // 199.2, 200d, 200.8, 201.7, 202.6, 203.4, 204.4, 205.3, 206.1,
      // 206.7, 207.3, 207.9, 208.5, 209.2, 209.9, 210.6, 211.4, 212.2,
      // 213.1, 214d, 214.9, 215.9, 216.9, 218d, 219.1, 220.3, 221.6,
      // 223d, 224.4, 225.9, 227.4, 229.1, 230.8, 232.7, 234.6, 236.6,
      // 238.8, 241d, 243.4, 245.9, 248.2, 250.3, 252.3, 254.3, 256.1,
      // 257.9, 259.6, 261.2, 262.8, 264.3, 265.7, 267.1, 268.4, 269.7,
      // 270.9, 272d, 273.1, 274.2, 275.2, 276.2, 277.1, 278d, 278.8,
      // 279.7, 280.4, 281.2};
      //
      // final int[] timeStr =
      // new int[]
      // {120000, 120050, 120140, 120230, 120320, 120410, 120500, 120550,
      // 120640, 120730, 120820, 120910, 121000, 121050, 121140, 121230,
      // 121320, 121410, 121500, 121550, 121640, 121730, 121820, 121910,
      // 122000, 122050, 122140, 122230, 122320, 122410, 122500, 122550,
      // 122640, 122730, 122820, 122910, 123000, 123050, 123140, 123230,
      // 123320, 123410, 123500, 123550, 123640, 123730, 123820, 123910,
      // 124000, 124050, 124140, 124230, 124320, 124410, 124500, 124550,
      // 124640, 124730, 124820, 124910, 125000, 125050, 125140, 125230,
      // 125320, 125410, 125500, 125550, 125640, 125730, 125820, 125910,
      // 130000, 130050, 130140, 130230, 130320, 130410, 130500, 130550,
      // 130640, 130730, 130820, 130910, 131000, 131050, 131140, 131230,
      // 131320, 131410, 131500, 131550, 131640, 131730, 131820, 131910,
      // 132000, 132050, 132140, 132230, 132320, 132410, 132500};

      // Long[] times =
      // new Long[]
      // {1248237792000L, 1248237799000L, 1248237896000L, 1248237944000L,
      // 1248237990000L, 1248238098000L, 1248238177000L, 1248238249000L,
      // 1248238321000L, 1248238393000L, 1248238484000L, 1248238556000L,
      // 1248238624000L, 1248238695000L, 1248238759000L, 1248238843000L,
      // 1248238931000L, 1248239006000L, 1248239074000L, 1248239162000L,
      // 1248239277000L, 1248239353000L, 1248239444000L, 1248239520000L,
      // 1248239600000L, 1248239644000L, 1248239735000L, 1248239799000L,
      // 1248239891000L, 1248239951000L, 1248240030000L, 1248240090000L,
      // 1248240142000L, 1248240198000L, 1248240257000L, 1248240305000L};
      // Double[] bearings =
      // new Double[]
      // {295.8, 295.5, 293.5, 293.0, 292.8, 290.3, 289.0, 288.3, 288.0,
      // 288.0, 288.8, 288.8, 288.8, 289.8, 289.8, 291.0, 291.5, 292.3,
      // 292.3, 293.0, 293.5, 294.0, 294.3, 294.8, 294.8, 294.8, 295.8,
      // 295.8, 295.8, 296.5, 296.5, 297.5, 297.8, 298.3, 299.0, 299.5};

      // Long[] times2 =
      // new Long[]
      // {946697610000L, 946697640000L, 946697670000L, 946697700000L,
      // 946697730000L, 946697760000L, 946697790000L, 946697820000L,
      // 946697850000L, 946697880000L, 946697910000L, 946697940000L,
      // 946697970000L, 946698000000L, 946698030000L, 946698060000L,
      // 946698090000L, 946698120000L, 946698150000L, 946698180000L,
      // 946698210000L, 946698240000L, 946698270000L, 946698300000L,
      // 946698330000L, 946698360000L, 946698390000L, 946698420000L,
      // 946698450000L, 946698480000L, 946698510000L, 946698540000L,
      // 946698570000L, 946698600000L, 946698630000L, 946698660000L,
      // 946698690000L, 946698720000L, 946698750000L, 946698780000L,
      // 946698810000L, 946698840000L, 946698870000L, 946698900000L,
      // 946698930000L, 946698960000L, 946698990000L, 946699020000L,
      // 946699050000L, 946699080000L, 946699110000L, 946699140000L,
      // 946699170000L, 946699200000L, 946699230000L, 946699260000L,
      // 946699290000L, 946699320000L, 946699350000L, 946699380000L,
      // 946699410000L, 946699440000L, 946699470000L, 946699500000L,
      // 946699530000L, 946699560000L, 946699590000L, 946699620000L,
      // 946699650000L, 946699680000L, 946699710000L, 946699740000L,
      // 946699770000L, 946699800000L, 946699830000L, 946699860000L,
      // 946699890000L, 946699920000L, 946699950000L, 946699980000L,
      // 946700010000L, 946700040000L, 946700070000L, 946700100000L,
      // 946700130000L, 946700160000L, 946700190000L, 946700220000L,
      // 946700250000L, 946700280000L, 946700310000L, 946700340000L,
      // 946700370000L, 946700400000L, 946700430000L, 946700460000L,
      // 946700490000L, 946700520000L, 946700550000L, 946700580000L,
      // 946700610000L, 946700640000L, 946700670000L, 946700700000L,
      // 946700730000L, 946700760000L, 946700790000L, 946700820000L,
      // 946700850000L, 946700880000L, 946700910000L, 946700940000L,
      // 946700970000L};
      //
      // Double[] bearings2 =
      // new Double[]
      // {170.095, 170.566, 171.404, 172.021, 172.757, 173.25, 173.767,
      // 174.391, 174.958, 175.839, 176.485, 177.282, 177.66, 178.444,
      // 179.09, 179.671, -179.482, -178.846, -178.363, -177.853,
      // -177.173, -175.994, -175.115, -174.628, -174.019, -173.208,
      // -172.378, -171.79, -170.932, -170.251, -169.526, -168.751,
      // -168.123, -167.354, -166.331, -165.639, -164.767, -164.272,
      // -163.407, -162.441, -161.783, -161.074, -159.886, -158.873,
      // -158.367, -157.495, -156.606, -155.92, -154.829, -153.856,
      // -152.983, -152.355, -151.561, -151.01, -149.65, -149.143,
      // -148.211, -147.211, -146.283, -145.55, -145.102, -144.119,
      // -143.22, -143.185, -141.704, -140.562, -139.975, -139.124,
      // -138.346, -137.36, -137.276, -135.746, -135.333, -134.338,
      // -133.295, -132.577, -131.86, -131.143, -130.278, -129.278,
      // -128.344, -127.83199999999998, -127.107, -126.345,
      // -125.40799999999999, -124.49999999999999, -123.88299999999998,
      // -123.195, -122.52800000000002, -122.17599999999999, -121.21,
      // -120.267, -120.11499999999998, -119.31799999999998, -118.507,
      // -117.99500000000002, -117.689, -117.71000000000001, -117.36,
      // -117.09399999999998, -117.23799999999999, -117.01, -116.633,
      // -116.74, -116.40300000000002, -116.296, -116.158,
      // -115.85400000000001, -115.82, -115.777, -115.56000000000002,
      // -115.071, -114.71999999999998};
      //

      // Long[] times =
      // new Long[]
      // {946699110000L, 946699140000L, 946699170000L, 946699200000L,
      // 946699230000L, 946699260000L, 946699290000L, 946699320000L,
      // 946699350000L, 946699380000L, 946699410000L, 946699440000L,
      // 946699470000L, 946699500000L, 946699530000L, 946699560000L,
      // 946699590000L, 946699620000L, 946699650000L, 946699680000L,
      // 946699710000L, 946699740000L, 946699770000L, 946699800000L,
      // 946699830000L, 946699860000L, 946699890000L, 946699920000L,
      // 946699950000L, 946699980000L, 946700010000L, 946700040000L,
      // 946700070000L, 946700100000L, 946700130000L, 946700160000L,
      // 946700190000L, 946700220000L, 946700250000L, 946700280000L,
      // 946700310000L, 946700340000L, 946700370000L, 946700400000L,
      // 946700430000L, 946700460000L, 946700490000L, 946700520000L,
      // 946700550000L, 946700580000L, 946700610000L, 946700640000L,
      // 946700670000L, 946700700000L, 946700730000L, 946700760000L,
      // 946700790000L, 946700820000L, 946700850000L, 946700880000L,
      // 946700910000L, 946700940000L, 946700970000L};
      // Double[] bearings =
      // new Double[]
      // {207.017, 207.645, 208.439, 208.99, 210.35, 210.857, 211.789,
      // 212.789, 213.717, 214.45, 214.898, 215.881, 216.78, 216.815,
      // 218.296, 219.438, 220.025, 220.876, 221.654, 222.64, 222.724,
      // 224.254, 224.667, 225.662, 226.705, 227.423, 228.14, 228.857,
      // 229.722, 230.722, 231.656, 232.168, 232.893, 233.655, 234.592,
      // 235.5, 236.11700000000002, 236.805, 237.47199999999998, 237.824,
      // 238.79000000000002, 239.733, 239.88500000000002,
      // 240.68200000000002, 241.493, 242.005, 242.311, 242.29, 242.64,
      // 242.906, 242.762, 242.99, 243.36700000000002, 243.26,
      // 243.59699999999998, 243.704, 243.84199999999998, 244.146, 244.18,
      // 244.223, 244.44, 244.929, 245.28000000000003};

      // Long[] times = new Long[]{946699110000L, 946699140000L, 946699170000L, 946699200000L,
      // 946699230000L, 946699260000L, 946699290000L, 946699320000L, 946699350000L, 946699380000L,
      // 946699410000L, 946699440000L, 946699470000L, 946699500000L, 946699530000L, 946699560000L,
      // 946699590000L, 946699620000L, 946699650000L, 946699680000L, 946699710000L, 946699740000L,
      // 946699770000L, 946699800000L, 946699830000L, 946699860000L, 946699890000L, 946699920000L,
      // 946699950000L, 946699980000L, 946700010000L, 946700040000L, 946700070000L, 946700100000L,
      // 946700130000L, 946700160000L, 946700190000L, 946700220000L, 946700250000L, 946700280000L,
      // 946700310000L, 946700340000L, 946700370000L, 946700400000L, 946700430000L, 946700460000L,
      // 946700490000L, 946700520000L, 946700550000L, 946700580000L};
      // Double[] bearings = new Double[]{-152.983, -152.355, -151.561, -151.01, -149.65, -149.143,
      // -148.211, -147.211, -146.283, -145.55, -145.102, -144.119, -143.22, -143.185, -141.704,
      // -140.562, -139.975, -139.124, -138.346, -137.36, -137.276, -135.746, -135.333, -134.338,
      // -133.295, -132.577, -131.86, -131.143, -130.278, -129.278, -128.344, -127.83199999999998,
      // -127.107, -126.345, -125.40799999999999, -124.49999999999999, -123.88299999999998,
      // -123.195, -122.52800000000002, -122.17599999999999, -121.21, -120.267, -120.11499999999998,
      // -119.31799999999998, -118.507, -117.99500000000002, -117.689, -117.71000000000001, -117.36,
      // -117.09399999999998};

    }

    public void testSlicing()
    {
      final List<TPeriod> q = new ArrayList<TPeriod>();
      final List<TPeriod> legs = new ArrayList<TPeriod>();
      final TPeriod outer = new TPeriod(0, 100);
      q.add(outer);
      TPeriod leg = new TPeriod(20, 30);

      assertEquals("one item", 1, q.size());
      assertTrue("correct item", q.contains(outer));

      handleNewSlices(q, legs, outer, leg, 5, true);

      assertEquals("two items", 2, q.size());
      assertEquals("first item", q.get(0), new TPeriod(0, 19));
      assertEquals("second item", q.get(1), new TPeriod(31, 100));
      assertEquals("correct legs", 1, legs.size());

      leg = new TPeriod(30, 40);
      handleNewSlices(q, legs, q.get(1), leg, 5, true);

      assertEquals("three items", 3, q.size());

      assertEquals("first item", new TPeriod(0, 19), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(72, 100), q.get(2));
      assertEquals("correct legs", 2, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 10);

      handleNewSlices(q, legs, q.get(2), leg, 5, true);
      assertEquals("three items", 3, q.size());

      assertEquals("first item", new TPeriod(0, 19), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(83, 100), q.get(2));
      assertEquals("correct legs", 3, legs.size());

      // ok, leave a short one
      leg = new TPeriod(13, 16);

      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("three items", 3, q.size());

      assertEquals("second item", new TPeriod(0, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(83, 100), q.get(2));
      assertEquals("correct legs", 4, legs.size());

      // ok, leave a short one
      leg = new TPeriod(4, 14);

      handleNewSlices(q, legs, q.get(2), leg, 5, true);
      assertEquals("two items", 2, q.size());

      assertEquals("second item", new TPeriod(0, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("correct legs", 5, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 6);
      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("two items", 2, q.size());

      assertEquals("second item", new TPeriod(7, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("correct legs", 6, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 6);
      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("two items", 1, q.size());

      assertEquals("second item", new TPeriod(31, 60), q.get(0));
      assertEquals("correct legs", 7, legs.size());

      listSlices(legs, null);

    }

    public void testStartTimes()
    {
      assertEquals(2, 5 / 2);

      final List<Long> times =
          new ArrayList<Long>(Arrays.asList(new Long[]
          {1000L, 1200L, 1500L, 1800L, 2100L, 2400L, 2700L, 3000L, 3300L,
              3600L, 3900L}));

      assertEquals("correct", -1, getEnd(0, times, 200, 0));
      assertEquals("correct", -1, getEnd(0, times, 200, 1));
      assertEquals("correct", -1, getEnd(0, times, 200, 2));
      assertEquals("correct", 2, getEnd(0, times, 200, 3));
      assertEquals("correct", 3, getEnd(0, times, 200, 4));
      assertEquals("correct", 4, getEnd(0, times, 200, 5));
      assertEquals("correct", 5, getEnd(0, times, 200, 6));
      assertEquals("correct", 6, getEnd(0, times, 200, 7));
      assertEquals("correct", 7, getEnd(0, times, 200, 8));
      assertEquals("correct", 8, getEnd(0, times, 200, 9));
      assertEquals("correct", 9, getEnd(0, times, 200, 10));

      assertEquals("correct", 2, getStart(0, times, 400, 0));
      assertEquals("correct", 2, getStart(0, times, 400, 1));
      assertEquals("correct", 3, getStart(0, times, 400, 2));
      assertEquals("correct", 4, getStart(0, times, 400, 3));
      assertEquals("correct", 5, getStart(0, times, 400, 4));
      assertEquals("correct", 6, getStart(0, times, 400, 5));
      assertEquals("correct", 7, getStart(0, times, 400, 6));
      assertEquals("correct", 8, getStart(0, times, 400, 7));
      assertEquals("correct", -1, getStart(0, times, 400, 8));
      assertEquals("correct", -1, getStart(0, times, 400, 9));
      assertEquals("correct", -1, getStart(0, times, 400, 10));
    }

    public void testWalker()
    {
      final ArrayWalker walker1 = new ArrayWalker(0, 2, "up");
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 0, walker1.next());
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 1, walker1.next());
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 2, walker1.next());
      assertFalse(walker1.hasNext());

      final ArrayWalker walker2 = new ArrayWalker(2, 0, "down");
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 2, walker2.next());
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 1, walker2.next());
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 0, walker2.next());
      assertFalse(walker2.hasNext());

    }
  }

  private static class TPeriod implements Comparable<TPeriod>
  {
    public int start;
    public int end;

    public TPeriod(final int start, final int end)
    {
      this.start = start;
      this.end = end;
    }

    @Override
    public int compareTo(final TPeriod other)
    {
      return Integer.compare(start, other.start);
    }

    @Override
    public boolean equals(final Object arg0)
    {
      if (arg0 instanceof TPeriod)
      {
        final TPeriod other = (TPeriod) arg0;
        return other.start == start && other.end == end;
      }
      else
      {
        return false;
      }
    }

    @Override
    public int hashCode()
    {
      return start * 10000 + end;
    }
<commit_msg>Pull back to Java 6 API<commit_after>package org.mwc.debrief.track_shift.zig_detector.target;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import junit.framework.TestCase;

import org.apache.commons.math3.stat.regression.SimpleRegression;
import org.eclipse.core.runtime.ILog;
import org.eclipse.core.runtime.ILogListener;
import org.eclipse.core.runtime.IStatus;
import org.mwc.debrief.track_shift.zig_detector.moving_average.TimeRestrictedMovingAverage;
import org.mwc.debrief.track_shift.zig_detector.ownship.LegOfData;
import org.osgi.framework.Bundle;

import flanagan.math.Minimisation;
import flanagan.math.MinimisationFunction;

public class ZigDetector
{
  private static class ArrayWalker
  {
    private final int _start;
    private final int _end;
    private int _current;
    private final int _step;

    private Boolean _lastAbove = null;
    private int _lastOpposite;
    private int _sameSideCount;
    private String _name;
    private double _lastError = 0d;

    public ArrayWalker(final int start, final int end, final String name)
    {
      _start = start;
      _end = end;
      _name = name;

      _current = start;
      _step = _end > _start ? 1 : -1;
    }

    @Override
    public String toString()
    {
      return _name;
    }

    final public boolean justStarted()
    {
      return _lastAbove == null;
    }

    final public void setLastOpposite(int i)
    {
      _lastOpposite = i;
      _lastError = 0d;
    }

    final public int getLastOpposite()
    {
      return _lastOpposite;
    }

    final public void setSameSideCount(int i)
    {
      _sameSideCount = i;
    }

    final public int getSameSideCount()
    {
      return _sameSideCount;
    }

    final public void setLastAbove(boolean thisAbove)
    {
      _lastAbove = thisAbove;
    }

    final public boolean getLastAbove()
    {
      return _lastAbove;
    }

    public boolean hasNext()
    {
      return _current != _end + _step;
    }

    public Integer next()
    {
      // check we haven't passed the end
      if (!hasNext())
      {
        throw new IllegalArgumentException(
            "Can't call next() once we've passed the end");
      }

      final Integer res = _current;
      _current += _step;
      return res;
    }

    public double getLastError()
    {
      return _lastError;
    }

    public void setLastError(double error2)
    {
      _lastError = error2;
    }
  }

  private static interface EventHappened
  {
    public void eventAt(long time, double score, double threshold);
  }

  static class FlanaganArctan implements MinimisationFunction
  {
    private static double calcForecast(final double[] params,
        final double elapsedSecs)
    {
      final double B = params[0];
      final double P = params[1];
      final double Q = params[2];
      final double dX = Math.cos(Math.toRadians(B)) + Q * elapsedSecs;
      final double dY = Math.sin(Math.toRadians(B)) + P * elapsedSecs;
      double res = Math.toDegrees(Math.atan2(dY, dX));
      if (res < 0)
      {
        res += 360;
      }
      return res;
    }

    final private List<Long> _times;

    final private List<Double> _bearings;

    public FlanaganArctan(final List<Long> beforeTimes,
        final List<Double> beforeBearings)
    {
      _times = beforeTimes;
      _bearings = beforeBearings;
    }

    // evaluation function
    @Override
    public double function(final double[] params)
    {
      double runningSum = 0;

      // ok, loop through the data
      for (int i = 0; i < _times.size(); i++)
      {
        final double thisForecast = calcForecast(params, _times.get(i));
        final double thisMeasured = _bearings.get(i);
        double thisError = thisForecast - thisMeasured;
        if (thisError > 180)
        {
          thisError -= 360;
        }
        else if (thisError < -180)
        {
          thisError += 360;
        }
        final double sqError = Math.pow(thisError, 2);

        runningSum += sqError;
      }
      // normalise by dividing by the number cuts - 3
      // (the -3 is relates to the number of variables being considered)
      final double mean = runningSum / (_times.size() - 3);

      return mean;

//       final double rms = Math.sqrt(mean);
//       return rms;
    }

  }

  static class FlanaganArctan_Legacy implements MinimisationFunction
  {
    private static double calcForecast(final double B, final double P,
        final double Q, final double elapsedSecs)
    {
      final double dX = Math.cos(Math.toRadians(B)) + Q * elapsedSecs;
      final double dY = Math.sin(Math.toRadians(B)) + P * elapsedSecs;
      return Math.toDegrees(Math.atan2(dY, dX));
    }

    final private Long[] _times;

    final private Double[] _bearings;

    public FlanaganArctan_Legacy(final List<Long> beforeTimes,
        final List<Double> beforeBearings)
    {
      _times = beforeTimes.toArray(new Long[]
      {});
      _bearings = beforeBearings.toArray(new Double[]
      {});
    }

    // evaluation function
    @Override
    public double function(final double[] point)
    {
      final double B = point[0];
      final double P = point[1];
      final double Q = point[2];

      double runningSum = 0;
      final Long firstTime = _times[0];

      // ok, loop through the data
      for (int i = 0; i < _times.length; i++)
      {
        final long elapsedMillis = _times[i] - firstTime;
        final double elapsedSecs = elapsedMillis / 1000d;
        final double thisForecast = calcForecast(B, P, Q, elapsedSecs);
        final double thisMeasured = _bearings[i];
        double thisError = thisForecast - thisMeasured;
        if (thisError > 180)
        {
          thisError -= 360;
        }
        else if (thisError < -180)
        {
          thisError += 360;
        }
        final double sqError = Math.pow(thisError, 2);
        runningSum += sqError;
      }
      final double mean = runningSum / _times.length;

      final double rms = Math.sqrt(mean);
      return rms;
    }

  }

  protected static class ScoredTime implements Comparable<ScoredTime>
  {
    private final long _time;
    private final double _score;

    public ScoredTime(final long time, final double score)
    {
      _time = time;
      _score = score;
    }

    @Override
    public int compareTo(final ScoredTime o)
    {
      final Long myTime = _time;
      final Long hisTime = o._time;
      return myTime.compareTo(hisTime);
    }
  }

  public static class TestMe extends TestCase
  {
    @SuppressWarnings("unused")
    private ILegStorer getLegStorer()
    {
      final ILegStorer legStorer = new ILegStorer()
      {

        @Override
        public void storeLeg(final String scenarioName, final long tStart,
            final long tEnd, final double rms)
        {
          // System.out.println("store it: " + new Date(tStart) + ", "
          // + new Date(tEnd));
        }
      };
      return legStorer;
    }

    @SuppressWarnings("unused")
    private ILog getLogger()
    {
      final ILog logger = new ILog()
      {

        @Override
        public void addLogListener(final ILogListener listener)
        {
          // not required, class just for testing
        }

        @Override
        public Bundle getBundle()
        {
          return null;
        }

        @Override
        public void log(final IStatus status)
        {
          // not required, class just for testing
        }

        @Override
        public void removeLogListener(final ILogListener listener)
        {
          // not required, class just for testing
        }
      };
      return logger;
    }

    @SuppressWarnings("unused")
    private IZigStorer getZigStorer()
    {
      final IZigStorer zigStorer = new IZigStorer()
      {

        @Override
        public void finish()
        {
          // TODO Auto-generated method stub

        }

        @Override
        public void storeZig(final String scenarioName, final long tStart,
            final long tEnd, final double rms)
        {
          // TODO Auto-generated method stub

        }
      };
      return zigStorer;
    }

    public void testCalcStart()
    {
      final List<Long> list = new ArrayList<Long>(Arrays.asList(new Long[]
      {0L, 40L, 100L, 140L, 180L, 220L, 260L, 280L}));
      assertEquals("correct time", 3, calculateNewStart(list, 1, 100));
      assertEquals("correct time", 5, calculateNewStart(list, 2, 100));
      assertEquals("correct time", 1, calculateNewStart(list, 0, 20));

      // and a reverse list
      Collections.reverse(list);
      assertEquals("correct time", 4, calculateNewStart(list, 1, 100));
      assertEquals("correct time", 5, calculateNewStart(list, 2, 100));
      assertEquals("correct time", 2, calculateNewStart(list, 0, 30));

    }

    public void testCleanValues()
    {
      List<Double> list = new ArrayList<Double>();
      list.add(355d);
      list.add(4d);
      list.add(2d);
      list.add(349d);
      list.add(2d);
      list = prepareBearings(list);

      assertEquals("correct length", 5, list.size());
      assertEquals("fixed bearings", 364d, list.get(1), 0.001);
      assertEquals("fixed bearings", 362d, list.get(2), 0.001);
      assertEquals("fixed bearings", 349d, list.get(3), 0.001);
      assertEquals("fixed bearings", 362d, list.get(4), 0.001);

      // and the other way around
      list.clear();

      list.add(55d);
      list.add(354d);
      list.add(339d);
      list.add(9d);
      list.add(2d);
      list = prepareBearings(list);

      assertEquals("correct length", 5, list.size());
      assertEquals("fixed bearings", -6d, list.get(1), 0.001);
      assertEquals("fixed bearings", -21d, list.get(2), 0.001);
      assertEquals("fixed bearings", 9d, list.get(3), 0.001);
      assertEquals("fixed bearings", 2d, list.get(4), 0.001);

    }

    public void testMultiSlice() throws ParseException
    {

      final double[] times =
          new double[]
          {0, 17.99999981, 33.99999978, 47.99999991, 62.00000003, 82.00000031,
              98.00000028, 109.9999999, 125.9999999, 140, 157.9999998, 172,
              186.0000001, 200.0000002, 210.0000001, 227.9999999, 243.9999998,
              258, 272.0000001, 294.0000002, 312, 332.0000003, 358.0000001,
              383.9999999, 410.0000003, 436.0000001, 472.0000003, 504.0000003,
              531.9999999, 563.9999998, 592.0000001, 624, 657.9999998,
              692.0000002, 727.9999998, 764.0000001, 810.0000001, 840.0000002,
              875.9999998, 907.9999998, 944, 982.0000001, 1020, 1056, 1092,
              1126, 1172, 1214, 1258, 1296, 1334, 1372, 1404, 1426, 1456, 1486,
              1514, 1540, 1562, 1590, 1624, 1652, 1678, 1710, 1744, 1774, 1808,
              1846, 1880, 1912, 1942, 1982, 2014, 2050, 2082, 2118, 2154, 2186,
              2220, 2246, 2276, 2298, 2324, 2354, 2386, 2414, 2446, 2476, 2510,
              2548, 2580, 2618, 2650, 2686, 2720, 2758, 2790, 2822, 2852, 2886,
              2922, 2952, 2980, 3012, 3038, 3066, 3090, 3114, 3140, 3172, 3202,
              3234, 3274, 3310, 3344, 3382, 3418, 3448, 3478, 3508, 3542, 3570,
              3606, 3642, 3676, 3710, 3746, 3778, 3808, 3846, 3884, 3922, 3954,
              3990, 4026, 4060, 4098, 4140, 4174, 4214, 4254, 4288, 4324, 4360,
              4398, 4440, 4484, 4512, 4542, 4574, 4610, 4644, 4668, 4690, 4716,
              4738, 4760, 4782, 4802, 4822, 4842, 4852, 4864, 4880, 4900, 4916,
              4932, 4952, 4972, 4992, 5014, 5040, 5068, 5090, 5112, 5138, 5158,
              5188, 5210, 5238, 5264, 5294, 5324, 5358, 5390, 5416, 5444, 5476,
              5514, 5556, 5592, 5634, 5680, 5720, 5756, 5800, 5842, 5882, 5918,
              5958, 6000, 6046, 6090, 6128, 6186, 6222, 6264, 6300, 6334, 6370,
              6408, 6448, 6488, 6534, 6576, 6620, 6664, 6712};
      final Double[] bearings =
          new Double[]
          {-147d, -143.6, -140.3, -137.3, -134d, -130d, -125.9, -122.6, -117.9,
              -114.9, -111.3, -107.3, -103.6, -99.6, -95.6, -91.3, -87.9,
              -84.2, -80.6, -76.9, -72.6, -68.9, -65.6, -60.6, -56.6, -52.9,
              -50.2, -47.2, -45.5, -43.9, -42.2, -40.2, -38.9, -37.9, -36.9,
              -36.5, -36.2, -35.5, -34.9, -34.2, -33.5, -32.9, -32.2, -31.9,
              -31.2, -30.9, -29.9, -28.5, -27.9, -27.2, -25.5, -24.9, -23.9,
              -23.9, -26.2, -28.5, -29.5, -31.5, -32.5, -31.5, -31.5, -31.2,
              -31.2, -30.9, -30.5, -29.9, -28.9, -28.2, -27.5, -27.9, -26.5,
              -25.5, -24.9, -24.2, -23.5, -22.9, -21.9, -20.9, -19.9, -19.5,
              -18.9, -18.2, -17.5, -16.8, -15.5, -15.2, -14.5, -13.8, -13.2,
              -12.2, -11.5, -10.2, -9.8, -8.5, -7.5, -6.5, -5.5, -4.8, -3.5,
              -3.2, -2.2, -1.5, -0.8, -0.2, 1.2, 1.5, 2.2, 4.2, 4.8, 5.8, 6.8,
              7.8, 8.8, 10.2, 10.5, 11.5, 12.5, 13.2, 14.5, 14.8, 16.5, 17.5,
              18.5, 19.9, 21.2, 21.9, 22.9, 23.2, 24.5, 25.5, 26.5, 26.9, 27.5,
              28.2, 28.5, 29.2, 29.5, 29.9, 30.2, 30.9, 31.5, 31.9, 32.5, 32.5,
              32.9, 32.9, 32.9, 32.9, 33.2, 33.2, 32.9, 31.9, 30.2, 27.5, 23.9,
              20.2, 14.8, 8.8, 3.2, -3.8, -9.5, -14.8, -19.5, -24.9, -31.2,
              -36.2, -40.2, -46.5, -51.5, -55.6, -60.6, -66.6, -72.9, -77.9,
              -82.9, -87.9, -91.9, -96.9, -101.6, -104.9, -107.9, -111.6,
              -114.9, -117.6, -120.3, -122.3, -123.3, -123.3, -123.3, -123.3,
              -122.3, -120.9, -118.9, -118.3, -116.9, -116.3, -115.3, -114.3,
              -113.3, -111.9, -110.9, -110.3, -109.3, -108.6, -107.6, -106.6,
              -105.6, -104.9, -104.3, -103.6, -102.3, -100.9, -99.9, -97.9,
              -96.6, -95.3, -92.9, -91.9};

      final Long[] normalTimes = new Long[times.length];
      final Long[] rawTimes = new Long[times.length];
      // final java.text.DateFormat sdf = new SimpleDateFormat("HHmmss");
      long startTime = 0;
      for (int i = 0; i < times.length; i++)
      {
        final long thisTime = (long) times[i];

        rawTimes[i] = thisTime;

        if (i == 0)
        {
          startTime = thisTime;
        }

        normalTimes[i] = (thisTime - startTime);
      }

      // start to collate the adta
      final List<Long> rawList1 = Arrays.asList(rawTimes);

      final List<Double> tBearings1 = Arrays.asList(bearings);

      // System.out
      // .println("last time:" + new Date(tList1.get(tList1.size() - 1)));

      // get the last 40 elements
      // final int start = tList1.size() - 50;
      // final int end = tList1.size() - 1;
      //
      // List<Long> tList = tList1.subList(start, end);
      // List<Double> tBearings = tBearings1.subList(start, end);

      final List<Double> tBearings = tBearings1;

      // final TPeriod flattest = findLowestRateIn(tList, tBearings);
      // assertNotNull("found data", flattest);
      // assertEquals(0, flattest.start);
      // assertEquals(6, flattest.end);

      // System.out.println("from:" + new Date(times[0]) + " // to:"
      // + new Date(times[times.length - 1]) + " // " + times.length + " entries");

      final ZigDetector detector = new ZigDetector();
      double zigRatio = 1000d;
      final double optimiseTolerance = 0.0000000004;

      // ILog logger = getLogger();
      // ILegStorer legStorer = getLegStorer();
      // IZigStorer zigStorer = getZigStorer();
      // detector.sliceThis(logger, "some name", "scenario", times[0],
      // times[times.length - 1], legStorer, zigStorer, zigRatio,
      // optimiseTolerance, tList, tBearings);

      // reverse the arrays
      // Collections.reverse(tList);
      // Collections.reverse(tBearings);
      //
      final long timeWindow = 240000;
      zigRatio = 15d;
      final EventHappened happened = new EventHappened()
      {
        @Override
        public void eventAt(final long time, final double score,
            final double threshold)
        {
          // System.out.println("event at " + new Date(time) + " score:" + score);
        }
      };

      final List<TPeriod> legs = new ArrayList<TPeriod>();

      detector.runThrough2(optimiseTolerance, rawList1, tBearings, happened,
          zigRatio, timeWindow, legs);

      listSlices(legs, rawList1);

      listSlicesForPlotting(legs, rawList1);

      // final Double[] bearings =
      // new Double[]
      // {180d, 180.3, 180.7, 181d, 181.4, 181.7, 182.1, 182.5, 182.8, 183.2,
      // 183.6, 184.1, 184.5, 184.9, 185.3, 185.8, 186.3, 186.7, 187.2,
      // 187.7, 188.2, 188.8, 189.3, 189.8, 190.4, 191d, 191.6, 192.2,
      // 192.8, 193.4, 194.1, 194.8, 195.5, 196.2, 196.9, 197.6, 198.4,
      // 199.2, 200d, 200.8, 201.7, 202.6, 203.4, 204.4, 205.3, 206.1,
      // 206.7, 207.3, 207.9, 208.5, 209.2, 209.9, 210.6, 211.4, 212.2,
      // 213.1, 214d, 214.9, 215.9, 216.9, 218d, 219.1, 220.3, 221.6,
      // 223d, 224.4, 225.9, 227.4, 229.1, 230.8, 232.7, 234.6, 236.6,
      // 238.8, 241d, 243.4, 245.9, 248.2, 250.3, 252.3, 254.3, 256.1,
      // 257.9, 259.6, 261.2, 262.8, 264.3, 265.7, 267.1, 268.4, 269.7,
      // 270.9, 272d, 273.1, 274.2, 275.2, 276.2, 277.1, 278d, 278.8,
      // 279.7, 280.4, 281.2};
      //
      // final int[] timeStr =
      // new int[]
      // {120000, 120050, 120140, 120230, 120320, 120410, 120500, 120550,
      // 120640, 120730, 120820, 120910, 121000, 121050, 121140, 121230,
      // 121320, 121410, 121500, 121550, 121640, 121730, 121820, 121910,
      // 122000, 122050, 122140, 122230, 122320, 122410, 122500, 122550,
      // 122640, 122730, 122820, 122910, 123000, 123050, 123140, 123230,
      // 123320, 123410, 123500, 123550, 123640, 123730, 123820, 123910,
      // 124000, 124050, 124140, 124230, 124320, 124410, 124500, 124550,
      // 124640, 124730, 124820, 124910, 125000, 125050, 125140, 125230,
      // 125320, 125410, 125500, 125550, 125640, 125730, 125820, 125910,
      // 130000, 130050, 130140, 130230, 130320, 130410, 130500, 130550,
      // 130640, 130730, 130820, 130910, 131000, 131050, 131140, 131230,
      // 131320, 131410, 131500, 131550, 131640, 131730, 131820, 131910,
      // 132000, 132050, 132140, 132230, 132320, 132410, 132500};

      // Long[] times =
      // new Long[]
      // {1248237792000L, 1248237799000L, 1248237896000L, 1248237944000L,
      // 1248237990000L, 1248238098000L, 1248238177000L, 1248238249000L,
      // 1248238321000L, 1248238393000L, 1248238484000L, 1248238556000L,
      // 1248238624000L, 1248238695000L, 1248238759000L, 1248238843000L,
      // 1248238931000L, 1248239006000L, 1248239074000L, 1248239162000L,
      // 1248239277000L, 1248239353000L, 1248239444000L, 1248239520000L,
      // 1248239600000L, 1248239644000L, 1248239735000L, 1248239799000L,
      // 1248239891000L, 1248239951000L, 1248240030000L, 1248240090000L,
      // 1248240142000L, 1248240198000L, 1248240257000L, 1248240305000L};
      // Double[] bearings =
      // new Double[]
      // {295.8, 295.5, 293.5, 293.0, 292.8, 290.3, 289.0, 288.3, 288.0,
      // 288.0, 288.8, 288.8, 288.8, 289.8, 289.8, 291.0, 291.5, 292.3,
      // 292.3, 293.0, 293.5, 294.0, 294.3, 294.8, 294.8, 294.8, 295.8,
      // 295.8, 295.8, 296.5, 296.5, 297.5, 297.8, 298.3, 299.0, 299.5};

      // Long[] times2 =
      // new Long[]
      // {946697610000L, 946697640000L, 946697670000L, 946697700000L,
      // 946697730000L, 946697760000L, 946697790000L, 946697820000L,
      // 946697850000L, 946697880000L, 946697910000L, 946697940000L,
      // 946697970000L, 946698000000L, 946698030000L, 946698060000L,
      // 946698090000L, 946698120000L, 946698150000L, 946698180000L,
      // 946698210000L, 946698240000L, 946698270000L, 946698300000L,
      // 946698330000L, 946698360000L, 946698390000L, 946698420000L,
      // 946698450000L, 946698480000L, 946698510000L, 946698540000L,
      // 946698570000L, 946698600000L, 946698630000L, 946698660000L,
      // 946698690000L, 946698720000L, 946698750000L, 946698780000L,
      // 946698810000L, 946698840000L, 946698870000L, 946698900000L,
      // 946698930000L, 946698960000L, 946698990000L, 946699020000L,
      // 946699050000L, 946699080000L, 946699110000L, 946699140000L,
      // 946699170000L, 946699200000L, 946699230000L, 946699260000L,
      // 946699290000L, 946699320000L, 946699350000L, 946699380000L,
      // 946699410000L, 946699440000L, 946699470000L, 946699500000L,
      // 946699530000L, 946699560000L, 946699590000L, 946699620000L,
      // 946699650000L, 946699680000L, 946699710000L, 946699740000L,
      // 946699770000L, 946699800000L, 946699830000L, 946699860000L,
      // 946699890000L, 946699920000L, 946699950000L, 946699980000L,
      // 946700010000L, 946700040000L, 946700070000L, 946700100000L,
      // 946700130000L, 946700160000L, 946700190000L, 946700220000L,
      // 946700250000L, 946700280000L, 946700310000L, 946700340000L,
      // 946700370000L, 946700400000L, 946700430000L, 946700460000L,
      // 946700490000L, 946700520000L, 946700550000L, 946700580000L,
      // 946700610000L, 946700640000L, 946700670000L, 946700700000L,
      // 946700730000L, 946700760000L, 946700790000L, 946700820000L,
      // 946700850000L, 946700880000L, 946700910000L, 946700940000L,
      // 946700970000L};
      //
      // Double[] bearings2 =
      // new Double[]
      // {170.095, 170.566, 171.404, 172.021, 172.757, 173.25, 173.767,
      // 174.391, 174.958, 175.839, 176.485, 177.282, 177.66, 178.444,
      // 179.09, 179.671, -179.482, -178.846, -178.363, -177.853,
      // -177.173, -175.994, -175.115, -174.628, -174.019, -173.208,
      // -172.378, -171.79, -170.932, -170.251, -169.526, -168.751,
      // -168.123, -167.354, -166.331, -165.639, -164.767, -164.272,
      // -163.407, -162.441, -161.783, -161.074, -159.886, -158.873,
      // -158.367, -157.495, -156.606, -155.92, -154.829, -153.856,
      // -152.983, -152.355, -151.561, -151.01, -149.65, -149.143,
      // -148.211, -147.211, -146.283, -145.55, -145.102, -144.119,
      // -143.22, -143.185, -141.704, -140.562, -139.975, -139.124,
      // -138.346, -137.36, -137.276, -135.746, -135.333, -134.338,
      // -133.295, -132.577, -131.86, -131.143, -130.278, -129.278,
      // -128.344, -127.83199999999998, -127.107, -126.345,
      // -125.40799999999999, -124.49999999999999, -123.88299999999998,
      // -123.195, -122.52800000000002, -122.17599999999999, -121.21,
      // -120.267, -120.11499999999998, -119.31799999999998, -118.507,
      // -117.99500000000002, -117.689, -117.71000000000001, -117.36,
      // -117.09399999999998, -117.23799999999999, -117.01, -116.633,
      // -116.74, -116.40300000000002, -116.296, -116.158,
      // -115.85400000000001, -115.82, -115.777, -115.56000000000002,
      // -115.071, -114.71999999999998};
      //

      // Long[] times =
      // new Long[]
      // {946699110000L, 946699140000L, 946699170000L, 946699200000L,
      // 946699230000L, 946699260000L, 946699290000L, 946699320000L,
      // 946699350000L, 946699380000L, 946699410000L, 946699440000L,
      // 946699470000L, 946699500000L, 946699530000L, 946699560000L,
      // 946699590000L, 946699620000L, 946699650000L, 946699680000L,
      // 946699710000L, 946699740000L, 946699770000L, 946699800000L,
      // 946699830000L, 946699860000L, 946699890000L, 946699920000L,
      // 946699950000L, 946699980000L, 946700010000L, 946700040000L,
      // 946700070000L, 946700100000L, 946700130000L, 946700160000L,
      // 946700190000L, 946700220000L, 946700250000L, 946700280000L,
      // 946700310000L, 946700340000L, 946700370000L, 946700400000L,
      // 946700430000L, 946700460000L, 946700490000L, 946700520000L,
      // 946700550000L, 946700580000L, 946700610000L, 946700640000L,
      // 946700670000L, 946700700000L, 946700730000L, 946700760000L,
      // 946700790000L, 946700820000L, 946700850000L, 946700880000L,
      // 946700910000L, 946700940000L, 946700970000L};
      // Double[] bearings =
      // new Double[]
      // {207.017, 207.645, 208.439, 208.99, 210.35, 210.857, 211.789,
      // 212.789, 213.717, 214.45, 214.898, 215.881, 216.78, 216.815,
      // 218.296, 219.438, 220.025, 220.876, 221.654, 222.64, 222.724,
      // 224.254, 224.667, 225.662, 226.705, 227.423, 228.14, 228.857,
      // 229.722, 230.722, 231.656, 232.168, 232.893, 233.655, 234.592,
      // 235.5, 236.11700000000002, 236.805, 237.47199999999998, 237.824,
      // 238.79000000000002, 239.733, 239.88500000000002,
      // 240.68200000000002, 241.493, 242.005, 242.311, 242.29, 242.64,
      // 242.906, 242.762, 242.99, 243.36700000000002, 243.26,
      // 243.59699999999998, 243.704, 243.84199999999998, 244.146, 244.18,
      // 244.223, 244.44, 244.929, 245.28000000000003};

      // Long[] times = new Long[]{946699110000L, 946699140000L, 946699170000L, 946699200000L,
      // 946699230000L, 946699260000L, 946699290000L, 946699320000L, 946699350000L, 946699380000L,
      // 946699410000L, 946699440000L, 946699470000L, 946699500000L, 946699530000L, 946699560000L,
      // 946699590000L, 946699620000L, 946699650000L, 946699680000L, 946699710000L, 946699740000L,
      // 946699770000L, 946699800000L, 946699830000L, 946699860000L, 946699890000L, 946699920000L,
      // 946699950000L, 946699980000L, 946700010000L, 946700040000L, 946700070000L, 946700100000L,
      // 946700130000L, 946700160000L, 946700190000L, 946700220000L, 946700250000L, 946700280000L,
      // 946700310000L, 946700340000L, 946700370000L, 946700400000L, 946700430000L, 946700460000L,
      // 946700490000L, 946700520000L, 946700550000L, 946700580000L};
      // Double[] bearings = new Double[]{-152.983, -152.355, -151.561, -151.01, -149.65, -149.143,
      // -148.211, -147.211, -146.283, -145.55, -145.102, -144.119, -143.22, -143.185, -141.704,
      // -140.562, -139.975, -139.124, -138.346, -137.36, -137.276, -135.746, -135.333, -134.338,
      // -133.295, -132.577, -131.86, -131.143, -130.278, -129.278, -128.344, -127.83199999999998,
      // -127.107, -126.345, -125.40799999999999, -124.49999999999999, -123.88299999999998,
      // -123.195, -122.52800000000002, -122.17599999999999, -121.21, -120.267, -120.11499999999998,
      // -119.31799999999998, -118.507, -117.99500000000002, -117.689, -117.71000000000001, -117.36,
      // -117.09399999999998};

    }

    public void testSlicing()
    {
      final List<TPeriod> q = new ArrayList<TPeriod>();
      final List<TPeriod> legs = new ArrayList<TPeriod>();
      final TPeriod outer = new TPeriod(0, 100);
      q.add(outer);
      TPeriod leg = new TPeriod(20, 30);

      assertEquals("one item", 1, q.size());
      assertTrue("correct item", q.contains(outer));

      handleNewSlices(q, legs, outer, leg, 5, true);

      assertEquals("two items", 2, q.size());
      assertEquals("first item", q.get(0), new TPeriod(0, 19));
      assertEquals("second item", q.get(1), new TPeriod(31, 100));
      assertEquals("correct legs", 1, legs.size());

      leg = new TPeriod(30, 40);
      handleNewSlices(q, legs, q.get(1), leg, 5, true);

      assertEquals("three items", 3, q.size());

      assertEquals("first item", new TPeriod(0, 19), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(72, 100), q.get(2));
      assertEquals("correct legs", 2, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 10);

      handleNewSlices(q, legs, q.get(2), leg, 5, true);
      assertEquals("three items", 3, q.size());

      assertEquals("first item", new TPeriod(0, 19), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(83, 100), q.get(2));
      assertEquals("correct legs", 3, legs.size());

      // ok, leave a short one
      leg = new TPeriod(13, 16);

      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("three items", 3, q.size());

      assertEquals("second item", new TPeriod(0, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("third item", new TPeriod(83, 100), q.get(2));
      assertEquals("correct legs", 4, legs.size());

      // ok, leave a short one
      leg = new TPeriod(4, 14);

      handleNewSlices(q, legs, q.get(2), leg, 5, true);
      assertEquals("two items", 2, q.size());

      assertEquals("second item", new TPeriod(0, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("correct legs", 5, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 6);
      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("two items", 2, q.size());

      assertEquals("second item", new TPeriod(7, 12), q.get(0));
      assertEquals("second item", new TPeriod(31, 60), q.get(1));
      assertEquals("correct legs", 6, legs.size());

      // ok, leave a short one
      leg = new TPeriod(2, 6);
      handleNewSlices(q, legs, q.get(0), leg, 5, true);
      assertEquals("two items", 1, q.size());

      assertEquals("second item", new TPeriod(31, 60), q.get(0));
      assertEquals("correct legs", 7, legs.size());

      listSlices(legs, null);

    }

    public void testStartTimes()
    {
      assertEquals(2, 5 / 2);

      final List<Long> times =
          new ArrayList<Long>(Arrays.asList(new Long[]
          {1000L, 1200L, 1500L, 1800L, 2100L, 2400L, 2700L, 3000L, 3300L,
              3600L, 3900L}));

      assertEquals("correct", -1, getEnd(0, times, 200, 0));
      assertEquals("correct", -1, getEnd(0, times, 200, 1));
      assertEquals("correct", -1, getEnd(0, times, 200, 2));
      assertEquals("correct", 2, getEnd(0, times, 200, 3));
      assertEquals("correct", 3, getEnd(0, times, 200, 4));
      assertEquals("correct", 4, getEnd(0, times, 200, 5));
      assertEquals("correct", 5, getEnd(0, times, 200, 6));
      assertEquals("correct", 6, getEnd(0, times, 200, 7));
      assertEquals("correct", 7, getEnd(0, times, 200, 8));
      assertEquals("correct", 8, getEnd(0, times, 200, 9));
      assertEquals("correct", 9, getEnd(0, times, 200, 10));

      assertEquals("correct", 2, getStart(0, times, 400, 0));
      assertEquals("correct", 2, getStart(0, times, 400, 1));
      assertEquals("correct", 3, getStart(0, times, 400, 2));
      assertEquals("correct", 4, getStart(0, times, 400, 3));
      assertEquals("correct", 5, getStart(0, times, 400, 4));
      assertEquals("correct", 6, getStart(0, times, 400, 5));
      assertEquals("correct", 7, getStart(0, times, 400, 6));
      assertEquals("correct", 8, getStart(0, times, 400, 7));
      assertEquals("correct", -1, getStart(0, times, 400, 8));
      assertEquals("correct", -1, getStart(0, times, 400, 9));
      assertEquals("correct", -1, getStart(0, times, 400, 10));
    }

    public void testWalker()
    {
      final ArrayWalker walker1 = new ArrayWalker(0, 2, "up");
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 0, walker1.next());
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 1, walker1.next());
      assertTrue(walker1.hasNext());
      assertEquals((Integer) 2, walker1.next());
      assertFalse(walker1.hasNext());

      final ArrayWalker walker2 = new ArrayWalker(2, 0, "down");
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 2, walker2.next());
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 1, walker2.next());
      assertTrue(walker2.hasNext());
      assertEquals((Integer) 0, walker2.next());
      assertFalse(walker2.hasNext());

    }
  }

  private static class TPeriod implements Comparable<TPeriod>
  {
    public int start;
    public int end;

    public TPeriod(final int start, final int end)
    {
      this.start = start;
      this.end = end;
    }

    @Override
    public int compareTo(final TPeriod other)
    {
      return new Integer(start).compareTo(other.start);
    }

    @Override
    public boolean equals(final Object arg0)
    {
      if (arg0 instanceof TPeriod)
      {
        final TPeriod other = (TPeriod) arg0;
        return other.start == start && other.end == end;
      }
      else
      {
        return false;
      }
    }

    @Override
    public int hashCode()
    {
      return start * 10000 + end;
    }

{'fraction_non_alphanumeric': 0.148875539125077, 'fraction_numerical': 0.2584257547751078, 'mean_word_length': 2.3865936358894104, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 88, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5330585', 'n_tokens_mistral': 35816, 'n_tokens_neox': 23885, 'n_words': 5547}