<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package ch.unizh.ini.jaer.projects.minliu;

import ch.unizh.ini.jaer.projects.davis.calibration.SingleCameraCalibration;
import java.awt.FlowLayout;
import java.awt.HeadlessException;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;

import static java.nio.file.Files.list;
import static java.rmi.Naming.list;

import java.util.ArrayList;
import static java.util.Collections.list;
import java.util.Random;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import java.util.List;
import java.util.Observable;
import java.util.Observer;

import org.opencv.core.Core; 
import org.opencv.core.CvType; 
import org.opencv.core.Mat; 
import org.opencv.core.MatOfByte; 
import org.opencv.core.MatOfFloat; 
import org.opencv.core.MatOfPoint; 
import org.opencv.core.TermCriteria;
import org.opencv.core.MatOfPoint2f; 
import org.opencv.core.Point; 
import org.opencv.core.Rect; 
import org.opencv.core.RotatedRect; 
import org.opencv.core.Scalar; 
import org.opencv.core.Size; 
import org.opencv.video.Video;
import org.opencv.videoio.VideoCapture;
import org.bytedeco.javacpp.opencv_videoio.VideoWriter;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.opencv.calib3d.Calib3d;

import ch.unizh.ini.jaer.projects.davis.frames.ApsFrameExtractor;
import ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlow;
import ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.v;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.vx;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.vy;
import com.jogamp.common.util.Bitstream.ByteStream;
import com.jogamp.opengl.GL;
import com.jogamp.opengl.GL2;
import com.jogamp.opengl.GLAutoDrawable;
import com.jogamp.opengl.GLException;
import com.jogamp.opengl.util.gl2.GLUT;
import java.awt.BorderLayout;
import java.awt.Dimension;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.swing.JFileChooser;

import net.sf.jaer.Description;
import net.sf.jaer.DevelopmentStatus;
import net.sf.jaer.chip.AEChip;
import net.sf.jaer.chip.Chip2D;
import net.sf.jaer.event.ApsDvsEvent;
import net.sf.jaer.event.ApsDvsEventPacket;
import net.sf.jaer.event.EventPacket;
import net.sf.jaer.event.PolarityEvent;
import net.sf.jaer.event.orientation.ApsDvsMotionOrientationEvent;
import net.sf.jaer.eventprocessing.EventFilter;
import static net.sf.jaer.eventprocessing.EventFilter.log;
import net.sf.jaer.eventprocessing.EventFilter2D;
import net.sf.jaer.eventprocessing.FilterChain;
import net.sf.jaer.graphics.AEFrameChipRenderer;
import net.sf.jaer.graphics.ImageDisplay;
import net.sf.jaer.util.TobiLogger;
import org.apache.commons.lang3.ArrayUtils;


/**
 *
 * @author minliu
 */
@Description("Optical Flow methods based on OpenCV")
@DevelopmentStatus(DevelopmentStatus.Status.Experimental)
public class OpenCVFlow extends AbstractMotionFlow 
                        implements PropertyChangeListener, Observer /* Observer needed to get change events on chip construction */ {

    static { 
    String jvmVersion = System.getProperty("sun.arch.data.model");
        
    try {
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    // System.loadLibrary("opencv_ffmpeg320_" + jvmVersion);   // Notice, cannot put the file type extension (.dll) here, it will add it automatically. 
    } catch (UnsatisfiedLinkError e) {
        System.err.println("Native code library failed to load.\n" + e);
        // System.exit(1);
        }
    }
    
    private final ImageDisplay OFResultDisplay;
    
    private JFrame OFResultFrame = null;
    protected boolean showAPSFrameDisplay = getBoolean("showAPSFrameDisplay", true);
    private int[][] color = new int[100][3];
    private float[] oldBuffer = null, newBuffer = null;
    private PatchMatchFlow patchFlow;
    private TobiLogger globalMotionVectorLogger;
    Mat RPos, tPos;
    private ApsFrameExtractor apsFrameExtractor;
    private int lastApsTS, currentApsTS;
    
    public OpenCVFlow(AEChip chip) {
        super(chip);
        System.out.println("Welcome to OpenCV " + Core.VERSION);

        OFResultDisplay = ImageDisplay.createOpenGLCanvas();
        
        OFResultFrame = new JFrame("Optical Flow Result Frame");
        OFResultFrame.setPreferredSize(new Dimension(800, 800));
        OFResultFrame.getContentPane().add(OFResultDisplay, BorderLayout.CENTER);
        OFResultFrame.pack();
        OFResultFrame.addWindowListener(new WindowAdapter() {
            @Override
            public void windowClosing(final WindowEvent e) {
                setShowAPSFrameDisplay(false);
            }
        }); 
        
        FilterChain chain = new FilterChain(chip);        
        try {
            patchFlow = new PatchMatchFlow(chip);
            patchFlow.imuFlowEstimator = this.imuFlowEstimator;
            patchFlow.setFilterEnabled(true);
            chain.add(patchFlow);
        } catch (Exception e) {
            log.warning("could not setup PatchMatchFlow fiter.");
        }        
        apsFrameExtractor = new ApsFrameExtractor(chip);
        chain.add(apsFrameExtractor);
        setEnclosedFilterChain(chain);

        tPos = Mat.zeros(3, 1, CvType.CV_32F);
        RPos = Mat.eye(3, 3, CvType.CV_32F);
        apsFrameExtractor.getSupport().addPropertyChangeListener(ApsFrameExtractor.EVENT_NEW_FRAME, this);   
        patchFlow.getSupport().addPropertyChangeListener(PatchMatchFlow.EVENT_NEW_SLICES,this);
        chip.addObserver(this); // to allocate memory once chip size is known
    }

    @Override
    public EventPacket filterPacket(EventPacket in) {   
        if(!isFilterEnabled()) {
                return in;
        }
        setupFilter(in);
        
        if (showAPSFrameDisplay && !OFResultFrame.isVisible()) {
            OFResultFrame.setVisible(true);
        }
        Iterator i = null;
        if (in instanceof ApsDvsEventPacket) {
            i = ((ApsDvsEventPacket) in).fullIterator();
        } else {
            i = ((EventPacket) in).inputIterator();
        }

        while (i.hasNext()) {
            Object o = i.next();
            if (o == null) {
                log.warning("null event passed in, returning input packet");
                return in;
            }
            if ((o instanceof ApsDvsEvent) && ((ApsDvsEvent) o).isApsData()) {
                continue;
            }
            PolarityEvent ein = (PolarityEvent) o;
            if (!extractEventInfo(o)) {
                continue;
            }
            if (measureAccuracy || discardOutliersForStatisticalMeasurementEnabled) {
                if (imuFlowEstimator.calculateImuFlow(o)) {
                    continue;
                }
            }
            if (xyFilter()) {
                continue;
            }
            countIn++;
        }
        
        /* This part is to start logging global motion vector according to user's configurations automatically. */
        int timeMarker = patchFlow.ts - patchFlow.getChip().getAeInputStream().getFirstTimestamp();
        int startTime = 500000, endTime = 6000000;
        if ( timeMarker >= startTime && timeMarker <= endTime && globalMotionVectorLogger == null) {                
            globalMotionVectorLogger = new TobiLogger(patchFlow.getChip().getAeInputStream().getFile().getName() + patchFlow.getSliceMethod() + patchFlow.getSliceDurationUs() , 
                                                        "Global Motion vector for every generated slice");
            globalMotionVectorLogger.setNanotimeEnabled(false);
            globalMotionVectorLogger.setHeaderLine("system_time(ms) relative_timestamp(us) sliceDeltaT(us) method globalVx(pps) globalVy(pps) globalRotation(degree/s) samples");
            globalMotionVectorLogger.setEnabled(true);
        } 

        if ( timeMarker >= endTime && globalMotionVectorLogger != null) {
            doStopLoggingGlobalMotion();
        } 
        
        OFResultDisplay.checkPixmapAllocation();       
        
//        final ApsDvsEventPacket packet = (ApsDvsEventPacket) in;
//        if (packet == null) {
//            return null;
//        }
//        if (packet.getEventClass() != ApsDvsEvent.class) {
//            EventFilter.log.warning("wrong input event class, got " + packet.getEventClass() + " but we need to have " + ApsDvsEvent.class);
//            return null;
//        }
//        final Iterator apsItr = packet.fullIterator();
//        while (apsItr.hasNext()) {
//            final ApsDvsEvent e = (ApsDvsEvent) apsItr.next();
//            if (e.isApsData()) {
//                apsFrameExtractor.putAPSevent(e);
//            }
//        }
        if(isShowAPSFrameDisplay()) {
            OFResultDisplay.repaint();            
        }
        // motionFlowStatistics.updatePacket(countIn, countOut);
        return isDisplayRawInput() ? in : dirPacket;
    }

    @Override
    public synchronized void resetFilter() {
        super.resetFilter();

//        if(patchFlow != null) {
//            patchFlow.resetFilter();            
//        }
        tPos = Mat.zeros(3, 1, CvType.CV_32F);
        RPos = Mat.eye(3, 3, CvType.CV_32F);
        if (OFResultDisplay != null) {
            OFResultDisplay.setImageSize(chip.getSizeX(), chip.getSizeY());            
        }
    }

    @Override
    public void initFilter() {
        resetFilter();
    }

    @Override
    public void update(Observable o, Object arg) {
        if ((o != null) && (arg != null)) {
            if ((o instanceof AEChip) && (arg.equals(Chip2D.EVENT_SIZEX) || arg.equals(Chip2D.EVENT_SIZEY))) {
                initFilter();
            }
        }    
    }
    
    @Override
    synchronized public void annotate(GLAutoDrawable drawable) {
        super.annotate(drawable);
        GL2 gl = drawable.getGL().getGL2();
        try {
            gl.glEnable(GL.GL_BLEND);
            gl.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_ONE_MINUS_SRC_ALPHA);
            gl.glBlendEquation(GL.GL_FUNC_ADD);
        } catch (GLException e) {
            e.printStackTrace();
        }   
        
        if (measureAccuracy) {
            gl.glPushMatrix();
            final int offset = -10;
            gl.glRasterPos2i(chip.getSizeX() / 2, -5);
            chip.getCanvas().getGlut().glutBitmapString(GLUT.BITMAP_HELVETICA_18,
                    motionFlowStatistics.endpointErrorAbs.graphicsString("OpenLK_AEE(abs):", "pps"));
            gl.glPopMatrix();
        }
    }
    
    @Override
    public synchronized void setFilterEnabled(final boolean yes) {
        super.setFilterEnabled(yes); // To change body of generated methods, choose Tools | Templates.
        if (!isFilterEnabled()) {
            if (OFResultFrame != null) {
                OFResultFrame.setVisible(false);
            }
        }
    } 
    
    @Override
    public synchronized void propertyChange(PropertyChangeEvent evt) {
        super.propertyChange(evt); // resets filter on rewind, etc
        if (evt.getPropertyName().equals(ApsFrameExtractor.EVENT_NEW_FRAME)) {
            if(newBuffer == null) {
                newBuffer = ((float[]) evt.getNewValue()).clone();
                for ( int i = 0; i < newBuffer.length; i++ ) {
                    newBuffer[i] = newBuffer[i] * 255;
                }
                lastApsTS = apsFrameExtractor.getLastFrameTimestamp();
                return;
            }
            lastApsTS = currentApsTS;
            currentApsTS = apsFrameExtractor.getLastFrameTimestamp();
            oldBuffer = new float[newBuffer.length];
            oldBuffer = newBuffer.clone();
            float[] buffer = OFResultDisplay.getPixmapArray();
            byte[] byteBuffer = new byte[buffer.length];
            for(int i=0; i<buffer.length;i++) {
                byteBuffer[i] = (byte) (buffer[i] * 255);
            }
            newBuffer = ((float[]) evt.getNewValue()).clone();
            for ( int i = 0; i < newBuffer.length; i++ ) {
                    newBuffer[i] = newBuffer[i] * 255;
            }            
            Mat newFrame = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_32F);
            newFrame.put(0, 0, newBuffer);   
            Mat oldFrame = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_32F);
            oldFrame.put(0, 0, oldBuffer); 
            
            Mat newFrameTmp = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_8U);
            Mat oldFrameTmp = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_8U);
            newFrame.convertTo(newFrameTmp, CvType.CV_8U);
            oldFrame.convertTo(oldFrameTmp, CvType.CV_8U);

            // params for ShiTomasi corner detection            
            FeatureParams feature_params  = new FeatureParams(100, 0.3, 7, 7);
            
            // Feature extraction
            MatOfPoint p0 = new MatOfPoint();
            Imgproc.goodFeaturesToTrack(newFrameTmp, p0, feature_params.maxCorners, feature_params.qualityLevel, feature_params.minDistance);       

            MatOfPoint2f prevPts = new MatOfPoint2f(p0.toArray());
            MatOfPoint2f nextPts = new MatOfPoint2f();
            MatOfByte status = new MatOfByte();
            MatOfFloat err = new MatOfFloat();

            int featureNum = prevPts.checkVector(2, CvType.CV_32F, true);
            System.out.println("The number of feature detected is : " + featureNum);     

            // No features are detected
            if (featureNum == -1) {
                return;
            }
            try {
                Video.calcOpticalFlowPyrLK(oldFrameTmp, newFrameTmp, prevPts, nextPts, status, err);            
            } catch (Exception e) {
                System.err.println(e);
                // newFrame.copyTo(oldFrame);
            }            
            
            // TODO: Select good points 

            // draw the tracks
            Point[] prevPoints = prevPts.toArray();
            Point[] nextPoints = nextPts.toArray();
            byte[] st = status.toArray();
            float[] er = err.toArray();    
            Mat mask = new Mat(newFrame.rows(), newFrame.cols(), CvType.CV_32F);
            for (int i = 0; i < prevPoints.length; i++) {
                Imgproc.arrowedLine(newFrame, prevPoints[i], nextPoints[i], new Scalar(color[i][0],color[i][1],color[i][2]), 5, 8, 0, 0.1);  
                Imgproc.circle(newFrame,prevPoints[i], 1, new Scalar(255,255,255),-1);
            }
            
            // Select good points and copy them for output
            int index = 0;
            for(byte stTmp: st) {
                if(stTmp == 1) {
                    // if (outlierFlg.length >0 ) {
                        // if (outlierFlg[index] == 1) {
                            e = new PolarityEvent();
                            x = (short)(prevPoints[index].x);
                            y = (short)prevPoints[index].y;
                            e.x = (short)x;
                            e.y = (short)y;    // e, x and y all of them are used in processGoodEvent();
        //                    e.timestamp = ts;
                            vx = (float)(nextPoints[index].x - prevPoints[index].x) * 1000000 / (currentApsTS - lastApsTS);
                            vy = (float)(nextPoints[index].y - prevPoints[index].y) * 1000000 / (currentApsTS - lastApsTS);
                            v = (float) Math.sqrt(vx * vx + vy * vy);
                            processGoodEvent();                      
                        // exportFlowToMatlab(false);                            
                        // }
                    // }

                }
                index++;                
            }
            motionFlowStatistics.updatePacket(countIn, countOut, lastApsTS);
            float[] return_buff = new float[(int) (newFrame.total() * 
                                            newFrame.channels())];
            newFrame.get(0, 0, return_buff);
            for ( int i = 0; i < return_buff.length; i++ ) {
                    return_buff[i] = return_buff[i]/255.0f;
            }      
            OFResultDisplay.setPixmapFromGrayArray(return_buff);  
            
            DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");
            File folder = new File("EventSlices/" + chip.getAeInputStream().getFile().getName() + "_" + df);
            // if the directory does not exist, create it
            if (!folder.exists()) {
                folder.mkdir();
            }            
            File outputfile = new File(folder, String.format("packet_pid_%d.jpg", lastApsTS));
            Core.flip(newFrameTmp, newFrameTmp, 0);
            // Uncomment this when you want to store the slice's pictures.
//            try {
//                ImageIO.write(Mat2BufferedImage(newFrameTmp), "jpg", outputfile);
//            } catch (IOException ex) {
//                Logger.getLogger(OpenCVFlow.class.getName()).log(Level.SEVERE, null, ex);
//            }<commit_msg>Added color output for OF vectors in the image display.<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package ch.unizh.ini.jaer.projects.minliu;

import ch.unizh.ini.jaer.projects.davis.calibration.SingleCameraCalibration;
import java.awt.FlowLayout;
import java.awt.HeadlessException;
import java.awt.Image;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;

import static java.nio.file.Files.list;
import static java.rmi.Naming.list;

import java.util.ArrayList;
import static java.util.Collections.list;
import java.util.Random;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import java.util.List;
import java.util.Observable;
import java.util.Observer;

import org.opencv.core.Core; 
import org.opencv.core.CvType; 
import org.opencv.core.Mat; 
import org.opencv.core.MatOfByte; 
import org.opencv.core.MatOfFloat; 
import org.opencv.core.MatOfPoint; 
import org.opencv.core.TermCriteria;
import org.opencv.core.MatOfPoint2f; 
import org.opencv.core.Point; 
import org.opencv.core.Rect; 
import org.opencv.core.RotatedRect; 
import org.opencv.core.Scalar; 
import org.opencv.core.Size; 
import org.opencv.video.Video;
import org.opencv.videoio.VideoCapture;
import org.bytedeco.javacpp.opencv_videoio.VideoWriter;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import org.opencv.calib3d.Calib3d;

import ch.unizh.ini.jaer.projects.davis.frames.ApsFrameExtractor;
import ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlow;
import ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.v;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.vx;
import static ch.unizh.ini.jaer.projects.rbodo.opticalflow.AbstractMotionFlowIMU.vy;
import com.jogamp.common.util.Bitstream.ByteStream;
import com.jogamp.opengl.GL;
import com.jogamp.opengl.GL2;
import com.jogamp.opengl.GLAutoDrawable;
import com.jogamp.opengl.GLException;
import com.jogamp.opengl.util.gl2.GLUT;
import java.awt.BorderLayout;
import java.awt.Dimension;
import java.awt.event.WindowAdapter;
import java.awt.event.WindowEvent;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.swing.JFileChooser;

import net.sf.jaer.Description;
import net.sf.jaer.DevelopmentStatus;
import net.sf.jaer.chip.AEChip;
import net.sf.jaer.chip.Chip2D;
import net.sf.jaer.event.ApsDvsEvent;
import net.sf.jaer.event.ApsDvsEventPacket;
import net.sf.jaer.event.EventPacket;
import net.sf.jaer.event.PolarityEvent;
import net.sf.jaer.event.orientation.ApsDvsMotionOrientationEvent;
import net.sf.jaer.eventprocessing.EventFilter;
import static net.sf.jaer.eventprocessing.EventFilter.log;
import net.sf.jaer.eventprocessing.EventFilter2D;
import net.sf.jaer.eventprocessing.FilterChain;
import net.sf.jaer.graphics.AEFrameChipRenderer;
import net.sf.jaer.graphics.ImageDisplay;
import net.sf.jaer.util.TobiLogger;
import org.apache.commons.lang3.ArrayUtils;
import java.util.Random;

/**
 *
 * @author minliu
 */
@Description("Optical Flow methods based on OpenCV")
@DevelopmentStatus(DevelopmentStatus.Status.Experimental)
public class OpenCVFlow extends AbstractMotionFlow 
                        implements PropertyChangeListener, Observer /* Observer needed to get change events on chip construction */ {

    static { 
    String jvmVersion = System.getProperty("sun.arch.data.model");
        
    try {
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    // System.loadLibrary("opencv_ffmpeg320_" + jvmVersion);   // Notice, cannot put the file type extension (.dll) here, it will add it automatically. 
    } catch (UnsatisfiedLinkError e) {
        System.err.println("Native code library failed to load.\n" + e);
        // System.exit(1);
        }
    }
    
    private final ImageDisplay OFResultDisplay;
    
    private JFrame OFResultFrame = null;
    protected boolean showAPSFrameDisplay = getBoolean("showAPSFrameDisplay", true);
    private int[][] color = new int[100][3];
    private float[] oldBuffer = null, newBuffer = null;
    private PatchMatchFlow patchFlow;
    private TobiLogger globalMotionVectorLogger;
    Mat RPos, tPos;
    private ApsFrameExtractor apsFrameExtractor;
    private int lastApsTS, currentApsTS;
    
    public OpenCVFlow(AEChip chip) {
        super(chip);
        System.out.println("Welcome to OpenCV " + Core.VERSION);

        OFResultDisplay = ImageDisplay.createOpenGLCanvas();
        
        OFResultFrame = new JFrame("Optical Flow Result Frame");
        OFResultFrame.setPreferredSize(new Dimension(800, 800));
        OFResultFrame.getContentPane().add(OFResultDisplay, BorderLayout.CENTER);
        OFResultFrame.pack();
        OFResultFrame.addWindowListener(new WindowAdapter() {
            @Override
            public void windowClosing(final WindowEvent e) {
                setShowAPSFrameDisplay(false);
            }
        }); 
        
        FilterChain chain = new FilterChain(chip);        
        try {
            patchFlow = new PatchMatchFlow(chip);
            patchFlow.imuFlowEstimator = this.imuFlowEstimator;
            patchFlow.setFilterEnabled(true);
            // chain.add(patchFlow);
        } catch (Exception e) {
            log.warning("could not setup PatchMatchFlow fiter.");
        }        
        apsFrameExtractor = new ApsFrameExtractor(chip);
        chain.add(apsFrameExtractor);
        setEnclosedFilterChain(chain);

        // Init random value for OF vectors.
        for (int i = 0; i < 100; i++) {
            Random rand = new Random();
            // Java 'Color' class takes 3 floats, from 0 to 1.
            float r = rand.nextFloat();
            float g = rand.nextFloat();
            float b = rand.nextFloat();     
            color[i][0] = (int)(1*255);
            color[i][1] = (int)(0*255);
            color[i][2] = (int)(0*255);
        }
        
        tPos = Mat.zeros(3, 1, CvType.CV_32F);
        RPos = Mat.eye(3, 3, CvType.CV_32F);
        apsFrameExtractor.getSupport().addPropertyChangeListener(ApsFrameExtractor.EVENT_NEW_FRAME, this);   
        patchFlow.getSupport().addPropertyChangeListener(PatchMatchFlow.EVENT_NEW_SLICES,this);
        chip.addObserver(this); // to allocate memory once chip size is known
    }

    @Override
    public EventPacket filterPacket(EventPacket in) {   
        if(!isFilterEnabled()) {
                return in;
        }
        setupFilter(in);
        
        if (showAPSFrameDisplay && !OFResultFrame.isVisible()) {
            OFResultFrame.setVisible(true);
        }
        Iterator i = null;
        if (in instanceof ApsDvsEventPacket) {
            i = ((ApsDvsEventPacket) in).fullIterator();
        } else {
            i = ((EventPacket) in).inputIterator();
        }

        while (i.hasNext()) {
            Object o = i.next();
            if (o == null) {
                log.warning("null event passed in, returning input packet");
                return in;
            }
            if ((o instanceof ApsDvsEvent) && ((ApsDvsEvent) o).isApsData()) {
                continue;
            }
            PolarityEvent ein = (PolarityEvent) o;
            if (!extractEventInfo(o)) {
                continue;
            }
            if (measureAccuracy || discardOutliersForStatisticalMeasurementEnabled) {
                if (imuFlowEstimator.calculateImuFlow(o)) {
                    continue;
                }
            }
            if (xyFilter()) {
                continue;
            }
            countIn++;
        }
        
        /* This part is to start logging global motion vector according to user's configurations automatically. */
        int timeMarker = patchFlow.ts - patchFlow.getChip().getAeInputStream().getFirstTimestamp();
        int startTime = 500000, endTime = 6000000;
        if ( timeMarker >= startTime && timeMarker <= endTime && globalMotionVectorLogger == null) {                
            globalMotionVectorLogger = new TobiLogger(patchFlow.getChip().getAeInputStream().getFile().getName() + patchFlow.getSliceMethod() + patchFlow.getSliceDurationUs() , 
                                                        "Global Motion vector for every generated slice");
            globalMotionVectorLogger.setNanotimeEnabled(false);
            globalMotionVectorLogger.setHeaderLine("system_time(ms) relative_timestamp(us) sliceDeltaT(us) method globalVx(pps) globalVy(pps) globalRotation(degree/s) samples");
            globalMotionVectorLogger.setEnabled(true);
        } 

        if ( timeMarker >= endTime && globalMotionVectorLogger != null) {
            doStopLoggingGlobalMotion();
        } 
        
        OFResultDisplay.checkPixmapAllocation();       
        
//        final ApsDvsEventPacket packet = (ApsDvsEventPacket) in;
//        if (packet == null) {
//            return null;
//        }
//        if (packet.getEventClass() != ApsDvsEvent.class) {
//            EventFilter.log.warning("wrong input event class, got " + packet.getEventClass() + " but we need to have " + ApsDvsEvent.class);
//            return null;
//        }
//        final Iterator apsItr = packet.fullIterator();
//        while (apsItr.hasNext()) {
//            final ApsDvsEvent e = (ApsDvsEvent) apsItr.next();
//            if (e.isApsData()) {
//                apsFrameExtractor.putAPSevent(e);
//            }
//        }
        if(isShowAPSFrameDisplay()) {
            OFResultDisplay.repaint();            
        }
        // motionFlowStatistics.updatePacket(countIn, countOut);
        return isDisplayRawInput() ? in : dirPacket;
    }

    @Override
    public synchronized void resetFilter() {
        super.resetFilter();

//        if(patchFlow != null) {
//            patchFlow.resetFilter();            
//        }
        tPos = Mat.zeros(3, 1, CvType.CV_32F);
        RPos = Mat.eye(3, 3, CvType.CV_32F);
        if (OFResultDisplay != null) {
            OFResultDisplay.setImageSize(chip.getSizeX(), chip.getSizeY());            
        }
    }

    @Override
    public void initFilter() {
        resetFilter();
    }

    @Override
    public void update(Observable o, Object arg) {
        if ((o != null) && (arg != null)) {
            if ((o instanceof AEChip) && (arg.equals(Chip2D.EVENT_SIZEX) || arg.equals(Chip2D.EVENT_SIZEY))) {
                initFilter();
            }
        }    
    }
    
    @Override
    synchronized public void annotate(GLAutoDrawable drawable) {
        super.annotate(drawable);
        GL2 gl = drawable.getGL().getGL2();
        try {
            gl.glEnable(GL.GL_BLEND);
            gl.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_ONE_MINUS_SRC_ALPHA);
            gl.glBlendEquation(GL.GL_FUNC_ADD);
        } catch (GLException e) {
            e.printStackTrace();
        }   
        
        if (measureAccuracy) {
            gl.glPushMatrix();
            final int offset = -10;
            gl.glRasterPos2i(chip.getSizeX() / 2, -5);
            chip.getCanvas().getGlut().glutBitmapString(GLUT.BITMAP_HELVETICA_18,
                    motionFlowStatistics.endpointErrorAbs.graphicsString("OpenLK_AEE(abs):", "pps"));
            gl.glPopMatrix();
        }
    }
    
    @Override
    public synchronized void setFilterEnabled(final boolean yes) {
        super.setFilterEnabled(yes); // To change body of generated methods, choose Tools | Templates.
        if (!isFilterEnabled()) {
            if (OFResultFrame != null) {
                OFResultFrame.setVisible(false);
            }
        }
    } 
    
    @Override
    public synchronized void propertyChange(PropertyChangeEvent evt) {
        super.propertyChange(evt); // resets filter on rewind, etc
        if (evt.getPropertyName().equals(ApsFrameExtractor.EVENT_NEW_FRAME)) {
            if(newBuffer == null) {
                newBuffer = ((float[]) evt.getNewValue()).clone();
                for ( int i = 0; i < newBuffer.length; i++ ) {
                    newBuffer[i] = newBuffer[i] * 255;
                }
                lastApsTS = apsFrameExtractor.getLastFrameTimestamp();
                return;
            }
            lastApsTS = currentApsTS;
            currentApsTS = apsFrameExtractor.getLastFrameTimestamp();
            oldBuffer = new float[newBuffer.length];
            oldBuffer = newBuffer.clone();
            float[] buffer = OFResultDisplay.getPixmapArray();
            byte[] byteBuffer = new byte[buffer.length];
            for(int i=0; i<buffer.length;i++) {
                byteBuffer[i] = (byte) (buffer[i] * 255);
            }
            newBuffer = ((float[]) evt.getNewValue()).clone();
            for ( int i = 0; i < newBuffer.length; i++ ) {
                    newBuffer[i] = newBuffer[i] * 255;
            }            
            Mat newFrame = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_32F);
            newFrame.put(0, 0, newBuffer);   
            Mat oldFrame = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_32F);
            oldFrame.put(0, 0, oldBuffer); 
            
            Mat newFrameTmp = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_8U);
            Mat oldFrameTmp = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_8U);
            newFrame.convertTo(newFrameTmp, CvType.CV_8U);
            oldFrame.convertTo(oldFrameTmp, CvType.CV_8U);

            // params for ShiTomasi corner detection            
            FeatureParams feature_params  = new FeatureParams(100, 0.3, 7, 7);
            
            // Feature extraction
            MatOfPoint p0 = new MatOfPoint();
            Imgproc.goodFeaturesToTrack(newFrameTmp, p0, feature_params.maxCorners, feature_params.qualityLevel, feature_params.minDistance);       

            MatOfPoint2f prevPts = new MatOfPoint2f(p0.toArray());
            MatOfPoint2f nextPts = new MatOfPoint2f();
            MatOfByte status = new MatOfByte();
            MatOfFloat err = new MatOfFloat();

            int featureNum = prevPts.checkVector(2, CvType.CV_32F, true);
            System.out.println("The number of feature detected is : " + featureNum);     

            // No features are detected
            if (featureNum == -1) {
                return;
            }
            try {
                Video.calcOpticalFlowPyrLK(oldFrameTmp, newFrameTmp, prevPts, nextPts, status, err);            
            } catch (Exception e) {
                System.err.println(e);
                // newFrame.copyTo(oldFrame);
            }            
            
            // TODO: Select good points 

            // draw the tracks
            Point[] prevPoints = prevPts.toArray();
            Point[] nextPoints = nextPts.toArray();
            byte[] st = status.toArray();
            float[] er = err.toArray();    
            
            Mat colorNewFrame = new Mat(chip.getSizeY(), chip.getSizeX(), CvType.CV_32FC3);
            Imgproc.cvtColor(newFrame, colorNewFrame, Imgproc.COLOR_GRAY2RGB);
            for (int i = 0; i < prevPoints.length; i++) {
                Imgproc.arrowedLine(colorNewFrame, prevPoints[i], nextPoints[i], new Scalar(color[i][0],color[i][1],color[i][2]), 2, 8, 0, 0.1);  
                Imgproc.circle(colorNewFrame, prevPoints[i], 1, new Scalar(255,255,255),-1);
            }
            
            // Select good points and copy them for output
            int index = 0;
            for(byte stTmp: st) {
                if(stTmp == 1) {
                    // if (outlierFlg.length >0 ) {
                        // if (outlierFlg[index] == 1) {
                            e = new PolarityEvent();
                            x = (short)(prevPoints[index].x);
                            y = (short)prevPoints[index].y;
                            e.x = (short)x;
                            e.y = (short)y;    // e, x and y all of them are used in processGoodEvent();
        //                    e.timestamp = ts;
                            vx = (float)(nextPoints[index].x - prevPoints[index].x) * 1000000 / (currentApsTS - lastApsTS);
                            vy = (float)(nextPoints[index].y - prevPoints[index].y) * 1000000 / (currentApsTS - lastApsTS);
                            v = (float) Math.sqrt(vx * vx + vy * vy);
                            processGoodEvent();                      
                        // exportFlowToMatlab(false);                            
                        // }
                    // }

                }
                index++;                
            }
            motionFlowStatistics.updatePacket(countIn, countOut, lastApsTS);
            float[] return_buff = new float[(int) (colorNewFrame.total() * 
                                            colorNewFrame.channels())];
            colorNewFrame.get(0, 0, return_buff);
            for ( int i = 0; i < return_buff.length; i++ ) {
                    return_buff[i] = return_buff[i]/255.0f;
            }      
            OFResultDisplay.setPixmapArray(return_buff);  
            
            DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");
            File folder = new File("EventSlices/" + chip.getAeInputStream().getFile().getName() + "_" + df);
            // if the directory does not exist, create it
            if (!folder.exists()) {
                folder.mkdir();
            }            
            File outputfile = new File(folder, String.format("packet_pid_%d.jpg", lastApsTS));
            Core.flip(newFrameTmp, newFrameTmp, 0);
            // Uncomment this when you want to store the slice's pictures.
//            try {
//                ImageIO.write(Mat2BufferedImage(newFrameTmp), "jpg", outputfile);
//            } catch (IOException ex) {
//                Logger.getLogger(OpenCVFlow.class.getName()).log(Level.SEVERE, null, ex);
//            }
{'fraction_non_alphanumeric': 0.11061318176849928, 'fraction_numerical': 0.009017378948518964, 'mean_word_length': 2.12848350145324, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5320910', 'n_tokens_mistral': 11080, 'n_tokens_neox': 10573, 'n_words': 2111}