<commit_before>import time
import numpy
import sqlite3
import PyQt4.QtGui as QG
import PyQt4.QtCore as QC
from matplotlib.backends.backend_qt4agg import (
        FigureCanvasQTAgg as FigureCanvas,
        NavigationToolbar2QTAgg as NavigationToolbar)
from matplotlib.figure import Figure
from matplotlib.ticker import MaxNLocator


class PlotDepth(QG.QWidget):
    def __init__(self, parent=None, timeout=1500):
        QG.QWidget.__init__(self, parent)

        self._depth = sqlite3.connect(':memory:')
        self._depth.execute("""CREATE TABLE
                bid (price REAL PRIMARY KEY, amount TEXT)""")
        self._depth.execute("""CREATE TABLE
                ask (price REAL PRIMARY KEY, amount TEXT)""")

        self.fig = Figure(figsize=(8, 4))
        self.canvas = FigureCanvas(self.fig)
        self.canvas.setParent(self)

        layout = QG.QVBoxLayout()
        layout.setMargin(0)
        layout.addWidget(self.canvas)
        self.setLayout(layout)

        self.ax = self.fig.add_subplot(111)
        self.ax.xaxis.set_major_locator(MaxNLocator(9))

        self.bid = self.ax.plot([], [], 'g', lw=3)[0]
        self.ask = self.ax.plot([], [], 'r', lw=3)[0]

        # Limit visual display to n points.
        self.nbin = 100
        self._bidfill = None
        self._askfill = None

        self.price_threshold = 10

        self.need_replot = 0
        self.replot_after = 20
        self.timer = QC.QTimer()
        self.timer.timeout.connect(self.replot)
        self.timer.start(timeout) # x ms.

        self.timer_clean = QC.QTimer()
        self.timer_clean.timeout.connect(self._clean_db)
        # Remove unused data from in-memory database each n seconds.
        self.timer_clean.start(30 * 1000)


    def _clean_db(self):
        print("Cleaning database..")

        # Clean bids.
        self._depth.execute("""
            DELETE FROM bid WHERE price IN (
                SELECT a.price FROM bid a
                WHERE ((SELECT b.price FROM bid b ORDER BY b.price DESC
                        LIMIT 1) - a.price) > ?)""", (self.price_threshold, ))

        # Clean asks.
        self._depth.execute("""
            DELETE FROM ask WHERE price IN (
                SELECT a.price FROM ask a
                WHERE (a.price - (SELECT b.price FROM ask b ORDER BY b.price
                        ASC LIMIT 1)) > ?)""", (self.price_threshold, ))

        self._depth.execute("VACUUM")


    def replot(self):
        # If you are calling this function, make sure you meant to do it.

        if not self.need_replot:<commit_msg>Also removed outdated data from the depth market. This happens mostly when the initial data is old (i.e., a pre-fetch)<commit_after>import time
import numpy
import sqlite3
import PyQt4.QtGui as QG
import PyQt4.QtCore as QC
from matplotlib.backends.backend_qt4agg import (
        FigureCanvasQTAgg as FigureCanvas,
        NavigationToolbar2QTAgg as NavigationToolbar)
from matplotlib.figure import Figure
from matplotlib.ticker import MaxNLocator


class PlotDepth(QG.QWidget):
    def __init__(self, parent=None, timeout=1500):
        QG.QWidget.__init__(self, parent)

        self._depth = sqlite3.connect(':memory:')
        self._depth.execute("""CREATE TABLE
                bid (price REAL PRIMARY KEY, amount TEXT)""")
        self._depth.execute("""CREATE TABLE
                ask (price REAL PRIMARY KEY, amount TEXT)""")

        self.fig = Figure(figsize=(8, 4))
        self.canvas = FigureCanvas(self.fig)
        self.canvas.setParent(self)

        layout = QG.QVBoxLayout()
        layout.setMargin(0)
        layout.addWidget(self.canvas)
        self.setLayout(layout)

        self.ax = self.fig.add_subplot(111)
        self.ax.xaxis.set_major_locator(MaxNLocator(9))

        self.bid = self.ax.plot([], [], 'g', lw=3)[0]
        self.ask = self.ax.plot([], [], 'r', lw=3)[0]

        # Limit visual display to n points.
        self.nbin = 100
        self._bidfill = None
        self._askfill = None

        self.price_threshold = 10

        self.need_replot = 0
        self.replot_after = 20
        self.timer = QC.QTimer()
        self.timer.timeout.connect(self.replot)
        self.timer.start(timeout) # x ms.

        self.timer_clean = QC.QTimer()
        self.timer_clean.timeout.connect(self._clean_db)
        # Remove unused/old data from in-memory database each n seconds.
        self.timer_clean.start(15 * 1000)


    def _clean_db(self):
        print("Cleaning database..")

        # Clean bids.
        dnum = self._depth.execute("""
            DELETE FROM bid WHERE price IN (
                SELECT a.price FROM bid a
                WHERE ((SELECT b.price FROM bid b ORDER BY b.price DESC
                        LIMIT 1) - a.price) > ?)""", (1.5*self.price_threshold,
                            )).rowcount
        print("  Bids removed:", dnum)

        # Clean asks.
        dnum = self._depth.execute("""
            DELETE FROM ask WHERE price IN (
                SELECT a.price FROM ask a
                WHERE (a.price - (SELECT b.price FROM ask b ORDER BY b.price
                        ASC LIMIT 1)) > ?)""", (1.5*self.price_threshold,
                            )).rowcount
        print("  Asks removed:", dnum)

        # Check if the curves are crossing and remove that data.
        # This might happen after a reconnect where we have outdated
        # information. This also happens when the initial is old.
        dnum = self._depth.execute("""
            DELETE FROM bid WHERE price IN (
                SELECT b.price FROM bid b WHERE b.price > (
                    SELECT a.price FROM ask a ORDER BY a.price ASC
                    LIMIT 1))""").rowcount
        print("  Old bids removed:", dnum)

        self._depth.execute("VACUUM")


    def replot(self):
        # If you are calling this function, make sure you meant to do it.

        if not self.need_replot:
{'fraction_non_alphanumeric': 0.10356479134989019, 'fraction_numerical': 0.013346849129920594, 'mean_word_length': 2.017329255861366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1941432', 'n_tokens_mistral': 1852, 'n_tokens_neox': 1711, 'n_words': 486}