<commit_before># cito - The Xenon1T experiments software trigger
# Copyright 2013.  All rights reserved.
# https://github.com/tunnell/cito
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of the Xenon experiment, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Speed tester

SciPyFFTWaveform is really freaking slow compared to others.  Same with FFTW.
"""

__author__ = 'tunnell'
import logging
import time

import numpy as np
import scipy
import pymongo
from cliff.command import Command

from cito.helpers import cInterfaceV1724
from cito.helpers import xedb, waveform


class TimingTask():
    def process(self, t0, t1, loops=1, verbose=False):
        sums = 0.0
        mins = 1.7976931348623157e+308
        maxs = 0.0
        print('====%s Timing====' % self.__class__.__name__)
        for i in range(0, loops):
            start_time = time.time()
            result = self.call(t0, t1)
            dt = time.time() - start_time
            mins = dt if dt < mins else mins
            maxs = dt if dt > maxs else maxs
            sums += dt
            if verbose == True:
                print('\t%r ran in %2.9f sec on run %s' %
                      (self.__class__.__name__, dt, i))
        print('%r min run time was %2.9f sec' %
              (self.__class__.__name__, mins))
        print('%r max run time was %2.9f sec' %
              (self.__class__.__name__, maxs))
        print('%r avg run time was %2.9f sec in %s runs' %
              (self.__class__.__name__, sums / loops, loops))

        size = result / 1024 / 1024  # MB
        print('%r size %d MB %s runs' % (self.__class__.__name__, size, loops))
        speed = size / (sums / loops)
        print('%r avg speed %2.9f MB/s in %s runs' %
              (self.__class__.__name__, speed, loops))
        print('==== end ====')
        return result

    def get_cursor(self, t0, t1):
        conn, mongo_db_obj, collection = xedb.get_mongo_db_objects()

        # $gte and $lt are special mongo functions for greater than and less than
        subset_query = {"triggertime": {'$gte': t0,
                                        '$lt': t1}}
        return collection.find(subset_query)

    def call(self, t0, t1):
        raise NotImplementedError()


class Fetch(TimingTask):
    """Fetch and decompress"""

    def call(self, t0, t1):
        cursor = self.get_cursor(t0, t1)
        size = 0.0
        for doc in cursor:
            size += len(xedb.get_data_from_doc(doc))<commit_msg>Another speed field<commit_after># cito - The Xenon1T experiments software trigger
# Copyright 2013.  All rights reserved.
# https://github.com/tunnell/cito
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of the Xenon experiment, nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Speed tester

SciPyFFTWaveform is really freaking slow compared to others.  Same with FFTW.
"""

__author__ = 'tunnell'
import logging
import time

import numpy as np
import scipy
import pymongo
from cliff.command import Command

from cito.helpers import cInterfaceV1724
from cito.helpers import xedb, waveform


class TimingTask():
    def process(self, t0, t1, loops=1, verbose=False):
        sums = 0.0
        mins = 1.7976931348623157e+308
        maxs = 0.0
        print('====%s Timing====' % self.__class__.__name__)
        for i in range(0, loops):
            start_time = time.time()
            result = self.call(t0, t1)
            dt = time.time() - start_time
            mins = dt if dt < mins else mins
            maxs = dt if dt > maxs else maxs
            sums += dt
            if verbose == True:
                print('\t%r ran in %2.9f sec on run %s' %
                      (self.__class__.__name__, dt, i))
        print('%r min run time was %2.9f sec' %
              (self.__class__.__name__, mins))
        print('%r max run time was %2.9f sec' %
              (self.__class__.__name__, maxs))
        print('%r avg run time was %2.9f sec in %s runs' %
              (self.__class__.__name__, sums / loops, loops))

        size = result / 1024
        print('%r size %d kB %s runs' % (self.__class__.__name__, size, loops))
        size = size / 1024  # MB
        print('%r size %d MB %s runs' % (self.__class__.__name__, size, loops))
        speed = size / (sums / loops)
        print('%r avg speed %2.9f MB/s in %s runs' %
              (self.__class__.__name__, speed, loops))
        print('==== end ====')
        return result

    def get_cursor(self, t0, t1):
        conn, mongo_db_obj, collection = xedb.get_mongo_db_objects()

        # $gte and $lt are special mongo functions for greater than and less than
        subset_query = {"triggertime": {'$gte': t0,
                                        '$lt': t1}}
        return collection.find(subset_query)

    def call(self, t0, t1):
        raise NotImplementedError()


class Fetch(TimingTask):
    """Fetch and decompress"""

    def call(self, t0, t1):
        cursor = self.get_cursor(t0, t1)
        size = 0.0
        for doc in cursor:
            size += len(xedb.get_data_from_doc(doc))
{'fraction_non_alphanumeric': 0.08832604782720493, 'fraction_numerical': 0.017742350218565184, 'mean_word_length': 2.7327255278310942, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '450475', 'n_tokens_mistral': 2367, 'n_tokens_neox': 2171, 'n_words': 904}