<commit_before>from flow import Node, Feature, Decoder, NotEnoughData
from timeseries import ConstantRateTimeSeries, TimeSlice
import numpy as np
from duration import Picoseconds
import struct

class MeasureOfTransience(Node):
    '''
    Measure of Transience, as defined in section 5.2.1 of 
    http://www.mp3-tech.org/programmer/docs/Masri_thesis.pdf
    
    Uses the ratio of high-frequency content in the signal to detect onsets.
    Effective for percussive onsets.
    '''
    def __init__(self, needs = None):
        super(MeasureOfTransience, self).__init__(needs = needs)
    
    def _first_chunk(self, data):
        self._bin_numbers = np.arange(1, data.shape[1] + 1)
        padding = np.zeros(data.shape[1])
        padding[:] = 1e-12
        return np.concatenate([padding[None,:], data])
    
    def _process(self, data):
        magnitude = (data[:, 2:] ** 2)
        energy = magnitude.sum(axis = 1)
        hfc = (magnitude * self._bin_numbers[2:]).sum(axis = 1)
        energy[energy == 0] = 1e-12
        hfc[hfc == 0] = 1e-12
        mot = (hfc[1:] / hfc[:-1]) * (hfc[1:] / energy[1:])
        yield ConstantRateTimeSeries(mot, data.frequency, data.duration)

class ComplexDomain(Node):
    '''
    Complex-domain onset detection as described in
    http://www.eecs.qmul.ac.uk/legacy/dafx03/proceedings/pdfs/dafx81.pdf
    '''
    def __init__(self, needs = None):
        super(ComplexDomain, self).__init__(needs = needs)
    
    def _first_chunk(self, data):
        first = np.zeros((2, 3, data.shape[-1]), dtype = np.complex128)
        first[0, 2:, :] = data[0, :1]
        first[1, 1:, :] = data[0, :2]
        return ConstantRateTimeSeries(\
              np.concatenate([first, data]),
              data.frequency,
              data.duration)
    
    def _process(self, data):
        # delta between expected and actual phase
        # TODO: unwrap phases before computing deltas, to avoid artifacts
        # or discontinuties from phase boundary wrapping
        angle = np.angle(data)
        angle = np.unwrap(angle, axis = 1)
        angle = np.angle(angle[:,2] - (2 * angle[:,1]) + angle[:,0])
        
        # expected magnitude
        expected = np.abs(data[:, 1, :])
        # actual magnitude
        actual = np.abs(data[:, 2, :])
        # detection function array
        detect = np.zeros(angle.shape)
        
        # where phase delta is zero, detection function is the difference 
        # between expected and actual magnitude
        zero_phase_delta_indices = np.where(angle == 0) 
        detect[zero_phase_delta_indices] = \
            (expected - actual)[zero_phase_delta_indices]
            
        # where phase delta is non-zero, detection function combines magnitude
        # and phase deltas
        nonzero_phase_delta_indices = np.where(angle != 0)  
        detect[nonzero_phase_delta_indices] = (
          ((expected**2) + (actual**2) - 
          (2 * expected * actual * np.cos(angle))) ** 0.5)[nonzero_phase_delta_indices]
          
        # TODO: This duration isn't right.  It should probably be 
        # data.duration // 3
        yield ConstantRateTimeSeries(\
             detect.sum(axis = 1),
             data.frequency,
             data.duration)
        
class Flux(Node):
    
    def __init__(self, needs = None):
        super(Flux, self).__init__(needs = needs)
        self._memory = None
    
    def _process(self, data):
        
        if self._memory is None:
            # prepend the first vector, so that the initial flux value is zero
            d = np.vstack([data[0], data])
        else:
            # prepend the last vector from the previous batch
            d = np.vstack([self._memory, data])
        
        self._memory = data[-1]
        
        # Take the difference, keeping only positive changes 
        # (the magnitude increased)
        diff = np.diff(d, axis = 0)
        diff[diff < 0] = 0
        
        # take the l1 norm of each frame
        yield ConstantRateTimeSeries(\
              diff.sum(axis = 1),
              data.frequency,
              data.duration)

class BasePeakPicker(Node):
    
    def __init__(self, needs = None):
        super(BasePeakPicker, self).__init__(needs = needs)
        self._pos = Picoseconds(0)
    
    def _onset_indices(self, data):
        raise NotImplementedError()
    
    def _last_chunk(self):
        yield self._pos
    
    def _process(self, data):
        if self._pos == Picoseconds(0):
            yield self._pos
        
        indices = self._onset_indices(data)
        timestamps = self._pos + (indices * data.frequency)
        self._pos += len(data) * data.frequency
        yield timestamps

class MovingAveragePeakPicker(BasePeakPicker):
    
    def __init__(self, aggregate = np.mean, needs = None):
        super(MovingAveragePeakPicker, self).__init__(needs = needs)
        self._aggregate = aggregate
    <commit_msg>Re-organize the Flux class a little<commit_after>from flow import Node, Feature, Decoder, NotEnoughData
from timeseries import ConstantRateTimeSeries, TimeSlice
import numpy as np
from duration import Picoseconds
import struct

class MeasureOfTransience(Node):
    '''
    Measure of Transience, as defined in section 5.2.1 of 
    http://www.mp3-tech.org/programmer/docs/Masri_thesis.pdf
    
    Uses the ratio of high-frequency content in the signal to detect onsets.
    Effective for percussive onsets.
    '''
    def __init__(self, needs = None):
        super(MeasureOfTransience, self).__init__(needs = needs)
    
    def _first_chunk(self, data):
        self._bin_numbers = np.arange(1, data.shape[1] + 1)
        padding = np.zeros(data.shape[1])
        padding[:] = 1e-12
        return np.concatenate([padding[None,:], data])
    
    def _process(self, data):
        magnitude = (data[:, 2:] ** 2)
        energy = magnitude.sum(axis = 1)
        hfc = (magnitude * self._bin_numbers[2:]).sum(axis = 1)
        energy[energy == 0] = 1e-12
        hfc[hfc == 0] = 1e-12
        mot = (hfc[1:] / hfc[:-1]) * (hfc[1:] / energy[1:])
        yield ConstantRateTimeSeries(mot, data.frequency, data.duration)

class ComplexDomain(Node):
    '''
    Complex-domain onset detection as described in
    http://www.eecs.qmul.ac.uk/legacy/dafx03/proceedings/pdfs/dafx81.pdf
    '''
    def __init__(self, needs = None):
        super(ComplexDomain, self).__init__(needs = needs)
    
    def _first_chunk(self, data):
        first = np.zeros((2, 3, data.shape[-1]), dtype = np.complex128)
        first[0, 2:, :] = data[0, :1]
        first[1, 1:, :] = data[0, :2]
        return ConstantRateTimeSeries(\
              np.concatenate([first, data]),
              data.frequency,
              data.duration)
    
    def _process(self, data):
        # delta between expected and actual phase
        # TODO: unwrap phases before computing deltas, to avoid artifacts
        # or discontinuties from phase boundary wrapping
        angle = np.angle(data)
        angle = np.unwrap(angle, axis = 1)
        angle = np.angle(angle[:,2] - (2 * angle[:,1]) + angle[:,0])
        
        # expected magnitude
        expected = np.abs(data[:, 1, :])
        # actual magnitude
        actual = np.abs(data[:, 2, :])
        # detection function array
        detect = np.zeros(angle.shape)
        
        # where phase delta is zero, detection function is the difference 
        # between expected and actual magnitude
        zero_phase_delta_indices = np.where(angle == 0) 
        detect[zero_phase_delta_indices] = \
            (expected - actual)[zero_phase_delta_indices]
            
        # where phase delta is non-zero, detection function combines magnitude
        # and phase deltas
        nonzero_phase_delta_indices = np.where(angle != 0)  
        detect[nonzero_phase_delta_indices] = (
          ((expected**2) + (actual**2) - 
          (2 * expected * actual * np.cos(angle))) ** 0.5)[nonzero_phase_delta_indices]
          
        # TODO: This duration isn't right.  It should probably be 
        # data.duration // 3
        yield ConstantRateTimeSeries(\
             detect.sum(axis = 1),
             data.frequency,
             data.duration)
        
class Flux(Node):
    
    def __init__(self, needs = None):
        super(Flux, self).__init__(needs = needs)
        self._memory = None
    
    def _enqueue(self, data, pusher):
        if self._memory is None:
            self._cache = np.vstack((data[0], data))
        else:
            self._cache = np.vstack((self._memory, data))
        self._memory = data[-1]
    
    def _process(self, data):
        diff = np.diff(data, axis = 0)
        yield ConstantRateTimeSeries(\
              np.linalg.norm(diff, axis = -1),
              data.frequency,
              data.duration)

class BasePeakPicker(Node):
    
    def __init__(self, needs = None):
        super(BasePeakPicker, self).__init__(needs = needs)
        self._pos = Picoseconds(0)
    
    def _onset_indices(self, data):
        raise NotImplementedError()
    
    def _last_chunk(self):
        yield self._pos
    
    def _process(self, data):
        if self._pos == Picoseconds(0):
            yield self._pos
        
        indices = self._onset_indices(data)
        timestamps = self._pos + (indices * data.frequency)
        self._pos += len(data) * data.frequency
        yield timestamps

class MovingAveragePeakPicker(BasePeakPicker):
    
    def __init__(self, aggregate = np.mean, needs = None):
        super(MovingAveragePeakPicker, self).__init__(needs = needs)
        self._aggregate = aggregate
    
{'fraction_non_alphanumeric': 0.11157282360274257, 'fraction_numerical': 0.014440058175773946, 'mean_word_length': 2.197276652274992, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '313351', 'n_tokens_mistral': 2976, 'n_tokens_neox': 2853, 'n_words': 728}