<commit_before>#BSD 2-Clause (c) 2014: A.Doherty (Oxford), D.Jackson, N.Hammerla (Newcastle)
"""
This python object:
    1) has attributes to represent an episode of behaviour.
        e.g. behavType, startTime, endTime, value 
    2) static methods to identify behaviour episodes from time series data
        e.g. to calculate nonWear 
e.g.
    nonWearBout = new behaviourEpisode('nonWear', '2013-11-16 02:28',
            '2013-11-16 02:48', 0.01, 0.01, 0.01)
    nonWearBouts = behaviourEpisode.identify_episodes(input.csv, 'nonWear', 60, 0, 15000, 2, 0, 1809)
"""

import sys
import datetime
from collections import deque

class behaviourEpisode:
        
    def __init__(self, behavType, startTime, endTime, xStd, yStd, zStd):
        """
        Constructor to initialise object attributes 
        """
        self.behavType = behavType #string
        self.startTime = startTime #datetime
        self.endTime = endTime
        self.xStd = xStd #float
        self.yStd = yStd
        self.zStd = zStd

    def __init__(self, behavType):
        """
        Simple constructor
        """
        self.behavType = behavType

    #class variable to store getSummaryNonWear() header information
    summaryHeaderNonWear = 'behaviourType,startTime,endTime,xStd,yStd,zStd'
        
    def getSummaryNonWear(self):
        output = self.behavType + ',' + str(self.startTime) +',' +str(self.endTime)
        output += ',' + str(self.xStd) + ',' + str(self.yStd)
        output += ',' + str(self.zStd)
        return output

'''
Static methods to identify behaviours of interest from epoch csv data
'''

def identifyNonWearEpisodes(epochFile, headerSize, timeCol,
                        timeFormat, xIndex, yIndex, zIndex, targetWeartimeDays,
                        behavType, minFreq, maxStd, graceMaxFreq):
    """
    This method reads through <epochFile.csv> and identifies episodes of <behavType>
    as being a bout/episode of at least <minFreq> minutes with all 2/3 axes
    having a standard deviation below <maxStd> except for some allowed minutes of
    grace <graceMaxFreq>
    This processing method is based on that of <NAME>, Ekelund, Brage and colleagues
    in PLos ONE 2011 6(7), "Estimation of Daily Energy Expenditure in Pregnant and
    Non-Pregnant Women Using a Wrist-Worn Tri-Axial Accelerometer"
    ===
    To identify episodes when reading each line/epoch of CSV, consider:
        1. Is this epoch a potential episode start point (need to see x items ahead)?
            a) i.e. is the value std, for at least two out of three axes, < maxStd
            b) if so create a new 'behaviourEpisode'
        -----
        2. Is this epoch a potential episode ending point?
            a) i.e. is the value std, for at least two out of three axes, > maxStd
            b) if so complete info for 'behaviourEpisode' (see 1b) and store it
        -----
        3. Is this epoch part of an existing episode?
            a) i.e. is there a start point and are current axes still < maxStd
            b) if so add the datetime & unit-count info to episode running total lists
    ===
    Inputs:
        epochFile, csv file, already summarised into epochs, e.g. input.csv
        headerSize, int, num lines in <epochFile> header
        timeCol, int, index of datetime column in CSV
        timeFormat, string, format of <epochFile> datetime values
        (x/y/z)Index, int, index of column in CSV relating to std of x/y/z values in epoch
        targetWeartimeDays, int,
        behavType, string, label episodes found e.g. 'MVPA'
        minFreq, int, how many consecutive instances each bout/episode should be e.g. 10
        maxStd, int, upper bound axis std e.g. 0.05 (50mg)
        graceMaxFreq, int, how many grace occurrences can be outside lower/upper bound e.g. 2
    Output:
        array of identified episodes, of type 'behaviourEpisode'
    """
    '''
    setup variables to help process & store newly detected episodes
    '''
    episodeList = [] #list of type 'behaviourEpisode' for method output
    #to detect start/end of episodes
    seekingNewEpisode = True #bool
    timesQ = deque() #datetime queue
    xQ = deque() #float queue
    yQ = deque()
    zQ = deque()
    #to store candidate episode info
    newEpisode = behaviourEpisode(behavType) #type 'behaviourEpisode'
    xVals = [] #float
    yVals = []
    zVals = []
    #to extract info from CSV <epochFile>
    lineParts = [] #string list
    epochTime = datetime.datetime.now()
    '''
    first determine start and end time of raw data to consider (i.e target days of wear)
    '''
    firstDay = datetime.datetime.now()
    lastDay = datetime.datetime.now()
    epochReader = open(epochFile,'rU') #open file reader...
    #ignore the first <headerSize> lines
    for counter in range(headerSize):
        next(epochReader)
    #get firstDay/epoch
    lineParts = next(epochReader).split(',')
    firstDay = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
    #traverse through rest of CSV file
    for line in epochReader:
        lineParts = line.split(',')
    lastDay = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
    #print 'total data: ', firstDay, lastDay
    #calculate if more days data collected than initially targetted
    dayDiff = (((lastDay-firstDay).days*3600*24) + ((lastDay-firstDay).seconds))
    dayDiff = dayDiff / 3600 / 24.0 #convert to days
    if dayDiff > targetWeartimeDays:
        #if we have more days than needed, ignore first day
        #then take next <targetWeartimeDays>
        firstDay = datetime.datetime(firstDay.year, firstDay.month, firstDay.day, 0, 0, 0)
        firstDay += datetime.timedelta(days=1)
        lastDay = firstDay + datetime.timedelta(days=targetWeartimeDays)
        lastDay += datetime.timedelta(microseconds=-1)
    #print 'considered data: ', firstDay, lastDay
    #finally close file reader as we're at the end of it
    epochReader.close()
    '''
    Now traverse through entire file, to identify episodes (i.e. start/end boundaries)
    '''
    xStd = 0.0
    yStd = 0.0
    zStd = 0.0
    epochReader = open(epochFile,'rU')
    #ignore the first <headerSize> lines
    for counter in range(headerSize):
        next(epochReader)
    #read in first <minFreq> instances from file to initialise queue
    while len(xQ) < minFreq:
        try:
            lineParts = next(epochReader).split(',') #read next from file
        except:
            sys.stderr.write('insufficient epoch data to identify episodes\n')
            break
        lineParts = next(epochReader).split(',') #read next from file
        epochTime = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
        if epochTime >= firstDay and epochTime <= lastDay:
            xStd = float(lineParts[xIndex])
            yStd = float(lineParts[yIndex])
            zStd = float(lineParts[zIndex])
            timesQ.append(epochTime)
            xQ.append(xStd)
            yQ.append(yStd)
            zQ.append(zStd)
    #now traverse through rest of CSV file, to identify episodes
    for line in epochReader:
        #file reading item: add the latest item to the end of the running queue
        lineParts = line.split(',')
        epochTime = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
        if epochTime >= firstDay and epochTime <= lastDay:
            xStd = float(lineParts[xIndex])
            yStd = float(lineParts[yIndex])
            zStd = float(lineParts[zIndex])
            timesQ.append(epochTime)
            xQ.append(xStd)
            yQ.append(yStd)
            zQ.append(zStd)
            #remove first item from queue to maintain size<commit_msg>forgot to delete line to be replaced<commit_after>#BSD 2-Clause (c) 2014: A.Doherty (Oxford), D.Jackson, N.Hammerla (Newcastle)
"""
This python object:
    1) has attributes to represent an episode of behaviour.
        e.g. behavType, startTime, endTime, value 
    2) static methods to identify behaviour episodes from time series data
        e.g. to calculate nonWear 
e.g.
    nonWearBout = new behaviourEpisode('nonWear', '2013-11-16 02:28',
            '2013-11-16 02:48', 0.01, 0.01, 0.01)
    nonWearBouts = behaviourEpisode.identify_episodes(input.csv, 'nonWear', 60, 0, 15000, 2, 0, 1809)
"""

import sys
import datetime
from collections import deque

class behaviourEpisode:
        
    def __init__(self, behavType, startTime, endTime, xStd, yStd, zStd):
        """
        Constructor to initialise object attributes 
        """
        self.behavType = behavType #string
        self.startTime = startTime #datetime
        self.endTime = endTime
        self.xStd = xStd #float
        self.yStd = yStd
        self.zStd = zStd

    def __init__(self, behavType):
        """
        Simple constructor
        """
        self.behavType = behavType

    #class variable to store getSummaryNonWear() header information
    summaryHeaderNonWear = 'behaviourType,startTime,endTime,xStd,yStd,zStd'
        
    def getSummaryNonWear(self):
        output = self.behavType + ',' + str(self.startTime) +',' +str(self.endTime)
        output += ',' + str(self.xStd) + ',' + str(self.yStd)
        output += ',' + str(self.zStd)
        return output

'''
Static methods to identify behaviours of interest from epoch csv data
'''

def identifyNonWearEpisodes(epochFile, headerSize, timeCol,
                        timeFormat, xIndex, yIndex, zIndex, targetWeartimeDays,
                        behavType, minFreq, maxStd, graceMaxFreq):
    """
    This method reads through <epochFile.csv> and identifies episodes of <behavType>
    as being a bout/episode of at least <minFreq> minutes with all 2/3 axes
    having a standard deviation below <maxStd> except for some allowed minutes of
    grace <graceMaxFreq>
    This processing method is based on that of <NAME>, Ekelund, Brage and colleagues
    in PLos ONE 2011 6(7), "Estimation of Daily Energy Expenditure in Pregnant and
    Non-Pregnant Women Using a Wrist-Worn Tri-Axial Accelerometer"
    ===
    To identify episodes when reading each line/epoch of CSV, consider:
        1. Is this epoch a potential episode start point (need to see x items ahead)?
            a) i.e. is the value std, for at least two out of three axes, < maxStd
            b) if so create a new 'behaviourEpisode'
        -----
        2. Is this epoch a potential episode ending point?
            a) i.e. is the value std, for at least two out of three axes, > maxStd
            b) if so complete info for 'behaviourEpisode' (see 1b) and store it
        -----
        3. Is this epoch part of an existing episode?
            a) i.e. is there a start point and are current axes still < maxStd
            b) if so add the datetime & unit-count info to episode running total lists
    ===
    Inputs:
        epochFile, csv file, already summarised into epochs, e.g. input.csv
        headerSize, int, num lines in <epochFile> header
        timeCol, int, index of datetime column in CSV
        timeFormat, string, format of <epochFile> datetime values
        (x/y/z)Index, int, index of column in CSV relating to std of x/y/z values in epoch
        targetWeartimeDays, int,
        behavType, string, label episodes found e.g. 'MVPA'
        minFreq, int, how many consecutive instances each bout/episode should be e.g. 10
        maxStd, int, upper bound axis std e.g. 0.05 (50mg)
        graceMaxFreq, int, how many grace occurrences can be outside lower/upper bound e.g. 2
    Output:
        array of identified episodes, of type 'behaviourEpisode'
    """
    '''
    setup variables to help process & store newly detected episodes
    '''
    episodeList = [] #list of type 'behaviourEpisode' for method output
    #to detect start/end of episodes
    seekingNewEpisode = True #bool
    timesQ = deque() #datetime queue
    xQ = deque() #float queue
    yQ = deque()
    zQ = deque()
    #to store candidate episode info
    newEpisode = behaviourEpisode(behavType) #type 'behaviourEpisode'
    xVals = [] #float
    yVals = []
    zVals = []
    #to extract info from CSV <epochFile>
    lineParts = [] #string list
    epochTime = datetime.datetime.now()
    '''
    first determine start and end time of raw data to consider (i.e target days of wear)
    '''
    firstDay = datetime.datetime.now()
    lastDay = datetime.datetime.now()
    epochReader = open(epochFile,'rU') #open file reader...
    #ignore the first <headerSize> lines
    for counter in range(headerSize):
        next(epochReader)
    #get firstDay/epoch
    lineParts = next(epochReader).split(',')
    firstDay = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
    #traverse through rest of CSV file
    for line in epochReader:
        lineParts = line.split(',')
    lastDay = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
    #print 'total data: ', firstDay, lastDay
    #calculate if more days data collected than initially targetted
    dayDiff = (((lastDay-firstDay).days*3600*24) + ((lastDay-firstDay).seconds))
    dayDiff = dayDiff / 3600 / 24.0 #convert to days
    if dayDiff > targetWeartimeDays:
        #if we have more days than needed, ignore first day
        #then take next <targetWeartimeDays>
        firstDay = datetime.datetime(firstDay.year, firstDay.month, firstDay.day, 0, 0, 0)
        firstDay += datetime.timedelta(days=1)
        lastDay = firstDay + datetime.timedelta(days=targetWeartimeDays)
        lastDay += datetime.timedelta(microseconds=-1)
    #print 'considered data: ', firstDay, lastDay
    #finally close file reader as we're at the end of it
    epochReader.close()
    '''
    Now traverse through entire file, to identify episodes (i.e. start/end boundaries)
    '''
    xStd = 0.0
    yStd = 0.0
    zStd = 0.0
    epochReader = open(epochFile,'rU')
    #ignore the first <headerSize> lines
    for counter in range(headerSize):
        next(epochReader)
    #read in first <minFreq> instances from file to initialise queue
    while len(xQ) < minFreq:
        try:
            lineParts = next(epochReader).split(',') #read next from file
        except:
            sys.stderr.write('insufficient epoch data to identify episodes\n')
            break
        epochTime = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
        if epochTime >= firstDay and epochTime <= lastDay:
            xStd = float(lineParts[xIndex])
            yStd = float(lineParts[yIndex])
            zStd = float(lineParts[zIndex])
            timesQ.append(epochTime)
            xQ.append(xStd)
            yQ.append(yStd)
            zQ.append(zStd)
    #now traverse through rest of CSV file, to identify episodes
    for line in epochReader:
        #file reading item: add the latest item to the end of the running queue
        lineParts = line.split(',')
        epochTime = datetime.datetime.strptime(lineParts[timeCol],timeFormat)
        if epochTime >= firstDay and epochTime <= lastDay:
            xStd = float(lineParts[xIndex])
            yStd = float(lineParts[yIndex])
            zStd = float(lineParts[zIndex])
            timesQ.append(epochTime)
            xQ.append(xStd)
            yQ.append(yStd)
            zQ.append(zStd)
            #remove first item from queue to maintain size
{'fraction_non_alphanumeric': 0.09185612264954465, 'fraction_numerical': 0.012841512153574002, 'mean_word_length': 2.629964328180737, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1957202', 'n_tokens_mistral': 4649, 'n_tokens_neox': 4471, 'n_words': 1525}