<commit_before>import itertools
import math
from configparser import ConfigParser
import numpy as np
import pandas

from utils import Phase  # noqa

_entry_wait = 2  # min seconds between counted entries to top
_freeze_min_time = 2  # min seconds to count lack of motion as a "freeze"
_freeze_max_speed = 0.1  # maximum speed to still consider a "freeze"


def groups_where(vals):
    '''Return group start indices and lengths for all
    groups of consecutive true entries in vals.

    Adapted from: http://stackoverflow.com/q/4651683
    '''
    # where: convert vals from true/false to 1/0
    # TODO: necessary? likely removable / work-aroundable
    where = np.where(vals, 1, 0)
    # diffs: 1 if start of 1s, -1 if start of 0s, 0 otherwise
    diffs = np.concatenate( ([where[0]], np.diff(where)) )
    # change_idx: indexes of starts of runs plus index at end of list
    change_idx = np.concatenate( (np.where(diffs)[0], [len(where)-1]) )
    # starts: start of 1 runs
    starts = np.where(diffs > 0)[0]
    # lens: length of 1 runs (every other length is for a 1 run)
    lens = np.diff(change_idx)[::2]

    return starts, lens


class TrackProcessor(object):
    def __init__(self, trackfile, just_raw_data=False):
        self.trackfile = trackfile
        self.setupfile = trackfile.replace("-track.csv", "-setup.txt")

        self._read_trackfile()
        if not just_raw_data:
            self._generate_columns()
        self._read_setupfile()

    def _read_trackfile(self):
        colnames = ['time', 'status', 'x', 'y', 'numpts', '_1', '_2']
        self.df = pandas.read_csv(self.trackfile, header=None, names=colnames, index_col=0)

    def _generate_columns(self):
        ''' Calculate several auxiliary columns from the raw data
            read from CSV by _read_trackfile().
        '''
        # shorthand
        df = self.df

        # calculate derivatives and other derived values
        df['dx'] = df.x.diff()
        df['dy'] = df.y.diff()
        df['dt'] = df.index.to_series().diff()

        if df.dt.min() < 0:
            time_travel_indices = np.where(df.dt < 0)[0]
            print("\n[31mWARNING:[m Time travel detected!\n[31mNegative time deltas in following indices:[m " +
                  " ".join(str(x) for x in time_travel_indices) +
                  "\n"
                  )

        df['dist'] = np.linalg.norm(df[['dx','dy']], axis=1)
        df['speed'] = df.dist / df.dt
        df['theta'] = np.arctan2(df.dy, df.dx)  # reversed params are correct for numpy.arctan2

        # produce boolean arrays
        df['lost'] = (df.status == 'lost') | (df.status == 'init')
        df['missing'] = (df.status == 'missing')
        df['sketchy'] = (df.numpts > 1)
        # valid if this index *and* previous are both not 'lost' or 'missing'
        # need *both* as many columns are diffs relying on previous value
        df['valid'] = ~df.lost & ~df.missing & ~np.roll(df.lost, 1) & ~np.roll(df.missing, 1)

        # TODO: base frozen off of some rolling average speed?
        df['frozen'] = df.speed < _freeze_max_speed

    def _read_setupfile(self):
        self.config = ConfigParser()
        self.config.read(self.setupfile)
        # parse phases_data
        if 'phases' in self.config and 'phases_data' in self.config['phases']:
            phases_data_str = self.config['phases']['phases_data']
            self.phase_list = eval(phases_data_str)  # uses Phase namedtuple imported from utils
        else:
            self.phase_list = None

    def num_phases(self):
        return len(self.phase_list)

    def phase_starts(self):
        return list(itertools.accumulate([0] + [phase.length for phase in self.phase_list]))

    def get_setup(self, sections):
        ret = {}
        ret['Setup file'] = self.setupfile

        for section in sections:
            if section not in self.config:
                continue

            for key in self.config[section]:
                ret[section + "__" + key] = self.config[section][key]

        return ret

    @property
    def len_minutes(self):
        # -0.1 so the *one* sample in the new minute (e.g. time=3600.0456)
        # still just counts as 60 minutes
        return int(math.ceil((self.df.index.max()-0.1) / 60.0))

    def get_stats(self, include_phases=False):
        ret = {}
        ret.update(self._get_stats_time_range(minutes='all'))
        # only include per-phase stats if we have more than one phase
        if include_phases and self.phase_list is not None and len(self.phase_list) > 1:
            start_min = 0
            for phase in self.phase_list:
                phase_stats = self._get_stats_time_range(
                                minutes=(start_min, start_min+phase.length)
                              )
                ret.update(phase_stats)
                start_min += phase.length
        return ret<commit_msg>Fix for crashing on analysis w/ 1 sample beyond minute boundary.<commit_after>import itertools
import math
from configparser import ConfigParser
import numpy as np
import pandas

from utils import Phase  # noqa

_entry_wait = 2  # min seconds between counted entries to top
_freeze_min_time = 2  # min seconds to count lack of motion as a "freeze"
_freeze_max_speed = 0.1  # maximum speed to still consider a "freeze"


def groups_where(vals):
    '''Return group start indices and lengths for all
    groups of consecutive true entries in vals.

    Adapted from: http://stackoverflow.com/q/4651683
    '''
    # where: convert vals from true/false to 1/0
    # TODO: necessary? likely removable / work-aroundable
    where = np.where(vals, 1, 0)
    # diffs: 1 if start of 1s, -1 if start of 0s, 0 otherwise
    diffs = np.concatenate( ([where[0]], np.diff(where)) )
    # change_idx: indexes of starts of runs plus index at end of list
    change_idx = np.concatenate( (np.where(diffs)[0], [len(where)-1]) )
    # starts: start of 1 runs
    starts = np.where(diffs > 0)[0]
    # lens: length of 1 runs (every other length is for a 1 run)
    lens = np.diff(change_idx)[::2]

    return starts, lens


class TrackProcessor(object):
    def __init__(self, trackfile, just_raw_data=False):
        self.trackfile = trackfile
        self.setupfile = trackfile.replace("-track.csv", "-setup.txt")

        self._read_trackfile()
        if not just_raw_data:
            self._generate_columns()
        self._read_setupfile()

    def _read_trackfile(self):
        colnames = ['time', 'status', 'x', 'y', 'numpts', '_1', '_2']
        self.df = pandas.read_csv(self.trackfile, header=None, names=colnames, index_col=0)

    def _generate_columns(self):
        ''' Calculate several auxiliary columns from the raw data
            read from CSV by _read_trackfile().
        '''
        # shorthand
        df = self.df

        # calculate derivatives and other derived values
        df['dx'] = df.x.diff()
        df['dy'] = df.y.diff()
        df['dt'] = df.index.to_series().diff()

        if df.dt.min() < 0:
            time_travel_indices = np.where(df.dt < 0)[0]
            print("\n[31mWARNING:[m Time travel detected!\n[31mNegative time deltas in following indices:[m " +
                  " ".join(str(x) for x in time_travel_indices) +
                  "\n"
                  )

        df['dist'] = np.linalg.norm(df[['dx','dy']], axis=1)
        df['speed'] = df.dist / df.dt
        df['theta'] = np.arctan2(df.dy, df.dx)  # reversed params are correct for numpy.arctan2

        # produce boolean arrays
        df['lost'] = (df.status == 'lost') | (df.status == 'init')
        df['missing'] = (df.status == 'missing')
        df['sketchy'] = (df.numpts > 1)
        # valid if this index *and* previous are both not 'lost' or 'missing'
        # need *both* as many columns are diffs relying on previous value
        df['valid'] = ~df.lost & ~df.missing & ~np.roll(df.lost, 1) & ~np.roll(df.missing, 1)

        # TODO: base frozen off of some rolling average speed?
        df['frozen'] = df.speed < _freeze_max_speed

    def _read_setupfile(self):
        self.config = ConfigParser()
        self.config.read(self.setupfile)
        # parse phases_data
        if 'phases' in self.config and 'phases_data' in self.config['phases']:
            phases_data_str = self.config['phases']['phases_data']
            self.phase_list = eval(phases_data_str)  # uses Phase namedtuple imported from utils
        else:
            self.phase_list = None

    def num_phases(self):
        return len(self.phase_list)

    def phase_starts(self):
        return list(itertools.accumulate([0] + [phase.length for phase in self.phase_list]))

    def get_setup(self, sections):
        ret = {}
        ret['Setup file'] = self.setupfile

        for section in sections:
            if section not in self.config:
                continue

            for key in self.config[section]:
                ret[section + "__" + key] = self.config[section][key]

        return ret

    @property
    def len_minutes(self):
        # Take second-to-last index so if just *one* sample in the new minute
        # (e.g. time=3600.0456), it still just counts as 60 minutes
        return int(math.ceil(self.df.index[-2] / 60.0))

    def get_stats(self, include_phases=False):
        ret = {}
        ret.update(self._get_stats_time_range(minutes='all'))
        # only include per-phase stats if we have more than one phase
        if include_phases and self.phase_list is not None and len(self.phase_list) > 1:
            start_min = 0
            for phase in self.phase_list:
                phase_stats = self._get_stats_time_range(
                                minutes=(start_min, start_min+phase.length)
                              )
                ret.update(phase_stats)
                start_min += phase.length
        return ret
{'fraction_non_alphanumeric': 0.11504874086108854, 'fraction_numerical': 0.012997562956945572, 'mean_word_length': 2.3375127075567605, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6157448', 'n_tokens_mistral': 3150, 'n_tokens_neox': 2975, 'n_words': 871}