<commit_before>import random
import threading
import numpy as np

import datetime
from rules import SendTriggers
from targets import Target
import config


class Stage:
    """"""

    def __init__(self, classifier, target_space, send_triggers, source=config.site_name):
        self.classifier_queue = StageClassifierQueue(classifier, target_space, self, send_triggers)
        self.target_space = target_space
        self.send_triggers = send_triggers
        self.data_queues = {}
        for stream in config.data_streams:
            self.data_queues[stream] = []
        # NIMS grouped by target_id, so change to dict {target_id: [indices]}
        self.data_queues['nims'] = {}
        self.recent_targets = []
        #self.startStageProcessing()

    def processDataBeforeStage(self, stream, data):
        """Performs whatever preprocessing necessitated for data from a
        particular stream, adds data to appropriate target list, then returns
        index for added data in TargetSpace.

        Assumes 'nims' passes a list inside a dict with different tracks.
        """
        if stream == 'adcp':
            data = [datetime.datetime.fromtimestamp(data[0]), data[1], data[2]]
            self.target_space.tables[stream].append(data)
            return len(self.target_space.tables[stream]) - 1<commit_msg>FOR TESTING USE ONLY: adds adcp data so it doesn't break classifier<commit_after>import random
import threading
import numpy as np

import datetime
from rules import SendTriggers
from targets import Target
import config


class Stage:
    """"""

    def __init__(self, classifier, target_space, send_triggers, source=config.site_name):
        self.classifier_queue = StageClassifierQueue(classifier, target_space, self, send_triggers)
        self.target_space = target_space
        self.send_triggers = send_triggers
        self.data_queues = {}
        for stream in config.data_streams:
            self.data_queues[stream] = []
        # NIMS grouped by target_id, so change to dict {target_id: [indices]}
        self.data_queues['nims'] = {}
        self.recent_targets = []
        # Adds ADCP (necessary for testing when not connected to ADCP)
        unixtime = (datetime.datetime.utcnow() - datetime.datetime(1970,1,1))
        self.addDataToStage('adcp', [unixtime.days*24*60*60 + unixtime.seconds, 1.2, 4.5])
        #self.startStageProcessing()

    def processDataBeforeStage(self, stream, data):
        """Performs whatever preprocessing necessitated for data from a
        particular stream, adds data to appropriate target list, then returns
        index for added data in TargetSpace.

        Assumes 'nims' passes a list inside a dict with different tracks.
        """
        if stream == 'adcp':
            data = [datetime.datetime.fromtimestamp(data[0]), data[1], data[2]]
            self.target_space.tables[stream].append(data)
            return len(self.target_space.tables[stream]) - 1
{'fraction_non_alphanumeric': 0.0907244414353419, 'fraction_numerical': 0.008124576844955992, 'mean_word_length': 3.087136929460581, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6371527', 'n_tokens_mistral': 842, 'n_tokens_neox': 814, 'n_words': 240}