<commit_before>import os

from reflowrestclient import utils
import numpy
import flowutils

BASE_DIR = '/var/tmp/ReFlow-data/'


class ProcessRequest(object):
    """
    A process request from a ReFlow server.
    Contains the list of Samples and a list of process input key/values
    """
    def __init__(self, host, token, pr_dict):
        self.host = host
        self.token = token
        self.process_request_id = pr_dict['id']
        self.directory = "%s%s/process_requests/%s/" % (
            BASE_DIR,
            self.host,
            self.process_request_id)
        self.process_id = pr_dict['process']
        self.inputs = pr_dict['inputs']
        self.use_fcs = False
        self.samples = list()
        self.__setup()

    def __setup(self):
        """
        Essentially populates the list of samples based on the inputs.
        The following keys will be used to filter the samples:
            'project_panel'
            'site' ***
            'site_panel' ***
            'subject' ***
            'visit' ***
            'specimen' ***
            'stimulation' ***
            'storage' ***
            'cytometer' ***
            'acquisition_date' ***
        Note that these keys(***) can appear more than once, i.e. more than one
        site panel.
        """

        if not os.path.exists(self.directory):
            os.makedirs(self.directory)

        project_panel = None
        sites = list()
        site_panels = list()
        subjects = list()
        visits = list()
        specimens = list()
        storages = list()
        pretreatments = list()
        stimulations = list()
        cytometers = list()
        acquisition_dates = list()

        for i in self.inputs:
            if i['key'] == 'project_panel' and not project_panel:
                project_panel = i['value']
            elif i['key'] == 'use_fcs':
                if i['value'] == 'True':
                    self.use_fcs = True
            elif i['key'] == 'site':
                sites.append(i['value'])
            elif i['key'] == 'site_panel':
                site_panels.append(i['value'])
            elif i['key'] == 'subject':
                subjects.append(i['value'])
            elif i['key'] == 'visit':
                visits.append(i['value'])
            elif i['key'] == 'specimen':
                specimens.append(i['value'])
            elif i['key'] == 'storage':
                storages.append(i['value'])
            elif i['key'] == 'pretreatment':
                pretreatments.append(i['value'])
            elif i['key'] == 'stimulation':
                stimulations.append(i['value'])
            elif i['key'] == 'cytometer':
                cytometers.append(i['value'])
            elif i['key'] == 'acquisition_date':
                acquisition_dates.append(i['value'])

        # start with the project panel and get all those samples
        response = utils.get_samples(
            self.host,
            self.token,
            project_panel_pk=project_panel)
        if not 'data' in response:
            return

        for sample_dict in response['data']:
            sample = Sample(self.host, self.process_request_id, sample_dict)

            if len(sites) > 0:
                if sample.site_id not in sites:
                    continue
            if len(site_panels) > 0:
                if sample.site_panel_id not in site_panels:
                    continue
            if len(subjects) > 0:
                if sample.subject_id not in subjects:
                    continue
            if len(visits) > 0:
                if sample.visit_id not in visits:
                    continue
            if len(specimens) > 0:
                if sample.specimen_id not in specimens:
                    continue
            if len(storages) > 0:
                if sample.storage not in storages:
                    continue
            if len(pretreatments) > 0:
                if sample.pretreatment not in pretreatments:
                    continue
            if len(stimulations) > 0:
                if sample.stimulation_id not in stimulations:
                    continue
            if len(cytometers) > 0:
                if sample.cytometer_id not in cytometers:
                    continue
            if len(acquisition_dates) > 0:
                if sample.acquisition_date not in acquisition_dates:
                    continue

            self.samples.append(sample)

    def download_samples(self):
        if self.use_fcs:
            for s in self.samples:
                s.download_fcs(self.token)
        else:
            for s in self.samples:
                s.download_subsample(self.token)

    def compensate_samples(self):
        directory = self.directory + 'comp/'
        if self.use_fcs:
            for s in self.samples:
                s.conpensate_fcs(self.token, directory)
        else:
            for s in self.samples:
                s.compensate_subsample(self.token, directory)


class Sample(object):
    """
    Represents the FCS sample downloaded from a ReFlow server.
    Used by a Worker to manage downloaded samples related to a
    ReFlow ProcessRequest
    """
    def __init__(self, host, process_request_id, sample_dict):<commit_msg>use strings for the sample filters<commit_after>import os

from reflowrestclient import utils
import numpy
import flowutils

BASE_DIR = '/var/tmp/ReFlow-data/'


class ProcessRequest(object):
    """
    A process request from a ReFlow server.
    Contains the list of Samples and a list of process input key/values
    """
    def __init__(self, host, token, pr_dict):
        self.host = host
        self.token = token
        self.process_request_id = pr_dict['id']
        self.directory = "%s%s/process_requests/%s/" % (
            BASE_DIR,
            self.host,
            self.process_request_id)
        self.process_id = pr_dict['process']
        self.inputs = pr_dict['inputs']
        self.use_fcs = False
        self.samples = list()
        self.__setup()

    def __setup(self):
        """
        Essentially populates the list of samples based on the inputs.
        The following keys will be used to filter the samples:
            'project_panel'
            'site' ***
            'site_panel' ***
            'subject' ***
            'visit' ***
            'specimen' ***
            'stimulation' ***
            'storage' ***
            'cytometer' ***
            'acquisition_date' ***
        Note that these keys(***) can appear more than once, i.e. more than one
        site panel.
        """

        if not os.path.exists(self.directory):
            os.makedirs(self.directory)

        project_panel = None
        sites = list()
        site_panels = list()
        subjects = list()
        visits = list()
        specimens = list()
        storages = list()
        pretreatments = list()
        stimulations = list()
        cytometers = list()
        acquisition_dates = list()

        for i in self.inputs:
            if i['key'] == 'project_panel' and not project_panel:
                project_panel = i['value']
            elif i['key'] == 'use_fcs':
                if i['value'] == 'True':
                    self.use_fcs = True
            elif i['key'] == 'site':
                sites.append(i['value'])
            elif i['key'] == 'site_panel':
                site_panels.append(i['value'])
            elif i['key'] == 'subject':
                subjects.append(i['value'])
            elif i['key'] == 'visit':
                visits.append(i['value'])
            elif i['key'] == 'specimen':
                specimens.append(i['value'])
            elif i['key'] == 'storage':
                storages.append(i['value'])
            elif i['key'] == 'pretreatment':
                pretreatments.append(i['value'])
            elif i['key'] == 'stimulation':
                stimulations.append(i['value'])
            elif i['key'] == 'cytometer':
                cytometers.append(i['value'])
            elif i['key'] == 'acquisition_date':
                acquisition_dates.append(i['value'])

        # start with the project panel and get all those samples
        response = utils.get_samples(
            self.host,
            self.token,
            project_panel_pk=project_panel)
        if not 'data' in response:
            return

        for sample_dict in response['data']:
            sample = Sample(self.host, self.process_request_id, sample_dict)
            
            # keep in mind the key/value inputs are strings
            if len(sites) > 0:
                if not str(sample.site_id) in sites:
                    continue
            if len(site_panels) > 0:
                if not str(sample.site_panel_id) in site_panels:
                    continue
            if len(subjects) > 0:
                if not str(sample.subject_id) in subjects:
                    continue
            if len(visits) > 0:
                if not str(sample.visit_id) in visits:
                    continue
            if len(specimens) > 0:
                if not str(sample.specimen_id) in specimens:
                    continue
            if len(storages) > 0:
                if not str(sample.storage) in storages:
                    continue
            if len(pretreatments) > 0:
                if not str(sample.pretreatment) in pretreatments:
                    continue
            if len(stimulations) > 0:
                if not str(sample.stimulation_id) in stimulations:
                    continue
            if len(cytometers) > 0:
                if not str(sample.cytometer_id) in cytometers:
                    continue
            if len(acquisition_dates) > 0:
                if not str(sample.acquisition_date) in acquisition_dates:
                    continue

            self.samples.append(sample)

    def download_samples(self):
        if self.use_fcs:
            for s in self.samples:
                s.download_fcs(self.token)
        else:
            for s in self.samples:
                s.download_subsample(self.token)

    def compensate_samples(self):
        directory = self.directory + 'comp/'
        if self.use_fcs:
            for s in self.samples:
                s.conpensate_fcs(self.token, directory)
        else:
            for s in self.samples:
                s.compensate_subsample(self.token, directory)


class Sample(object):
    """
    Represents the FCS sample downloaded from a ReFlow server.
    Used by a Worker to manage downloaded samples related to a
    ReFlow ProcessRequest
    """
    def __init__(self, host, process_request_id, sample_dict):
{'fraction_non_alphanumeric': 0.10098176718092566, 'fraction_numerical': 0.0018700327255726976, 'mean_word_length': 1.5551839464882944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1197383', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2816, 'n_words': 762}