<commit_before>"""Abstract collection for experiment and integration of all experiment types."""

from snovault import (
    abstract_collection,
    calculated_property,
    collection,
    load_schema
)
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    paths_filtered_by_status,
    get_item_if_you_can,
    ALLOW_SUBMITTER_ADD,
    lab_award_attribution_embed_list
)

EXP_CATEGORIZER_SCHEMA = {
    "title": "Categorizer",
    "description": "Fields used as an additional level of categorization for an experiment",
    "type": "object",
    "properties": {
        "field": {
            "type": "string",
            "description": "The name of the field as to be displayed in tables."
        },
        "value": {
            "type": "string",
            "description": "The value displayed for the field"
        },
        "combined": {
            "type": "string",
            "description": "Combined field:value string used for categorization of this experiment."
        }
    }
}


@abstract_collection(
    name='experiments',
    unique_key='accession',
    acl=ALLOW_SUBMITTER_ADD,
    properties={
        'title': "Experiments",
        'description': 'Listing of all types of experiments.',
    })
class Experiment(Item):
    """The main experiment class."""

    base_types = ['Experiment'] + Item.base_types
    schema = load_schema('encoded:schemas/experiment.json')
    name_key = 'accession'
    rev = {
        'experiment_sets': ('ExperimentSet', 'experiments_in_set'),
    }
    embedded_list = lab_award_attribution_embed_list + [
        "experiment_sets.experimentset_type",
        "experiment_sets.@type",
        "experiment_sets.accession",
        "produced_in_pub.title",
        "produced_in_pub.abstract",
        "produced_in_pub.journal",
        "produced_in_pub.authors",
        "publications_of_exp.title",
        "publications_of_exp.abstract",
        "publications_of_exp.journal",
        "publications_of_exp.authors",
        "biosample.accession",
        "biosample.modifications_summary",
        "biosample.treatments_summary",
        "biosample.biosource_summary",
        "biosample.biosample_type",
        "biosample.biosource.biosource_type",
        "biosample.biosource.cell_line.slim_terms",
        "biosample.biosource.cell_line.synonyms",
        "biosample.biosource.tissue.slim_terms",
        "biosample.biosource.tissue.synonyms",
        "biosample.biosource.individual.organism.name",
        'biosample.modifications.modification_type',
        'biosample.modifications.display_title',
        'biosample.treatments.treatment_type',
        'biosample.treatments.display_title',
        'biosample.badges.*',

        "files.href",
        "files.accession",
        "files.uuid",
        "files.file_size",
        "files.upload_key",
        "files.file_format",
        "files.file_classification",
        "files.paired_end",
        'files.badges.*',

        "processed_files.href",
        "processed_files.accession",
        "processed_files.uuid",
        "processed_files.file_size",
        "processed_files.upload_key",
        "processed_files.file_format",
        "processed_files.file_classification",

        "other_processed_files.files.href",
    ]

    def generate_mapid(self, experiment_type, num):
        delim = '_'
        mapid = str(type(self).__name__)
        mapid = mapid + delim + ''.join(experiment_type.split())
        return mapid + delim + str(num)

    def has_bad_status(self, status):
        bad_statuses = ["revoked", "deleted", "obsolete", "replaced"]
        return status in bad_statuses

    def find_current_sop_map(self, experiment_type, sop_coll=None):
        maps = []
        suffnum = 1
        mapid = self.generate_mapid(experiment_type, suffnum)
        if sop_coll is not None:
            while(True):
                m = sop_coll.get(mapid)
                if not m:
                    break
                if not self.has_bad_status(m.properties.get('status')):
                    maps.append(m)<commit_msg>Trying to fix test that passes locally.<commit_after>"""Abstract collection for experiment and integration of all experiment types."""

from snovault import (
    abstract_collection,
    calculated_property,
    collection,
    load_schema
)
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    paths_filtered_by_status,
    get_item_if_you_can,
    ALLOW_SUBMITTER_ADD,
    lab_award_attribution_embed_list
)

EXP_CATEGORIZER_SCHEMA = {
    "title": "Categorizer",
    "description": "Fields used as an additional level of categorization for an experiment",
    "type": "object",
    "properties": {
        "field": {
            "type": "string",
            "description": "The name of the field as to be displayed in tables."
        },
        "value": {
            "type": "string",
            "description": "The value displayed for the field"
        },
        "combined": {
            "type": "string",
            "description": "Combined field:value string used for categorization of this experiment."
        }
    }
}


@abstract_collection(
    name='experiments',
    unique_key='accession',
    acl=ALLOW_SUBMITTER_ADD,
    properties={
        'title': "Experiments",
        'description': 'Listing of all types of experiments.',
    })
class Experiment(Item):
    """The main experiment class."""

    base_types = ['Experiment'] + Item.base_types
    schema = load_schema('encoded:schemas/experiment.json')
    name_key = 'accession'
    rev = {
        'experiment_sets': ('ExperimentSet', 'experiments_in_set'),
    }
    embedded_list = lab_award_attribution_embed_list + [
        "experiment_sets.experimentset_type",
        "experiment_sets.@type",
        "experiment_sets.accession",
        "produced_in_pub.title",
        "produced_in_pub.abstract",
        "produced_in_pub.journal",
        "produced_in_pub.authors",
        "publications_of_exp.title",
        "publications_of_exp.abstract",
        "publications_of_exp.journal",
        "publications_of_exp.authors",
        "biosample.accession",
        "biosample.modifications_summary",
        "biosample.treatments_summary",
        "biosample.biosource_summary",
        "biosample.biosample_type",
        "biosample.biosource.biosource_type",
        "biosample.biosource.cell_line.slim_terms",
        "biosample.biosource.cell_line.synonyms",
        "biosample.biosource.tissue.slim_terms",
        "biosample.biosource.tissue.synonyms",
        "biosample.biosource.individual.organism.name",
        'biosample.modifications.modification_type',
        'biosample.modifications.display_title',
        'biosample.treatments.treatment_type',
        'biosample.treatments.display_title',
        'biosample.badges.*',

        "files.href",
        "files.accession",
        "files.uuid",
        "files.file_size",
        "files.upload_key",
        "files.file_format",
        "files.file_classification",
        "files.paired_end",
        'files.badges.*',

        "processed_files.href",
        "processed_files.accession",
        "processed_files.uuid",
        "processed_files.file_size",
        "processed_files.upload_key",
        "processed_files.file_format",
        "processed_files.file_classification",

        "other_processed_files.files.href"
    ]

    def generate_mapid(self, experiment_type, num):
        delim = '_'
        mapid = str(type(self).__name__)
        mapid = mapid + delim + ''.join(experiment_type.split())
        return mapid + delim + str(num)

    def has_bad_status(self, status):
        bad_statuses = ["revoked", "deleted", "obsolete", "replaced"]
        return status in bad_statuses

    def find_current_sop_map(self, experiment_type, sop_coll=None):
        maps = []
        suffnum = 1
        mapid = self.generate_mapid(experiment_type, suffnum)
        if sop_coll is not None:
            while(True):
                m = sop_coll.get(mapid)
                if not m:
                    break
                if not self.has_bad_status(m.properties.get('status')):
                    maps.append(m)
{'fraction_non_alphanumeric': 0.11201952410006101, 'fraction_numerical': 0.00024405125076266016, 'mean_word_length': 2.5696864111498257, 'pattern_counts': {'":': 26, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '645261', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2470, 'n_words': 435}