<commit_before>from contentbase import (
    calculated_property
)

from .base import (
    paths_filtered_by_status,
)


class CalculatedBiosampleSlims:
    @calculated_property(condition='biosample_term_id', schema={
        "title": "Organ slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def organ_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['organs']
        return []

    @calculated_property(condition='biosample_term_id', schema={
        "title": "System slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def system_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['systems']
        return []

    @calculated_property(condition='biosample_term_id', schema={
        "title": "Developmental slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def developmental_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['developmental']
        return []


class CalculatedBiosampleSynonyms:
    @calculated_property(condition='biosample_term_id', schema={
        "title": "Biosample synonyms",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def biosample_synonyms(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['synonyms']
        return []


class CalculatedAssaySynonyms:
    @calculated_property(condition='assay_term_id', schema={
        "title": "Assay synonyms",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def assay_synonyms(self, registry, assay_term_id):
        if assay_term_id in registry['ontology']:
            return registry['ontology'][assay_term_id]['synonyms'] + [
                registry['ontology'][assay_term_id]['name'],
            ]
        return []


class CalculatedFileSetBiosample:
    @calculated_property(schema={
        "title": "Biosample term name",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def biosample_term_name(self, request, related_files):
        biosamples = []
        if related_files:
            for path in related_files:
                related_file = request.embed(path, '@@object')
                dataset = request.embed(related_file['dataset'], '@@object')
                if 'biosample_term_name' in dataset:
                    biosamples.append(dataset['biosample_term_name'])
            return list(set(biosamples))
        return []

    @calculated_property(schema={
        "title": "Biosample type",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def biosample_type(self, request, related_files):
        biosamples = []
        if related_files:
            for path in related_files:
                related_file = request.embed(path, '@@object')
                dataset = request.embed(related_file['dataset'], '@@object')
                if 'biosample_type' in dataset:
                    biosamples.append(dataset['biosample_type'])
            return list(set(biosamples))
        return []

    @calculated_property(schema={
        "title": "Organism",
        "type": "array",
        "items": {
            "type": ['string', 'object'],
            "linkTo": "Organism"
        },
    })
    def organism(self, request, related_files):
        organisms = []
        if related_files:
            for path in related_files:
                related_file = request.embed(path, '@@object')
                dataset = request.embed(related_file['dataset'], '@@object')
                if 'replicate' in dataset:
                    rep = request.embed(related_file['replicate'], '@@object')
                    if 'library' in rep:
                        lib = request.embed(rep['library'], '@@object')
                        if 'biosample' in lib:
                            bio = request.embed(lib['biosample'], '@@object')
                            if 'organism' in bio:
                                organisms.append(bio['organism'])
            if organisms:
                return paths_filtered_by_status(request, list(set(organisms)))


class CalculatedFileSetAssay:
    @calculated_property(schema={
        "title": "Assay term name",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def assay_term_name(self, request, related_files):
        assays = []
        if related_files:
            for path in related_files:
                related_file = request.embed(path, '@@object')
                dataset = request.embed(related_file['dataset'], '@@object')
                if 'assay_term_name' in dataset:
                    assays.append(dataset['assay_term_name'])
            return list(set(assays))
        return []

    @calculated_property(schema={
        "title": "Assay term id",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def assay_term_id(self, request, related_files):
        assays = []
        if related_files:
            for path in related_files:
                related_file = request.embed(path, '@@object')
                dataset = request.embed(related_file['dataset'], '@@object')
                if 'assay_term_id' in dataset:
                    assays.append(dataset['assay_term_id'])
            return list(set(assays))<commit_msg>Add a limit for number of files to calculate FileSet properties from<commit_after>from contentbase import (
    calculated_property
)

from .base import (
    paths_filtered_by_status,
)


class CalculatedBiosampleSlims:
    @calculated_property(condition='biosample_term_id', schema={
        "title": "Organ slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def organ_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['organs']
        return []

    @calculated_property(condition='biosample_term_id', schema={
        "title": "System slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def system_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['systems']
        return []

    @calculated_property(condition='biosample_term_id', schema={
        "title": "Developmental slims",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def developmental_slims(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['developmental']
        return []


class CalculatedBiosampleSynonyms:
    @calculated_property(condition='biosample_term_id', schema={
        "title": "Biosample synonyms",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def biosample_synonyms(self, registry, biosample_term_id):
        if biosample_term_id in registry['ontology']:
            return registry['ontology'][biosample_term_id]['synonyms']
        return []


class CalculatedAssaySynonyms:
    @calculated_property(condition='assay_term_id', schema={
        "title": "Assay synonyms",
        "type": "array",
        "items": {
            "type": "string",
        },
    })
    def assay_synonyms(self, registry, assay_term_id):
        if assay_term_id in registry['ontology']:
            return registry['ontology'][assay_term_id]['synonyms'] + [
                registry['ontology'][assay_term_id]['name'],
            ]
        return []


class CalculatedFileSetBiosample:
    @calculated_property(schema={
        "title": "Biosample term name",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def biosample_term_name(self, request, related_files):
        biosamples = []
        if related_files:
            for idx, path in enumerate(related_files):
                # Need to cap this due to the large numbers of files in related_files
                if idx < 100:
                    related_file = request.embed(path, '@@object')
                    dataset = request.embed(related_file['dataset'], '@@object')
                    if 'biosample_term_name' in dataset:
                        biosamples.append(dataset['biosample_term_name'])
            return list(set(biosamples))
        return []

    @calculated_property(schema={
        "title": "Biosample type",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def biosample_type(self, request, related_files):
        biosamples = []
        if related_files:
            for idx, path in enumerate(related_files):
                # Need to cap this due to the large numbers of files in related_files
                if idx < 100:
                    related_file = request.embed(path, '@@object')
                    dataset = request.embed(related_file['dataset'], '@@object')
                    if 'biosample_type' in dataset:
                        biosamples.append(dataset['biosample_type'])
            return list(set(biosamples))
        return []

    @calculated_property(schema={
        "title": "Organism",
        "type": "array",
        "items": {
            "type": ['string', 'object'],
            "linkTo": "Organism"
        },
    })
    def organism(self, request, related_files):
        organisms = []
        if related_files:
            for idx, path in enumerate(related_files):
                # Need to cap this due to the large numbers of files in related_files
                if idx < 100:
                    related_file = request.embed(path, '@@object')
                    dataset = request.embed(related_file['dataset'], '@@object')
                    if 'replicate' in dataset:
                        rep = request.embed(related_file['replicate'], '@@object')
                        if 'library' in rep:
                            lib = request.embed(rep['library'], '@@object')
                            if 'biosample' in lib:
                                bio = request.embed(lib['biosample'], '@@object')
                                if 'organism' in bio:
                                    organisms.append(bio['organism'])
            if organisms:
                return paths_filtered_by_status(request, list(set(organisms)))


class CalculatedFileSetAssay:
    @calculated_property(schema={
        "title": "Assay term name",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def assay_term_name(self, request, related_files):
        assays = []
        if related_files:
            for idx, path in enumerate(related_files):
                # Need to cap this due to the large numbers of files in related_files
                if idx < 100:
                    related_file = request.embed(path, '@@object')
                    dataset = request.embed(related_file['dataset'], '@@object')
                    if 'assay_term_name' in dataset:
                        assays.append(dataset['assay_term_name'])
            return list(set(assays))
        return []

    @calculated_property(schema={
        "title": "Assay term id",
        "type": "array",
        "items": {
            "type": ['string'],
        },
    })
    def assay_term_id(self, request, related_files):
        assays = set()
        if related_files:
            for idx, path in enumerate(related_files):
                # Need to cap this due to the large numbers of files in related_files
                if idx < 100:
                    related_file = request.embed(path, '@@object')
                    dataset = request.embed(related_file['dataset'], '@@object')
                    if 'assay_term_id' in dataset:
                        if dataset['assay_term_id'] not in assays:
                            assays.add(dataset['assay_term_id'])
            return list(assays)
{'fraction_non_alphanumeric': 0.12387905721768114, 'fraction_numerical': 0.0011903817157368462, 'mean_word_length': 1.9876718824087245, 'pattern_counts': {'":': 82, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6980556', 'n_tokens_mistral': 3627, 'n_tokens_neox': 3436, 'n_words': 743}