<commit_before>from rdflib import ConjunctiveGraph, exceptions, Namespace
from rdflib import RDFS, RDF, BNode
from rdflib.collection import Collection
import json

EPILOG = __doc__

OWLNS = Namespace("http://www.w3.org/2002/07/owl#")
OBO_OWL = Namespace("http://www.geneontology.org/formats/oboInOwl#")
EFO = Namespace("http://www.ebi.ac.uk/efo/")
OBO = Namespace("http://purl.obolibrary.org/obo/")

EFO_Synonym = EFO["alternative_term"]
OBO_Synonym = OBO["IAO_0000118"]
Synonym = OBO_OWL["hasExactSynonym"]
Ontology = OWLNS["Ontology"]

Restriction = OWLNS["Restriction"]
Class = OWLNS["Class"]
Thing = OWLNS["Thing"]
OnProperty = OWLNS["onProperty"]
SomeValuesFrom = OWLNS["someValuesFrom"]
IntersectionOf = OWLNS["intersectionOf"]

PART_OF = "http://purl.obolibrary.org/obo/BFO_0000050"
DEVELOPS_FROM = "http://purl.obolibrary.org/obo/RO_0002202"
HUMAN_TAXON = "http://purl.obolibrary.org/obo/NCBITaxon_9606"
HAS_PART = "http://purl.obolibrary.org/obo/BFO_0000051"
ACHIEVES_PLANNED_OBJECTIVE = "http://purl.obolibrary.org/obo/OBI_0000417"
DEFAULT_LANGUAGE = "en"

developental_slims = {
    'UBERON:0000926': 'mesoderm',
    'UBERON:0000924': 'ectoderm',
    'UBERON:0000925': 'endoderm'
}

system_slims = {
    'UBERON:0000383': 'musculature of body',
    'UBERON:0000949': 'endocrine system',
    'UBERON:0000990': 'reproductive system',
    'UBERON:0001004': 'respiratory system',
    'UBERON:0001007': 'digestive system',
    'UBERON:0001008': 'excretory system',
    'UBERON:0001009': 'circulatory system',
    'UBERON:0001434': 'skeletal system',
    'UBERON:0002405': 'immune system',
    'UBERON:0002416': 'integumental system',
    'UBERON:0001032': 'sensory system',
    'UBERON:0001017': 'central nervous system',
    'UBERON:0000010': 'peripheral nervous system'
}

organ_slims = {
    'UBERON:0002369': 'adrenal gland',
    'UBERON:0002110': 'gallbladder',
    'UBERON:0002106': 'spleen',
    'UBERON:0001173': 'billary tree',
    'UBERON:0001043': 'esophagus',
    'UBERON:0000004': 'nose',
    'UBERON:0000056': 'ureter',
    'UBERON:0000057': 'urethra',
    'UBERON:0000059': 'large intestine',
    'UBERON:0000165': 'mouth',
    'UBERON:0000945': 'stomach',
    'UBERON:0000948': 'heart',
    'UBERON:0000955': 'brain',
    'UBERON:0000970': 'eye',
    'UBERON:0000991': 'gonad',
    'UBERON:0001043': 'esophagus',
    'UBERON:0001255': 'urinary bladder',
    'UBERON:0001264': 'pancreas',
    'UBERON:0001474': 'bone element',
    'UBERON:0002003': 'peripheral nerve',
    'UBERON:0002048': 'lung',
    'UBERON:0002097': 'skin of body',
    'UBERON:0002107': 'liver',
    'UBERON:0000059': 'large intestine',
    'UBERON:0002108': 'small intestine',
    'UBERON:0002113': 'kidney',
    'UBERON:0002240': 'spinal cord',
    'UBERON:0002367': 'prostate gland',
    'UBERON:0002370': 'thymus',
    'UBERON:0003126': 'trachea',
    'UBERON:0001723': 'tongue',
    'UBERON:0001737': 'larynx',
    'UBERON:0006562': 'pharynx',
    'UBERON:0001103': 'diaphragm',
    'UBERON:0002185': 'bronchus',
    'UBERON:0000029': 'lymph node',
    'UBERON:0002391': 'lymph',
    'UBERON:0010133': 'neuroendocrine gland',
    'UBERON:0001132': 'parathyroid gland',
    'UBERON:0002046': 'thyroid gland',
    'UBERON:0001981': 'blood vessel',
    'UBERON:0001473': 'lymphatic vessel',
    'UBERON:0000178': 'blood',
    'UBERON:0002268': 'olfactory organ',
    'UBERON:0007844': 'cartilage element',
    'UBERON:0001690': 'ear',
    'UBERON:0001987': 'placenta',
    'UBERON:0001911': 'mammary gland',
    'UBERON:0001630': 'muscle organ',
    'UBERON:0000007': 'pituitary gland',
    'UBERON:0002370': 'thymus',
    'UBERON:0000478': 'extraembryonic structure'
}

assay_slims = {
    # Note shortened synonyms are provided
    'OBI:0000634': 'DNA methylation',  # 'DNA methylation profiling'
    'OBI:0000424': 'Transcription',  # 'transcription profiling'
    'OBI:0001398': 'DNA binding',  # "protein and DNA interaction"
    'OBI:0001854': 'RNA binding',  # "protein and RNA interaction"
    'OBI:0001917': '3D chromatin structure',  # 'chromosome conformation identification objective'
    'OBI:0000870': 'DNA accessibility',  # 'single-nucleotide-resolution nucleic acid structure mapping assay'
    'OBI:0001916': 'Replication timing',
    'OBI:0000435': 'Genotyping',
    'OBI:0000615': 'Proteomics',
}

slim_shims = {
    # this allows us to manually assign term X to slim Y while waiting for ontology updates
    'assay': {
        # DNA accessibility
        'OBI:0001924': 'DNA accessibility',  # 'OBI:0000870' / MNase-seq
        'OBI:0002039': 'DNA accessibility',  # 'OBI:0000870', / ATAC-seq
        'OBI:0001853': 'DNA accessibility',  # 'OBI:0000870', / DNase-seq
        'OBI:0001859': 'DNA accessibility',  # 'OBI:0000870', / OBI:0000424  / FAIRE-seq
        'OBI:0002042': '3D chromatin structure',  # 'OBI:0000870' (Hi-C)
        'OBI:0001848': '3D chromatin structure',  # ChIA-PET / OBI:000870
        'OBI:0001923': 'Proteomics',  # OBI:0000615': 'MS-MS'
        'OBI:0001849': 'Genotyping',  # OBI:0000435 (DNA-PET)
        'OBI:0002044': 'RNA binding',  # OBI:0001854 (RNA-Bind-N-Seq)
    }

}

preferred_name = {
    "OBI:0000626": "WGS",
    "OBI:0001247": "genotyping HTS",
    "OBI:0001332": "DNAme array",
    "OBI:0001335": "microRNA counts",
    "OBI:0001463": "RNA microarray",
    "OBI:0001863": "WGBS",
    "OBI:0001923": "MS-MS",
    "OBI:0001271": "RNA-seq",
    "OBI:0000716": "ChIP-seq",
    "OBI:0001853": "DNase-seq",
    "OBI:0001920": "Repli-seq",
    "OBI:0001864": "RAMPAGE",
    "OBI:0001393": "genotyping array",
    "OBI:0002042": "Hi-C",
}

category_slims = {
    'OBI:0000634': 'DNA methylation profiling',
    'OBI:0000424': 'transcription profiling',
    'OBI:0000435': 'genotyping',
    'OBI:0000615': 'proteomics',
    'OBI:0001916': 'replication',
    'OBI:0001398': "protein and DNA interaction",<commit_msg>slimming 3 RACE assays to transcription category<commit_after>from rdflib import ConjunctiveGraph, exceptions, Namespace
from rdflib import RDFS, RDF, BNode
from rdflib.collection import Collection
import json

EPILOG = __doc__

OWLNS = Namespace("http://www.w3.org/2002/07/owl#")
OBO_OWL = Namespace("http://www.geneontology.org/formats/oboInOwl#")
EFO = Namespace("http://www.ebi.ac.uk/efo/")
OBO = Namespace("http://purl.obolibrary.org/obo/")

EFO_Synonym = EFO["alternative_term"]
OBO_Synonym = OBO["IAO_0000118"]
Synonym = OBO_OWL["hasExactSynonym"]
Ontology = OWLNS["Ontology"]

Restriction = OWLNS["Restriction"]
Class = OWLNS["Class"]
Thing = OWLNS["Thing"]
OnProperty = OWLNS["onProperty"]
SomeValuesFrom = OWLNS["someValuesFrom"]
IntersectionOf = OWLNS["intersectionOf"]

PART_OF = "http://purl.obolibrary.org/obo/BFO_0000050"
DEVELOPS_FROM = "http://purl.obolibrary.org/obo/RO_0002202"
HUMAN_TAXON = "http://purl.obolibrary.org/obo/NCBITaxon_9606"
HAS_PART = "http://purl.obolibrary.org/obo/BFO_0000051"
ACHIEVES_PLANNED_OBJECTIVE = "http://purl.obolibrary.org/obo/OBI_0000417"
DEFAULT_LANGUAGE = "en"

developental_slims = {
    'UBERON:0000926': 'mesoderm',
    'UBERON:0000924': 'ectoderm',
    'UBERON:0000925': 'endoderm'
}

system_slims = {
    'UBERON:0000383': 'musculature of body',
    'UBERON:0000949': 'endocrine system',
    'UBERON:0000990': 'reproductive system',
    'UBERON:0001004': 'respiratory system',
    'UBERON:0001007': 'digestive system',
    'UBERON:0001008': 'excretory system',
    'UBERON:0001009': 'circulatory system',
    'UBERON:0001434': 'skeletal system',
    'UBERON:0002405': 'immune system',
    'UBERON:0002416': 'integumental system',
    'UBERON:0001032': 'sensory system',
    'UBERON:0001017': 'central nervous system',
    'UBERON:0000010': 'peripheral nervous system'
}

organ_slims = {
    'UBERON:0002369': 'adrenal gland',
    'UBERON:0002110': 'gallbladder',
    'UBERON:0002106': 'spleen',
    'UBERON:0001173': 'billary tree',
    'UBERON:0001043': 'esophagus',
    'UBERON:0000004': 'nose',
    'UBERON:0000056': 'ureter',
    'UBERON:0000057': 'urethra',
    'UBERON:0000059': 'large intestine',
    'UBERON:0000165': 'mouth',
    'UBERON:0000945': 'stomach',
    'UBERON:0000948': 'heart',
    'UBERON:0000955': 'brain',
    'UBERON:0000970': 'eye',
    'UBERON:0000991': 'gonad',
    'UBERON:0001043': 'esophagus',
    'UBERON:0001255': 'urinary bladder',
    'UBERON:0001264': 'pancreas',
    'UBERON:0001474': 'bone element',
    'UBERON:0002003': 'peripheral nerve',
    'UBERON:0002048': 'lung',
    'UBERON:0002097': 'skin of body',
    'UBERON:0002107': 'liver',
    'UBERON:0000059': 'large intestine',
    'UBERON:0002108': 'small intestine',
    'UBERON:0002113': 'kidney',
    'UBERON:0002240': 'spinal cord',
    'UBERON:0002367': 'prostate gland',
    'UBERON:0002370': 'thymus',
    'UBERON:0003126': 'trachea',
    'UBERON:0001723': 'tongue',
    'UBERON:0001737': 'larynx',
    'UBERON:0006562': 'pharynx',
    'UBERON:0001103': 'diaphragm',
    'UBERON:0002185': 'bronchus',
    'UBERON:0000029': 'lymph node',
    'UBERON:0002391': 'lymph',
    'UBERON:0010133': 'neuroendocrine gland',
    'UBERON:0001132': 'parathyroid gland',
    'UBERON:0002046': 'thyroid gland',
    'UBERON:0001981': 'blood vessel',
    'UBERON:0001473': 'lymphatic vessel',
    'UBERON:0000178': 'blood',
    'UBERON:0002268': 'olfactory organ',
    'UBERON:0007844': 'cartilage element',
    'UBERON:0001690': 'ear',
    'UBERON:0001987': 'placenta',
    'UBERON:0001911': 'mammary gland',
    'UBERON:0001630': 'muscle organ',
    'UBERON:0000007': 'pituitary gland',
    'UBERON:0002370': 'thymus',
    'UBERON:0000478': 'extraembryonic structure'
}

assay_slims = {
    # Note shortened synonyms are provided
    'OBI:0000634': 'DNA methylation',  # 'DNA methylation profiling'
    'OBI:0000424': 'Transcription',  # 'transcription profiling'
    'OBI:0001398': 'DNA binding',  # "protein and DNA interaction"
    'OBI:0001854': 'RNA binding',  # "protein and RNA interaction"
    'OBI:0001917': '3D chromatin structure',  # 'chromosome conformation identification objective'
    'OBI:0000870': 'DNA accessibility',  # 'single-nucleotide-resolution nucleic acid structure mapping assay'
    'OBI:0001916': 'Replication timing',
    'OBI:0000435': 'Genotyping',
    'OBI:0000615': 'Proteomics',
}

slim_shims = {
    # this allows us to manually assign term X to slim Y while waiting for ontology updates
    'assay': {
        # DNA accessibility
        'OBI:0001924': 'DNA accessibility',  # 'OBI:0000870' / MNase-seq
        'OBI:0002039': 'DNA accessibility',  # 'OBI:0000870', / ATAC-seq
        'OBI:0001853': 'DNA accessibility',  # 'OBI:0000870', / DNase-seq
        'OBI:0001859': 'DNA accessibility',  # 'OBI:0000870', / OBI:0000424  / FAIRE-seq
        'OBI:0002042': '3D chromatin structure',  # 'OBI:0000870' (Hi-C)
        'OBI:0001848': '3D chromatin structure',  # ChIA-PET / OBI:000870
        'OBI:0001923': 'Proteomics',  # OBI:0000615': 'MS-MS'
        'OBI:0001849': 'Genotyping',  # OBI:0000435 (DNA-PET)
        'OBI:0002044': 'RNA binding',  # OBI:0001854 (RNA-Bind-N-Seq)
        'OBI:0002091': 'Transcription',
        'OBI:0002092': 'Transcription',
        'OBI:0002093': 'Transcription'
    }

}

preferred_name = {
    "OBI:0000626": "WGS",
    "OBI:0001247": "genotyping HTS",
    "OBI:0001332": "DNAme array",
    "OBI:0001335": "microRNA counts",
    "OBI:0001463": "RNA microarray",
    "OBI:0001863": "WGBS",
    "OBI:0001923": "MS-MS",
    "OBI:0001271": "RNA-seq",
    "OBI:0000716": "ChIP-seq",
    "OBI:0001853": "DNase-seq",
    "OBI:0001920": "Repli-seq",
    "OBI:0001864": "RAMPAGE",
    "OBI:0001393": "genotyping array",
    "OBI:0002042": "Hi-C",
}

category_slims = {
    'OBI:0000634': 'DNA methylation profiling',
    'OBI:0000424': 'transcription profiling',
    'OBI:0000435': 'genotyping',
    'OBI:0000615': 'proteomics',
    'OBI:0001916': 'replication',
    'OBI:0001398': "protein and DNA interaction",
{'fraction_non_alphanumeric': 0.17261103633916555, 'fraction_numerical': 0.1465343203230148, 'mean_word_length': 4.44617498854787, 'pattern_counts': {'":': 28, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3005053', 'n_tokens_mistral': 6093, 'n_tokens_neox': 4613, 'n_words': 835}