<commit_before>import argparse
import os

from scheduled_bots.ontology.obographs import Graph, Node
from wikidataintegrator import wdi_login
from scheduled_bots import PROPS


class DONode(Node):
    def set_aliases(self, wd_item):
        # filter out aliases containing these strings
        bad_things = {'(disorder)', '[obs]'}
        if self.synonyms is not None:
            aliases = [x for x in self.synonyms if all(y not in x for y in bad_things)]
            wd_item.set_aliases(aliases=aliases, append=True)

    def _pre_create(self):
        # remove SNOMEDCT_US_2016_03_01 xrefs
        prefixes = {'SNOMED', 'EFO:', 'KEGG:', 'MEDDRA:', 'CSP:'}
        self.xrefs = set(x for x in self.xrefs if all(not x.lower().startswith(prefix.lower()) for prefix in prefixes))


class DOGraph(Graph):
    NAME = "Disease Ontology"
    GRAPH_URI = 'http://purl.obolibrary.org/obo/doid.owl'
    QID = "Q5282129"
    DEFAULT_DESCRIPTION = "human disease"
    APPEND_PROPS = {PROPS['subclass of'], PROPS['instance of'],
                    PROPS['has cause'], PROPS['location'],
                    PROPS['OMIM ID'], PROPS['Orphanet ID'],
                    PROPS['MeSH ID'], PROPS['ICD-10-CM'],
                    PROPS['ICD-10'], PROPS['ICD-9-CM'],
                    PROPS['ICD-9'], PROPS['NCI Thesaurus ID'],
                    PROPS['UMLS CUI'], PROPS['Disease Ontology ID']}
    FAST_RUN = True<commit_msg>obographs_DO filter out bad synonyms<commit_after>import argparse
import os

from scheduled_bots.ontology.obographs import Graph, Node
from wikidataintegrator import wdi_login
from scheduled_bots import PROPS


class DONode(Node):
    def set_aliases(self, wd_item):
        # filter out aliases containing these strings
        bad_things = {'(disorder)', '[obs]', '(finding)', 'subac.', 'morphologic abnormality', '(qualifier value)', '[dup]'}
        if self.synonyms is not None:
            aliases = [x for x in self.synonyms if all(y not in x for y in bad_things)]
            wd_item.set_aliases(aliases=aliases, append=True)

    def _pre_create(self):
        # remove SNOMEDCT_US_2016_03_01 xrefs
        prefixes = {'SNOMED', 'EFO:', 'KEGG:', 'MEDDRA:', 'CSP:', 'URL:'}
        self.xrefs = set(x for x in self.xrefs if all(not x.lower().startswith(prefix.lower()) for prefix in prefixes))


class DOGraph(Graph):
    NAME = "Disease Ontology"
    GRAPH_URI = 'http://purl.obolibrary.org/obo/doid.owl'
    QID = "Q5282129"
    DEFAULT_DESCRIPTION = "human disease"
    APPEND_PROPS = {PROPS['subclass of'], PROPS['instance of'],
                    PROPS['has cause'], PROPS['location'],
                    PROPS['OMIM ID'], PROPS['Orphanet ID'],
                    PROPS['MeSH ID'], PROPS['ICD-10-CM'],
                    PROPS['ICD-10'], PROPS['ICD-9-CM'],
                    PROPS['ICD-9'], PROPS['NCI Thesaurus ID'],
                    PROPS['UMLS CUI'], PROPS['Disease Ontology ID']}
    FAST_RUN = True
{'fraction_non_alphanumeric': 0.13337870023817625, 'fraction_numerical': 0.014290575025518884, 'mean_word_length': 2.788659793814433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352006', 'n_tokens_mistral': 1049, 'n_tokens_neox': 967, 'n_words': 245}