<commit_before>#!/usr/bin/env python3
from collections import defaultdict, OrderedDict
from pathlib import Path
import json, yaml
import redo
from data import Databases, Evaluations, Mappings, Dnf, ErrorAnalysis, ErrorAnalyses
from utils import get_logger
import comap
from comap import cui_names, translation_read_2to3


logger = get_logger(__name__)


def create_error_analyses(events, databases, mappings, dnfs,
                          evaluations,
                          residuals_info):
    error_analyses_fn = defaultdict(dict)
    error_analyses_fp = defaultdict(dict)
    for event in events:
        mapping = mappings.get(event)
        for database in databases.databases():
            evaluation = evaluations[event][database]
            if evaluation:
                error_analysis_fn = create_error_analysis_fn(event,
                                                             database,
                                                             databases,
                                                             dnfs[event],
                                                             evaluation,
                                                             mapping.exclusion_codes(database),
                                                             residuals_info[database][event],
                                                             mapping)
                error_analysis_fp = create_error_analysis_fp(event,
                                                             database,
                                                             databases,
                                                             dnfs[event],
                                                             evaluation,
                                                             mapping)
            else:
                error_analysis_fn, error_analysis_fp = None, None
            error_analyses_fn[database][event] = error_analysis_fn
            error_analyses_fp[database][event] = error_analysis_fp
    return ErrorAnalyses(error_analyses_fn, error_analyses_fp)


def create_error_analysis_fp(event, database, databases, dnf, evaluation, mapping):
    coding_system = databases.coding_system(database)
    tp, fp, fn = evaluation.tp, evaluation.fp, evaluation.fn
    dnf_codes = dnf.codes(coding_system)

    def categorize(code):
        if code in dnf_codes:
            return 'in-dnf'
        else:
            return 'other-fp'

    code_categories, unassigned = dict(), set()
    for code in fp:
        category = categorize(code)
        if category:
            code_categories[code] = category
        else:
            unassigned.add(code)
    if unassigned:
        logger.warn("Unassigned codes for %s: %s", coding_system, ', '.join(unassigned))
    return ErrorAnalysis(code_categories, unassigned)


def database_specific(code, database, databases, dnf, mapping):
    """Checks if all cosynonyms of `code` are false positive for the
    mapping.
    """
    voc = databases.coding_system(database)
    return all(
        not (generated & reference)
        # All CUI sets that have `code`
        for cuis in dnf.cui_sets() if code in dnf.get(cuis).get(voc, set())
        # All *other* databases
        for database1 in databases.databases() if database1 != database
        for generated in [dnf.get(cuis).get(databases.coding_system(database1), set())]
        for reference in [mapping.codes(database1) or set()]
    )


def get_relateds(cuis, databases):
    # {cui: {rel: [sourceConcept]}}
    relateds0 = comap.get_client().related(cuis, ['RN', 'CHD', 'RB', 'PAR'], databases.coding_systems())
    # {cui: {cui}}
    return {
        cui: {sourceConcept['cui'] for sourceConcept in sourceConcepts}
        for cui in relateds0
        for sourceConcepts in relateds0[cui].values()
    }


def get_cuis(codes, coding_system, dnf):
    return {cui
            for cuis, for_cuis in dnf.disjunction().items()
            if codes & for_cuis.get(coding_system, set())
            for cui in cuis}


def create_error_analysis_fn(event, database, databases, dnf, evaluation,
                             exclusion_codes, residuals_info, mapping):
    coding_system = databases.coding_system(database)
    fn = evaluation.fn
    known_fn = comap.get_client().known_codes(fn, coding_system)
    fn_cuis = get_cuis(fn, coding_system, dnf)
    dnf_codes = dnf.codes(coding_system)
    relateds = get_relateds(fn_cuis, databases)

    def categorize_fn(code):
        # if code in residuals_info:
        #     return residuals_info[code]['category']
        if code not in dnf_codes:
            return 'not-in-dnf'
        if code not in known_fn:
            return 'not-in-umls'
        if not any(relateds.get(cui) for cui in dnf.cuis(code, coding_system)):
            return 'isolated'
        if any(evaluation.cuis & relateds.get(cui, set()) for cui in fn_cuis):
            return 'next-expansion'
        if exclusion_codes and code in exclusion_codes:
            return 'exclusion'
        if database_specific(code, database, databases, dnf, mapping):
            return 'database-specific'
        return None

    code_categories, unassigned = dict(), set()
    for code in fn:
        category = categorize_fn(code)
        if category:
            code_categories[code] = category
        else:
            unassigned.add(code)

    if unassigned:
        logger.warn("Unasssigned codes for %s: %s", coding_system, ", ".join(unasssigned))
        if coding_system == 'RCD2':
            t = translation_read_2to3(unassigned)
        names = cui_names(fn_cuis)
        def for_code(code):
            cuis = dnf.cuis(code, coding_system)
            res = OrderedDict([
                ('cuis', {
                    cui: OrderedDict([
                        ('name', names.get(cui, '?')),
                        ('relateds', ' '.join(relateds.get(cui, []))),
                    ])<commit_msg>Fix typo<commit_after>#!/usr/bin/env python3
from collections import defaultdict, OrderedDict
from pathlib import Path
import json, yaml
import redo
from data import Databases, Evaluations, Mappings, Dnf, ErrorAnalysis, ErrorAnalyses
from utils import get_logger
import comap
from comap import cui_names, translation_read_2to3


logger = get_logger(__name__)


def create_error_analyses(events, databases, mappings, dnfs,
                          evaluations,
                          residuals_info):
    error_analyses_fn = defaultdict(dict)
    error_analyses_fp = defaultdict(dict)
    for event in events:
        mapping = mappings.get(event)
        for database in databases.databases():
            evaluation = evaluations[event][database]
            if evaluation:
                error_analysis_fn = create_error_analysis_fn(event,
                                                             database,
                                                             databases,
                                                             dnfs[event],
                                                             evaluation,
                                                             mapping.exclusion_codes(database),
                                                             residuals_info[database][event],
                                                             mapping)
                error_analysis_fp = create_error_analysis_fp(event,
                                                             database,
                                                             databases,
                                                             dnfs[event],
                                                             evaluation,
                                                             mapping)
            else:
                error_analysis_fn, error_analysis_fp = None, None
            error_analyses_fn[database][event] = error_analysis_fn
            error_analyses_fp[database][event] = error_analysis_fp
    return ErrorAnalyses(error_analyses_fn, error_analyses_fp)


def create_error_analysis_fp(event, database, databases, dnf, evaluation, mapping):
    coding_system = databases.coding_system(database)
    tp, fp, fn = evaluation.tp, evaluation.fp, evaluation.fn
    dnf_codes = dnf.codes(coding_system)

    def categorize(code):
        if code in dnf_codes:
            return 'in-dnf'
        else:
            return 'other-fp'

    code_categories, unassigned = dict(), set()
    for code in fp:
        category = categorize(code)
        if category:
            code_categories[code] = category
        else:
            unassigned.add(code)
    if unassigned:
        logger.warn("Unassigned codes for %s: %s", coding_system, ', '.join(unassigned))
    return ErrorAnalysis(code_categories, unassigned)


def database_specific(code, database, databases, dnf, mapping):
    """Checks if all cosynonyms of `code` are false positive for the
    mapping.
    """
    voc = databases.coding_system(database)
    return all(
        not (generated & reference)
        # All CUI sets that have `code`
        for cuis in dnf.cui_sets() if code in dnf.get(cuis).get(voc, set())
        # All *other* databases
        for database1 in databases.databases() if database1 != database
        for generated in [dnf.get(cuis).get(databases.coding_system(database1), set())]
        for reference in [mapping.codes(database1) or set()]
    )


def get_relateds(cuis, databases):
    # {cui: {rel: [sourceConcept]}}
    relateds0 = comap.get_client().related(cuis, ['RN', 'CHD', 'RB', 'PAR'], databases.coding_systems())
    # {cui: {cui}}
    return {
        cui: {sourceConcept['cui'] for sourceConcept in sourceConcepts}
        for cui in relateds0
        for sourceConcepts in relateds0[cui].values()
    }


def get_cuis(codes, coding_system, dnf):
    return {cui
            for cuis, for_cuis in dnf.disjunction().items()
            if codes & for_cuis.get(coding_system, set())
            for cui in cuis}


def create_error_analysis_fn(event, database, databases, dnf, evaluation,
                             exclusion_codes, residuals_info, mapping):
    coding_system = databases.coding_system(database)
    fn = evaluation.fn
    known_fn = comap.get_client().known_codes(fn, coding_system)
    fn_cuis = get_cuis(fn, coding_system, dnf)
    dnf_codes = dnf.codes(coding_system)
    relateds = get_relateds(fn_cuis, databases)

    def categorize_fn(code):
        # if code in residuals_info:
        #     return residuals_info[code]['category']
        if code not in dnf_codes:
            return 'not-in-dnf'
        if code not in known_fn:
            return 'not-in-umls'
        if not any(relateds.get(cui) for cui in dnf.cuis(code, coding_system)):
            return 'isolated'
        if any(evaluation.cuis & relateds.get(cui, set()) for cui in fn_cuis):
            return 'next-expansion'
        if exclusion_codes and code in exclusion_codes:
            return 'exclusion'
        if database_specific(code, database, databases, dnf, mapping):
            return 'database-specific'
        return None

    code_categories, unassigned = dict(), set()
    for code in fn:
        category = categorize_fn(code)
        if category:
            code_categories[code] = category
        else:
            unassigned.add(code)

    if unassigned:
        logger.warn("Unassigned codes for %s: %s", coding_system, ", ".join(unassigned))
        if coding_system == 'RCD2':
            t = translation_read_2to3(unassigned)
        names = cui_names(fn_cuis)
        def for_code(code):
            cuis = dnf.cuis(code, coding_system)
            res = OrderedDict([
                ('cuis', {
                    cui: OrderedDict([
                        ('name', names.get(cui, '?')),
                        ('relateds', ' '.join(relateds.get(cui, []))),
                    ])
{'fraction_non_alphanumeric': 0.08591324775568421, 'fraction_numerical': 0.002181391056296669, 'mean_word_length': 1.7490774907749078, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6603784', 'n_tokens_mistral': 3229, 'n_tokens_neox': 3092, 'n_words': 777}