<commit_before>#!/usr/bin/env python3
import pandas as pd
from io import StringIO
import argparse


EXCLUDE_RCD2_CODES = ['_DRUG', '_NONE', '2....']


VOCABULARY_DESCRIPTIONS = """
ICD10	ICD-10	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD10/
ICD10CM	ICD-10 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD10CM/
ICD9CM	ICD-9 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD9CM/
MTHICD9	ICD-9 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/MTHICD9/
ICPC	ICPC	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICPC/, Local versions (e.g. ICPCDUT) can be added if necessary
ICPC2EENG	ICPC-2	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICPC2EENG/
RCD	READ CTv3	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/RCD/
RCD2	READ v2	Translation of RCD based on mappings by Health & Social Care Information Centre at https://isd.hscic.gov.uk
"""


def open_xls_files(xls_files):
    dfs = []
    for xls_file in xls_files:
        event_name = pd.read_excel(xls_file, sheetname='Info', header=None).iloc[0, 1]
        codes = (
            pd.read_excel(xls_file, sheetname='Codes', na_values='-')
            .pipe(lambda df: df[df.Code.notnull()])
            .assign(Event=event_name)
        )
        def relevant_tag(t):
            return not t or t.isupper()
        tag = (
            codes.Tags
            .fillna('').str.split(', ')
            .apply(lambda s: pd.Series(sorted({t for t in s if relevant_tag(t)})))
            .stack()
            .reset_index(level=1, drop=True)
            .to_frame('Tag')
        )
        df = pd.merge(codes, tag, how='outer', left_index=True, right_index=True)
        dfs.append(df)
    return pd.concat(dfs, axis=0)


def compile_codes(codes, rev_voc_map):
    res = (
        codes
        .assign(Target=lambda df: df['Coding system'].map(rev_voc_map))
        .pipe(lambda df: df[df.Target.notnull()])
    )
    res['Coding system (tag)'] = res.Target + res.Tag.map(lambda t: ' (' + t + ')' if t else '')
    return res


def output(codes, mappings_df, out_xls_file):
    with pd.ExcelWriter(out_xls_file) as writer:
        for event in sorted(codes.Event.unique()):
            code_filtered = (codes['Coding system'] == 'RCD2') & codes.Code.isin(EXCLUDE_RCD2_CODES)
            (codes[(codes.Event == event) & ~code_filtered]
             .sort_values(['Coding system (tag)', 'Code'])
             .set_index(['Coding system (tag)', 'Code', 'Code name'])
             [['Coding system', 'Concept', 'Concept name']]
             .to_excel(writer, event))
        descrs = pd.read_csv(StringIO(VOCABULARY_DESCRIPTIONS),
                             sep='\t', index_col=False,
                             names=["Coding system (CodeMapper/UMLS)",
                                    "Coding system family",
                                    "Description"])
        descrs.to_excel(writer, "Coding systems", index=False)
        mappings_df.to_excel(writer, "Coding systems", index=False, startrow=len(descrs)+2)


def reverse_voc_map(voc_map):
    return {
        voc0: voc
        for voc in voc_map
        for voc0 in voc_map[voc]
    }


def create_mappings_df(voc_map):
    return pd.DataFrame.from_records([
        (voc, "+".join(voc_map[voc]))
        for voc in voc_map
    ], columns=["Coding system (Resulting mapping)", "Coding systems (UMLS)"])


def run(xls_files, voc_map, out_xls_file):
    codes0 = open_xls_files(xls_files)
    codes1 = compile_codes(codes0, reverse_voc_map(voc_map))
    mappings_df = create_mappings_df(voc_map)
    output(codes1, mappings_df, out_xls_file)


def parse_voc_map(voc_map):
    return {
        v1: v2s.split('+')
        for s in voc_map
        for v1, v2s in [s.split(':')]
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--inputs', metavar='FILE', nargs='+', required=True,
                        help='CodeMapper XLS files')
    parser.add_argument('--voc-map', dest='voc_map', nargs='*', required=True,
                        help='A mapping like ICD9:ICD9CM+MTHICD9 READ2:RCD2 ICD10:ICD10CM ICPC2:ICPC2EENG')
    parser.add_argument('--output', metavar='FILE', required=True,
                        help='Output XLS file')
    args = parser.parse_args()<commit_msg>compile.py: Filter out subsumed codes, warn high-level Read-v2 codes<commit_after>#!/usr/bin/env python3
import pandas as pd
import re
from io import StringIO
import argparse


EXCLUDE_RCD2_CODES = ['_DRUG', '_NONE']


VOCABULARY_DESCRIPTIONS = """
ICD10	ICD-10	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD10/
ICD10CM	ICD-10 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD10CM/
ICD9CM	ICD-9 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICD9CM/
MTHICD9	ICD-9 CM	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/MTHICD9/
ICPC	ICPC	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICPC/, Local versions (e.g. ICPCDUT) can be added if necessary
ICPC2EENG	ICPC-2	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/ICPC2EENG/
RCD	READ CTv3	https://www.nlm.nih.gov/research/umls/sourcereleasedocs/current/RCD/
RCD2	READ v2	Translation of RCD based on mappings by Health & Social Care Information Centre at https://isd.hscic.gov.uk
"""

SUBSUMPTION_BY_PREFIX = ['ICD9', 'ICD9CM', 'MTHICD9', 'ICD10', 'ICD10CM', 'RCD2']
IGNORE_SUFFIX_REGEX = {'RCD2': re.compile('\.+$')}


def subsumed_codes(codes):
    grouped_codes = codes.groupby(['Event', 'Target']).Code.aggregate(lambda s: set(s))
    return codes.apply(lambda r: is_subsumed(r.Code, grouped_codes[r.Event, r.Target], r['Coding system']), axis=1)


def ignore_suffix(code, vocabulary):
    if vocabulary in IGNORE_SUFFIX_REGEX:
        return IGNORE_SUFFIX_REGEX[vocabulary].sub('', code)
    else:
        return code


def is_subsumed(code, codes, vocabulary):
    code = ignore_suffix(code, vocabulary)
    codes = [ignore_suffix(c, vocabulary) for c in codes]
    if vocabulary in SUBSUMPTION_BY_PREFIX:
        for code1 in codes:
            if code1 != code and len(code1) > 1 and code.startswith(code1):
                return True
    return False


def open_xls_files(xls_files):
    dfs = []
    for xls_file in xls_files:
        event_name = pd.read_excel(xls_file, sheetname='Info', header=None).iloc[0, 1]
        codes = (
            pd.read_excel(xls_file, sheetname='Codes', na_values='-')
            .pipe(lambda df: df[df.Code.notnull()])
            .assign(Event=event_name)
        )
        def relevant_tag(t):
            return not t or t.isupper()
        tag = (
            codes.Tags
            .fillna('').str.split(', ')
            .apply(lambda s: pd.Series(sorted({t for t in s if relevant_tag(t)})))
            .stack()
            .reset_index(level=1, drop=True)
            .to_frame('Tag')
        )
        df = pd.merge(codes, tag, how='outer', left_index=True, right_index=True)
        dfs.append(df)
    return pd.concat(dfs, axis=0)


def compile_codes(codes, rev_voc_map):
    res = (
        codes
        .assign(Target=lambda df: df['Coding system'].map(rev_voc_map))
        .pipe(lambda df: df[df.Target.notnull()])
    )
    res['Coding system (tag)'] = res.Target + res.Tag.fillna('').map(lambda t: ' (' + t + ')' if t else '')
    return res


def filter_codes(codes):
    subsumed = subsumed_codes(codes)
    print('Ignore subsumed codes')
    print(codes[subsumed][['Event', 'Target', 'Code', 'Code name']].sort_values(['Event', 'Target', 'Code']).drop_duplicates())
    excluded = (codes['Coding system'] == 'RCD2') & codes.Code.isin(EXCLUDE_RCD2_CODES)
    res = codes[~subsumed & ~excluded]
    highlevel_rcd2 = codes[(codes['Coding system'] == 'RCD2') & codes.Code.str.endswith('....')]
    print('High-level RCD2 codes')
    print(highlevel_rcd2[['Event', 'Target', 'Code', 'Code name']].sort_values(['Event', 'Target', 'Code']).drop_duplicates())
    return res


def output(codes, mappings_df, out_xls_file):
    with pd.ExcelWriter(out_xls_file) as writer:
        for event in sorted(codes.Event.unique()):
            (codes[codes.Event == event]
             .sort_values(['Coding system (tag)', 'Code'])
             .set_index(['Coding system (tag)', 'Code', 'Code name'])
             [['Coding system', 'Concept', 'Concept name']]
             .to_excel(writer, event))
        descrs = pd.read_csv(StringIO(VOCABULARY_DESCRIPTIONS),
                             sep='\t', index_col=False,
                             names=["Coding system (CodeMapper/UMLS)",
                                    "Coding system family",
                                    "Description"])
        descrs.to_excel(writer, "Coding systems", index=False)
        mappings_df.to_excel(writer, "Coding systems", index=False, startrow=len(descrs)+2)


def reverse_voc_map(voc_map):
    return {
        voc0: voc
        for voc in voc_map
        for voc0 in voc_map[voc]
    }


def create_mappings_df(voc_map):
    return pd.DataFrame.from_records([
        (voc, "+".join(voc_map[voc]))
        for voc in voc_map
    ], columns=["Coding system (Resulting mapping)", "Coding systems (UMLS)"])


def run(xls_files, voc_map, out_xls_file):
    codes0 = open_xls_files(xls_files)
    codes1 = compile_codes(codes0, reverse_voc_map(voc_map))
    codes2 = filter_codes(codes1)
    mappings_df = create_mappings_df(voc_map)
    output(codes2, mappings_df, out_xls_file)


def parse_voc_map(voc_map):
    return {
        v1: v2s.split('+')
        for s in voc_map
        for v1, v2s in [s.split(':')]
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--inputs', metavar='FILE', nargs='+', required=True,
                        help='CodeMapper XLS files')
    parser.add_argument('--voc-map', dest='voc_map', nargs='*', required=True,
                        help='A mapping like ICD9:ICD9CM+MTHICD9 READ2:RCD2 ICD10:ICD10CM ICPC2:ICPC2EENG')
    parser.add_argument('--output', metavar='FILE', required=True,
                        help='Output XLS file')
    args = parser.parse_args()
{'fraction_non_alphanumeric': 0.14055751864939145, 'fraction_numerical': 0.012858264625049078, 'mean_word_length': 2.859469696969697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 16, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4089429', 'n_tokens_mistral': 3745, 'n_tokens_neox': 3525, 'n_words': 696}