<commit_before>"""
Mapping data extracted from a prepared source. A first level mapping from
the sources file has been completed. Writing ETLs for the OMOP CDM is complex
because of the mappings and that a single source file can map to multiple tables.

Fields in the prepared source start either start with s_  for source or m_ mapped
to a name part of the OHDSI vocabulary.
"""

import os
import sys

try:
    from source_to_cdm_functions import *
    from omop_cdm_classes_5_2 import *
    from prepared_source_classes import *
    from mapping_classes import *
except ImportError:
    sys.path.insert(0, os.path.abspath(os.path.join(os.path.split(__file__)[0], os.path.pardir, "src")))
    from source_to_cdm_functions import *
    from omop_cdm_classes_5_2 import *
    from prepared_source_classes import *
    from mapping_classes import *

import logging
import argparse

logging.basicConfig(level=logging.INFO)


def main(input_csv_directory, output_csv_directory, json_map_directory):
    # TODO: Add Provider
    # TODO: Add Patient Location

    output_class_obj = OutputClassDirectory()
    in_out_map_obj = InputOutputMapperDirectory()
    output_directory_obj = OutputClassDirectory()

    #### Person ####

    input_person_csv = os.path.join(input_csv_directory, "source_person.csv")
    output_person_csv = os.path.join(output_csv_directory, "person_cdm.csv")

    person_rules = create_person_rules(json_map_directory)

    person_runner_obj = generate_mapper_obj(input_person_csv, SourcePersonObject(), output_person_csv, PersonObject(),
                                            person_rules,
                                            output_class_obj, in_out_map_obj, person_router_obj)

    person_runner_obj.run()

    #### Death ####

    # Generate look up for s_person_id
    output_person_csv = os.path.join(output_csv_directory, "person_cdm.csv")
    person_json_file_name = create_json_map_from_csv_file(output_person_csv, "person_source_value", "person_id")
    s_person_id_mapper = CoderMapperJSONClass(person_json_file_name)

    death_rules = create_death_person_rules(json_map_directory, s_person_id_mapper)

    output_death_csv = os.path.join(output_csv_directory, "death_cdm.csv")
    death_runner_obj = generate_mapper_obj(input_person_csv, SourcePersonObject(), output_death_csv, DeathObject(),
                                           death_rules, output_class_obj, in_out_map_obj, death_router_obj)
    death_runner_obj.run()

    #### Observation_Period ####

    obs_per_rules = create_observation_period_rules(json_map_directory, s_person_id_mapper)

    output_obs_per_csv = os.path.join(output_csv_directory, "observation_period_cdm.csv")

    input_obs_per_csv = os.path.join(input_csv_directory, "source_observation_period.csv")

    def obs_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            return ObservationPeriodObject()
        else:
            return NoOutputClass()

    obs_per_runner_obj = generate_mapper_obj(input_obs_per_csv, SourceObservationPeriodObject(), output_obs_per_csv,
                                             ObservationPeriodObject(),
                                             obs_per_rules, output_class_obj, in_out_map_obj, obs_router_obj)
    obs_per_runner_obj.run()

    #### Care Sites ####

    care_site_rules = [
        (":row_id", "care_site_id"),
        ("s_care_site_name", "care_site_name"),
        ("k_care_site", "care_site_source_value")]

    input_care_site_csv = os.path.join(input_csv_directory, "source_care_site.csv")
    output_care_site_csv = os.path.join(output_csv_directory, "care_site_cdm.csv")

    def care_site_router_obj(input_dict):
        return CareSiteObject()

    care_site_runner_obj = generate_mapper_obj(input_care_site_csv, SourceCareSiteObject(), output_care_site_csv,
                                           CareSiteObject(), care_site_rules,
                                           output_class_obj, in_out_map_obj, care_site_router_obj)

    care_site_runner_obj.run()

    care_site_json_file_name = create_json_map_from_csv_file(output_care_site_csv, "care_site_source_value",
                                                             "care_site_id")

    k_care_site_mapper = CoderMapperJSONClass(care_site_json_file_name, "k_care_site")

    #### Visit_Occurrence ###

    snomed_code_json = os.path.join(json_map_directory, "CONCEPT_CODE_SNOMED.json")
    snomed_code_mapper = CodeMapperClassSqliteJSONClass(snomed_code_json)

    visit_rules = create_visit_rules(json_map_directory, s_person_id_mapper, k_care_site_mapper, snomed_code_mapper)
    input_encounter_csv = os.path.join(input_csv_directory, "source_encounter.csv")
    output_visit_occurrence_csv = os.path.join(output_csv_directory, "visit_occurrence_cdm.csv")

    def visit_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != '1':
                return VisitOccurrenceObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    visit_runner_obj = generate_mapper_obj(input_encounter_csv, SourceEncounterObject(), output_visit_occurrence_csv,
                                           VisitOccurrenceObject(), visit_rules,
                                           output_class_obj, in_out_map_obj, visit_router_obj)

    visit_runner_obj.run()

    # Visit ID Map
    encounter_json_file_name = create_json_map_from_csv_file(output_visit_occurrence_csv, "visit_source_value",
                                                             "visit_occurrence_id")

    s_encounter_id_mapper = CoderMapperJSONClass(encounter_json_file_name, "s_encounter_id")

    # Visit ID Map
    encounter_json_file_name = create_json_map_from_csv_file(output_visit_occurrence_csv, "visit_source_value",
                                                             "visit_occurrence_id")

    #### Benefit Coverage Period ####

    payer_plan_period_rules = create_payer_plan_period_rules(s_person_id_mapper)

    output_ppp_csv = os.path.join(output_csv_directory, "payer_plan_period_cdm.csv")

    input_ppp_csv = os.path.join(input_csv_directory, "source_encounter_coverage.csv")

    def payer_plan_period_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            return PayerPlanPeriodObject()
        else:
            return NoOutputClass()

    payer_plan_period_runner_obj = generate_mapper_obj(input_ppp_csv, SourceEncounterCoverageObject(), output_ppp_csv,
                                                       PayerPlanPeriodObject(),
                                                       payer_plan_period_rules, output_class_obj, in_out_map_obj,
                                                       payer_plan_period_router_obj
                                                       )
    payer_plan_period_runner_obj.run()

    #### MEASUREMENT and OBSERVATION dervived from 'source_observation.csv' ####

    snomed_code_result_mapper = ChainMapper(FilterHasKeyValueMapper(["s_type_code"]), snomed_code_mapper)

    def measurement_router_obj(input_dict):
        """Determine if the result contains a LOINC code"""
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                mapped_result_code = snomed_code_result_mapper.map(input_dict)
                if "CONCEPT_CLASS_ID".lower() in mapped_result_code:
                    if mapped_result_code["DOMAIN_ID".lower()] == "Measurement":
                        return MeasurementObject()
                    elif mapped_result_code["DOMAIN_ID".lower()] == "Observation":
                        return ObservationObject()
                    else:
                        return NoOutputClass()
                else:
                    return MeasurementObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    snomed_json = os.path.join(json_map_directory, "CONCEPT_NAME_SNOMED.json")
    snomed_mapper = CodeMapperClassSqliteJSONClass(snomed_json)

    measurement_rules, observation_measurement_rules = \
        create_measurement_and_observation_rules(json_map_directory, s_person_id_mapper, s_encounter_id_mapper, snomed_mapper,
                                                 snomed_code_mapper)

    input_result_csv = os.path.join(input_csv_directory, "source_result.csv")
    output_measurement_csv = os.path.join(output_csv_directory, "measurement_encounter_cdm.csv")

    measurement_runner_obj = generate_mapper_obj(input_result_csv, SourceResultObject(), output_measurement_csv,
                                                 MeasurementObject(),
                                                 measurement_rules, output_class_obj, in_out_map_obj,
                                                 measurement_router_obj)

    output_observation_csv = os.path.join(output_csv_directory, "observation_measurement_encounter_cdm.csv")
    register_to_mapper_obj(input_result_csv, SourceResultObject(), output_observation_csv,
                           ObservationObject(), observation_measurement_rules, output_class_obj, in_out_map_obj)

    measurement_runner_obj.run()

    #### CONDITION / DX ####

    condition_type_name_json = os.path.join(json_map_directory, "CONCEPT_NAME_Condition_Type.json")
    condition_type_name_map = CoderMapperJSONClass(condition_type_name_json)

    condition_claim_type_map = \
        ChainMapper(
            ReplacementMapper({"Primary": "Primary Condition", "Secondary": "Secondary Condition"}),
            condition_type_name_map
        )

    input_condition_csv = os.path.join(input_csv_directory, "source_condition.csv")
    hi_condition_csv_obj = InputClassCSVRealization(input_condition_csv, SourceConditionObject())

    output_condition_csv = os.path.join(output_csv_directory, "condition_occurrence_dx_cdm.csv")
    cdm_condition_csv_obj = OutputClassCSVRealization(output_condition_csv, ConditionOccurrenceObject())

    icd9cm_json = os.path.join(json_map_directory, "ICD9CM_with_parent.json")
    icd10cm_json = os.path.join(json_map_directory, "ICD10CM_with_parent.json")

    def case_mapper_icd9_icd10(input_dict, field="m_condition_code_oid"):
        """Map ICD9 and ICD10 to the CDM vocabularies"""
        coding_system_oid = input_dict[field]
        coding_version = condition_coding_system(coding_system_oid)

        if coding_version == "ICD9CM":
            return 0
        else:
            return 1

    # TODO: condition_type_concept_id
    condition_encounter_mapper = ChainMapper(ConstantMapper({"diagnosis_type_name": "Observation recorded from EHR"}),
                                             condition_type_name_map)

    condition_type_concept_mapper = CascadeMapper(condition_claim_type_map, condition_encounter_mapper)

    def clean_concept_ids(input_dict):

        if "CONCEPT_ID".lower() in input_dict:
            if input_dict["CONCEPT_ID".lower()] == "" or input_dict["CONCEPT_ID".lower()] is None:
                input_dict["CONCEPT_ID".lower()] = 0
        else:
            input_dict["CONCEPT_ID".lower()] = 0

        if "MAPPED_CONCEPT_ID".lower() in input_dict:

            if input_dict["MAPPED_CONCEPT_ID".lower()] == "" or input_dict["MAPPED_CONCEPT_ID".lower()] is None:
                input_dict["MAPPED_CONCEPT_ID".lower()] = 0

        else:
            input_dict["MAPPED_CONCEPT_ID".lower()] = 0

        return input_dict


    ICDMapper = ChainMapper(CaseMapper(case_mapper_icd9_icd10,
                            CodeMapperClassSqliteJSONClass(icd9cm_json, "s_condition_code"),
                            CodeMapperClassSqliteJSONClass(icd10cm_json, "s_condition_code")), PassThroughFunctionMapper(clean_concept_ids))

    s_condition_type_dict = {"Admitting": "52870002", "Final": "89100005", "Preliminary": "148006"}
    condition_status_snomed_mapper = CodeMapperDictClass(s_condition_type_dict, "s_condition_type")
    condition_status_mapper = ChainMapper(condition_status_snomed_mapper, snomed_code_mapper)


    # Required: condition_occurrence_id, person_id, condition_concept_id, condition_start_date
    condition_rules_dx = [(":row_id", "condition_occurrence_id"),
                          ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                          ("s_encounter_id", s_encounter_id_mapper, {"visit_occurrence_id": "visit_occurrence_id"}),
                          (("s_condition_code", "m_condition_code_oid"),
                           ICDMapper,
                           {"CONCEPT_ID".lower(): "condition_source_concept_id", "MAPPED_CONCEPT_ID".lower(): "condition_concept_id"}),
                          ("s_condition_code", "condition_source_value"),
                          ("m_rank", condition_type_concept_mapper, {"CONCEPT_ID".lower(): "condition_type_concept_id"}),
                          ("s_condition_type", condition_status_mapper, {"CONCEPT_ID".lower(): "condition_status_concept_id"}),
                          ("s_condition_type", "condition_status_source_value"),
                          ("s_start_condition_datetime", SplitDateTimeWithTZ(), {"date": "condition_start_date"}),
                          ("s_start_condition_datetime", DateTimeWithTZ(), {"datetime": "condition_start_datetime"}),
                          ("s_end_condition_datetime", SplitDateTimeWithTZ(), {"date": "condition_end_date"}),
                          ("s_end_condition_datetime", DateTimeWithTZ(), {"datetime": "condition_end_datetime"})
                          ]

    condition_rules_dx_class = build_input_output_mapper(condition_rules_dx)

    # ICD9 and ICD10 conditions which map to measurements according to the CDM Vocabulary
    in_out_map_obj.register(SourceConditionObject(), ConditionOccurrenceObject(), condition_rules_dx_class)
    output_directory_obj.register(ConditionOccurrenceObject(), cdm_condition_csv_obj)

    measurement_row_offset = measurement_runner_obj.rows_run
    measurement_rules_dx = [(":row_id", row_map_offset("measurement_id", measurement_row_offset),
                             {"measurement_id": "measurement_id"}),
                            (":row_id", ConstantMapper({"measurement_type_concept_id": 0}),
                             {"measurement_type_concept_id": "measurement_type_concept_id"}),
                            ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                            ("s_encounter_id", s_encounter_id_mapper,
                             {"visit_occurrence_id": "visit_occurrence_id"}),
                            ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                             {"date": "measurement_date", "time": "measurement_time"}),
                            ("s_start_condition_datetime", DateTimeWithTZ(),
                             {"datetime": "measurement_datetime"}),
                            ("s_condition_code", "measurement_source_value"),
                            (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                             {"CONCEPT_ID".lower(): "measurement_source_concept_id",
                              "MAPPED_CONCEPT_ID".lower(): "measurement_concept_id"})]

    measurement_rules_dx_class = build_input_output_mapper(measurement_rules_dx)
    in_out_map_obj.register(SourceConditionObject(), MeasurementObject(), measurement_rules_dx_class)

    # The mapped ICD9 to measurements get mapped to a separate code
    output_measurement_dx_encounter_csv = os.path.join(output_csv_directory, "measurement_dx_cdm.csv")
    output_measurement_dx_encounter_csv_obj = OutputClassCSVRealization(output_measurement_dx_encounter_csv,
                                                                        MeasurementObject())

    output_directory_obj.register(MeasurementObject(), output_measurement_dx_encounter_csv_obj)

    observation_row_offset = measurement_runner_obj.rows_run

    # ICD9 and ICD10 codes which map to observations according to the CDM Vocabulary
    observation_rules_dx = [(":row_id", row_map_offset("observation_id", observation_row_offset),
                             {"observation_id": "observation_id"}),
                            (":row_id", ConstantMapper({"observation_type_concept_id": 0}),
                             {"observation_type_concept_id": "observation_type_concept_id"}),
                            ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                            (("s_encounter_id",),
                             s_encounter_id_mapper,
                             {"visit_occurrence_id": "visit_occurrence_id"}),
                            ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                             {"date": "observation_date", "time": "observation_time"}),
                            ("s_start_condition_datetime", DateTimeWithTZ(),
                             {"datetime": "observation_datetime"}),
                            ("s_condition_code", "observation_source_value"),
                            (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                             {"CONCEPT_ID".lower(): "observation_source_concept_id",
                              "MAPPED_CONCEPT_ID".lower(): "observation_concept_id"}),
                            ("m_rank", condition_claim_type_map,
                             {"CONCEPT_ID".lower(): "condition_type_concept_id"})]

    observation_rules_dx_class = build_input_output_mapper(observation_rules_dx)

    output_observation_dx_encounter_csv = os.path.join(output_csv_directory, "observation_dx_cdm.csv")
    output_observation_dx_encounter_csv_obj = OutputClassCSVRealization(output_observation_dx_encounter_csv,
                                                                        ObservationObject())

    output_directory_obj.register(ObservationObject(), output_observation_dx_encounter_csv_obj)
    in_out_map_obj.register(SourceConditionObject(), ObservationObject(), observation_rules_dx_class)

    procedure_type_json = os.path.join(json_map_directory, "CONCEPT_NAME_Procedure_Type.json")
    procedure_type_mapper = CoderMapperJSONClass(procedure_type_json)

    # ICD9 and ICD10 codes which map to procedures according to the CDM Vocabulary
    # "Procedure recorded as diagnostic code"
    # TODO: Map procedure_type_concept_id

    procedure_rules_dx_encounter = [(":row_id", "procedure_occurrence_id"),
                                    (":row_id",
                                     ChainMapper(ConstantMapper({"name": "Procedure recorded as diagnostic code"}),
                                                 procedure_type_mapper),
                                     {"CONCEPT_ID".lower(): "procedure_type_concept_id"}),
                                    ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                                    ("s_encounter_id", s_encounter_id_mapper,
                                     {"visit_occurrence_id": "visit_occurrence_id"}),
                                    ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                                     {"date": "procedure_date"}),
                                    ("s_start_condition_datetime", DateTimeWithTZ(),
                                     {"datetime": "procedure_datetime"}),
                                    ("s_condition_code", "procedure_source_value"),
                                    (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                                     {"CONCEPT_ID".lower(): "procedure_source_concept_id",
                                      "MAPPED_CONCEPT_ID".lower(): "procedure_concept_id"})]

    procedure_rules_dx_encounter_class = build_input_output_mapper(procedure_rules_dx_encounter)

    output_procedure_dx_encounter_csv = os.path.join(output_csv_directory, "procedure_dx_cdm.csv")
    output_procedure_dx_encounter_csv_obj = OutputClassCSVRealization(output_procedure_dx_encounter_csv,
                                                                      ProcedureOccurrenceObject())

    output_directory_obj.register(ProcedureOccurrenceObject(), output_procedure_dx_encounter_csv_obj)
    in_out_map_obj.register(SourceConditionObject(), ProcedureOccurrenceObject(), procedure_rules_dx_encounter_class)

    def condition_router_obj(input_dict):
        """ICD9 / ICD10 CM contain codes which could either be a procedure, observation, or measurement"""
        coding_system_oid = input_dict["m_condition_code_oid"]
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                if coding_system_oid:
                    result_dict = ICDMapper.map(input_dict)

                    if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict or "DOMAIN_ID".lower() in result_dict:
                        if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict:
                            domain = result_dict["MAPPED_CONCEPT_DOMAIN".lower()]
                        else:
                            domain = result_dict["DOMAIN_ID".lower()]
                    else:
                        domain = ""

                    if result_dict != {}:
                        if domain == "Condition":
                            return ConditionOccurrenceObject()
                        elif domain == "Observation":
                            return ObservationObject()
                        elif domain == "Procedure":
                            return ProcedureOccurrenceObject()
                        elif domain == "Measurement":
                            return MeasurementObject()
                        else:
                            return NoOutputClass()
                    else:
                        return NoOutputClass()
                else:
                    return NoOutputClass()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    condition_runner_obj = RunMapperAgainstSingleInputRealization(hi_condition_csv_obj, in_out_map_obj,
                                                                  output_directory_obj,
                                                                  condition_router_obj)

    condition_runner_obj.run()

    # Update needed offsets
    condition_row_offset = condition_runner_obj.rows_run
    procedure_row_offset = condition_runner_obj.rows_run
    measurement_row_offset += condition_row_offset
    observation_row_offset += condition_row_offset

    procedure_rules_encounter = create_procedure_rules(json_map_directory, s_person_id_mapper, s_encounter_id_mapper,
                                                       procedure_row_offset)
    procedure_rule = procedure_rules_encounter[0]
    procedure_code_map = procedure_rule[1]

    procedure_rules_encounter_class = build_input_output_mapper(procedure_rules_encounter)

    input_proc_csv = os.path.join(input_csv_directory, "source_procedure.csv")
    hi_proc_csv_obj = InputClassCSVRealization(input_proc_csv, SourceProcedureObject())

    in_out_map_obj.register(SourceProcedureObject(), ProcedureOccurrenceObject(), procedure_rules_encounter_class)

    output_proc_encounter_csv = os.path.join(output_csv_directory, "procedure_cdm.csv")
    output_proc_encounter_csv_obj = OutputClassCSVRealization(output_proc_encounter_csv,
                                                              ProcedureOccurrenceObject())

    output_directory_obj.register(ProcedureOccurrenceObject(), output_proc_encounter_csv_obj)

    #### Measurements from Procedures #####
    measurement_rules_proc_encounter = [(":row_id", row_map_offset("measurement_id", measurement_row_offset),
                                         {"measurement_id": "measurement_id"}),
                                        ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                                        (":row_id", ConstantMapper({"measurement_type_concept_id": 0}),
                                         # TODO: Add measurement_type_concept_id
                                         {"measurement_type_concept_id": "measurement_type_concept_id"}),
                                        ("s_encounter_id", s_encounter_id_mapper,
                                         {"visit_occurrence_id": "visit_occurrence_id"}),
                                        ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                                         {"date": "measurement_date", "time": "measurement_time"}),
                                        ("s_start_procedure_datetime", DateTimeWithTZ(),
                                         {"datetime": "measurement_datetime"}),
                                        ("s_procedure_code", "measurement_source_value"),
                                        (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                                         {"CONCEPT_ID".lower(): "measurement_source_concept_id",
                                          "MAPPED_CONCEPT_ID".lower(): "measurement_concept_id"})]

    measurement_rules_proc_encounter_class = build_input_output_mapper(measurement_rules_proc_encounter)

    output_measurement_proc_encounter_csv = os.path.join(output_csv_directory, "measurement_proc_cdm.csv")
    output_measurement_proc_encounter_csv_obj = OutputClassCSVRealization(output_measurement_proc_encounter_csv,
                                                                          MeasurementObject())

    output_directory_obj.register(MeasurementObject(), output_measurement_proc_encounter_csv_obj)

    in_out_map_obj.register(SourceProcedureObject(), MeasurementObject(), measurement_rules_proc_encounter_class)

    #### Observations from Procedures #####
    observation_rules_proc = [(":row_id", row_map_offset("observation_id", observation_row_offset),
                               {"observation_id": "observation_id"}),
                              (":row_id", ConstantMapper({"observation_type_concept_id": 0}),
                               {"observation_type_concept_id": "observation_type_concept_id"}),
                              ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                              ("s_encounter_id",
                               s_encounter_id_mapper,
                               {"visit_occurrence_id": "visit_occurrence_id"}),
                              ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                               {"date": "observation_date", "time": "observation_time"}),
                              ("s_procedure_code", "observation_source_value"),
                              ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "observation_datetime"}),
                              (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                               {"CONCEPT_ID".lower(): "observation_source_concept_id",
                                "MAPPED_CONCEPT_ID".lower(): "observation_concept_id"})]

    observation_rules_proc_class = build_input_output_mapper(observation_rules_proc)
    output_observation_proc_csv = os.path.join(output_csv_directory, "observation_proc_cdm.csv")
    output_observation_proc_csv_obj = OutputClassCSVRealization(output_observation_proc_csv,
                                                                ObservationObject())

    output_directory_obj.register(ObservationObject(), output_observation_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), ObservationObject(), observation_rules_proc_class)

    ##### DrugExposure from Procedures ####
    drug_rules_proc = [(":row_id", "drug_exposure_id"),
                       (":row_id", ConstantMapper({"drug_type_concept_id": 0}),
                        {"drug_type_concept_id": "drug_type_concept_id"}),
                       ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                       ("s_encounter_id", s_encounter_id_mapper,
                        {"visit_occurrence_id": "visit_occurrence_id"}),
                       ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                        {"date": "drug_exposure_start_date"}),
                       ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                        {"date": "drug_exposure_end_date"}),
                       ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "drug_exposure_start_datetime"}),
                       ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "drug_exposure_end_datetime"}),
                       ("s_procedure_code", "drug_source_value"),
                       (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                        {"CONCEPT_ID".lower(): "drug_source_concept_id",
                         "MAPPED_CONCEPT_ID".lower(): "drug_concept_id"})]

    drug_rules_proc_class = build_input_output_mapper(drug_rules_proc)
    output_drug_proc_csv = os.path.join(output_csv_directory, "drug_exposure_proc_cdm.csv")
    output_drug_proc_csv_obj = OutputClassCSVRealization(output_drug_proc_csv,
                                                         DrugExposureObject())

    output_directory_obj.register(DrugExposureObject(), output_drug_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), DrugExposureObject(), drug_rules_proc_class)

    #### Device Exposure from Procedures ####

    device_rules_proc = [(":row_id", "device_exposure_id"),
                         (":row_id", ConstantMapper({"device_type_concept_id": 0}),
                          {"device_type_concept_id": "device_type_concept_id"}),
                         ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                         ("s_encounter_id",
                          s_encounter_id_mapper,
                          {"visit_occurrence_id": "visit_occurrence_id"}),
                         ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                          {"date": "device_exposure_start_date"}),
                         ("s_start_procedure_datetime", DateTimeWithTZ(),
                          {"datetime": "device_exposure_start_datetime"}),
                         ("s_procedure_code", "device_source_value"),
                         (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                          {"CONCEPT_ID".lower(): "device_source_concept_id",
                           "MAPPED_CONCEPT_ID".lower(): "device_concept_id"})]

    device_rules_proc_class = build_input_output_mapper(device_rules_proc)
    output_device_proc_csv = os.path.join(output_csv_directory, "device_exposure_proc_cdm.csv")
    output_device_proc_csv_obj = OutputClassCSVRealization(output_device_proc_csv,
                                                           DeviceExposureObject())

    output_directory_obj.register(DeviceExposureObject(), output_device_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), DeviceExposureObject(), device_rules_proc_class)

    def procedure_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):

            if "m_procedure_code_oid" in input_dict:

                if procedure_coding_system(input_dict) in ("ICD9 Procedure Codes", "ICD10 Procedure Codes", "CPT Codes",
                                                           "HCPCS"):
                    result_dict = procedure_code_map.map(input_dict)

                    if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict or "DOMAIN_ID".lower() in result_dict:
                        if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict:
                            domain = result_dict["MAPPED_CONCEPT_DOMAIN".lower()]
                        else:
                            domain = result_dict["DOMAIN_ID".lower()]

                        if domain == "Procedure":
                            return ProcedureOccurrenceObject()
                        elif domain == "Measurement":
                            return MeasurementObject()
                        elif domain == "Observation":
                            return ObservationObject()
                        elif domain == "Drug":
                            return DrugExposureObject()
                        elif domain == "Device":
                            return DeviceExposureObject()
                        else:
                            return NoOutputClass()
                    else:
                        return NoOutputClass()
                else:
                    return NoOutputClass()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    procedure_runner_obj = RunMapperAgainstSingleInputRealization(hi_proc_csv_obj, in_out_map_obj,
                                                                  output_directory_obj,
                                                                  procedure_router_obj)

    procedure_runner_obj.run()

    drug_row_offset = procedure_runner_obj.rows_run

    #### DRUG EXPOSURE ####
    def drug_exposure_router_obj(input_dict):
        """Route mapping of drug_exposure"""

        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                return DrugExposureObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    def drug_post_processing(output_dict):
        """For concept_id"""
        fields = ["drug_concept_id", "drug_source_concept_id"]
        for field in fields:
            if field not in output_dict:
                output_dict[field] = 0
            else:
                if output_dict[field] is not None:
                    if not len(output_dict[field]):
                        output_dict[field] = 0
        return output_dict<commit_msg>Final run for 7/2018<commit_after>"""
Mapping data extracted from a prepared source. A first level mapping from
the sources file has been completed. Writing ETLs for the OMOP CDM is complex
because of the mappings and that a single source file can map to multiple tables.

Fields in the prepared source start either start with s_  for source or m_ mapped
to a name part of the OHDSI vocabulary.
"""

import os
import sys

try:
    from source_to_cdm_functions import *
    from omop_cdm_classes_5_2 import *
    from prepared_source_classes import *
    from mapping_classes import *
except ImportError:
    sys.path.insert(0, os.path.abspath(os.path.join(os.path.split(__file__)[0], os.path.pardir, "src")))
    from source_to_cdm_functions import *
    from omop_cdm_classes_5_2 import *
    from prepared_source_classes import *
    from mapping_classes import *

import logging
import argparse

logging.basicConfig(level=logging.INFO)


def drug_post_processing(output_dict):
    """For concept_id"""
    fields = ["drug_concept_id", "drug_source_concept_id"]
    for field in fields:
        if field not in output_dict:
            output_dict[field] = 0
        else:
            if output_dict[field] is not None:
                if not len(output_dict[field]):
                    output_dict[field] = 0
    return output_dict


def main(input_csv_directory, output_csv_directory, json_map_directory):
    # TODO: Add Provider
    # TODO: Add Patient Location

    output_class_obj = OutputClassDirectory()
    in_out_map_obj = InputOutputMapperDirectory()
    output_directory_obj = OutputClassDirectory()

    #### Person ####

    input_person_csv = os.path.join(input_csv_directory, "source_person.csv")
    output_person_csv = os.path.join(output_csv_directory, "person_cdm.csv")

    person_rules = create_person_rules(json_map_directory)

    person_runner_obj = generate_mapper_obj(input_person_csv, SourcePersonObject(), output_person_csv, PersonObject(),
                                            person_rules,
                                            output_class_obj, in_out_map_obj, person_router_obj)

    person_runner_obj.run()

    #### Death ####

    # Generate look up for s_person_id
    output_person_csv = os.path.join(output_csv_directory, "person_cdm.csv")
    person_json_file_name = create_json_map_from_csv_file(output_person_csv, "person_source_value", "person_id")
    s_person_id_mapper = CoderMapperJSONClass(person_json_file_name)

    death_rules = create_death_person_rules(json_map_directory, s_person_id_mapper)

    output_death_csv = os.path.join(output_csv_directory, "death_cdm.csv")
    death_runner_obj = generate_mapper_obj(input_person_csv, SourcePersonObject(), output_death_csv, DeathObject(),
                                           death_rules, output_class_obj, in_out_map_obj, death_router_obj)
    death_runner_obj.run()

    #### Observation_Period ####

    obs_per_rules = create_observation_period_rules(json_map_directory, s_person_id_mapper)

    output_obs_per_csv = os.path.join(output_csv_directory, "observation_period_cdm.csv")

    input_obs_per_csv = os.path.join(input_csv_directory, "source_observation_period.csv")

    def obs_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            return ObservationPeriodObject()
        else:
            return NoOutputClass()

    obs_per_runner_obj = generate_mapper_obj(input_obs_per_csv, SourceObservationPeriodObject(), output_obs_per_csv,
                                             ObservationPeriodObject(),
                                             obs_per_rules, output_class_obj, in_out_map_obj, obs_router_obj)
    obs_per_runner_obj.run()

    #### Care Sites ####

    care_site_rules = [
        (":row_id", "care_site_id"),
        ("s_care_site_name", "care_site_name"),
        ("k_care_site", "care_site_source_value")]

    input_care_site_csv = os.path.join(input_csv_directory, "source_care_site.csv")
    output_care_site_csv = os.path.join(output_csv_directory, "care_site_cdm.csv")

    def care_site_router_obj(input_dict):
        return CareSiteObject()

    care_site_runner_obj = generate_mapper_obj(input_care_site_csv, SourceCareSiteObject(), output_care_site_csv,
                                           CareSiteObject(), care_site_rules,
                                           output_class_obj, in_out_map_obj, care_site_router_obj)

    care_site_runner_obj.run()

    care_site_json_file_name = create_json_map_from_csv_file(output_care_site_csv, "care_site_source_value",
                                                             "care_site_id")

    k_care_site_mapper = CoderMapperJSONClass(care_site_json_file_name, "k_care_site")

    #### Visit_Occurrence ###

    snomed_code_json = os.path.join(json_map_directory, "CONCEPT_CODE_SNOMED.json")
    snomed_code_mapper = CodeMapperClassSqliteJSONClass(snomed_code_json)

    visit_rules = create_visit_rules(json_map_directory, s_person_id_mapper, k_care_site_mapper, snomed_code_mapper)
    input_encounter_csv = os.path.join(input_csv_directory, "source_encounter.csv")
    output_visit_occurrence_csv = os.path.join(output_csv_directory, "visit_occurrence_cdm.csv")

    def visit_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != '1':
                return VisitOccurrenceObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    visit_runner_obj = generate_mapper_obj(input_encounter_csv, SourceEncounterObject(), output_visit_occurrence_csv,
                                           VisitOccurrenceObject(), visit_rules,
                                           output_class_obj, in_out_map_obj, visit_router_obj)

    visit_runner_obj.run()

    # Visit ID Map
    encounter_json_file_name = create_json_map_from_csv_file(output_visit_occurrence_csv, "visit_source_value",
                                                             "visit_occurrence_id")

    s_encounter_id_mapper = CoderMapperJSONClass(encounter_json_file_name, "s_encounter_id")

    # Visit ID Map
    encounter_json_file_name = create_json_map_from_csv_file(output_visit_occurrence_csv, "visit_source_value",
                                                             "visit_occurrence_id")

    #### Benefit Coverage Period ####

    payer_plan_period_rules = create_payer_plan_period_rules(s_person_id_mapper)

    output_ppp_csv = os.path.join(output_csv_directory, "payer_plan_period_cdm.csv")

    input_ppp_csv = os.path.join(input_csv_directory, "source_encounter_coverage.csv")

    def payer_plan_period_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            return PayerPlanPeriodObject()
        else:
            return NoOutputClass()

    payer_plan_period_runner_obj = generate_mapper_obj(input_ppp_csv, SourceEncounterCoverageObject(), output_ppp_csv,
                                                       PayerPlanPeriodObject(),
                                                       payer_plan_period_rules, output_class_obj, in_out_map_obj,
                                                       payer_plan_period_router_obj
                                                       )
    payer_plan_period_runner_obj.run()

    #### MEASUREMENT and OBSERVATION dervived from 'source_observation.csv' ####

    snomed_code_result_mapper = ChainMapper(FilterHasKeyValueMapper(["s_type_code"]), snomed_code_mapper)

    def measurement_router_obj(input_dict):
        """Determine if the result contains a LOINC code"""
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                mapped_result_code = snomed_code_result_mapper.map(input_dict)
                if "CONCEPT_CLASS_ID".lower() in mapped_result_code:
                    if mapped_result_code["DOMAIN_ID".lower()] == "Measurement":
                        return MeasurementObject()
                    elif mapped_result_code["DOMAIN_ID".lower()] == "Observation":
                        return ObservationObject()
                    else:
                        return NoOutputClass()
                else:
                    return MeasurementObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    snomed_json = os.path.join(json_map_directory, "CONCEPT_NAME_SNOMED.json")
    snomed_mapper = CodeMapperClassSqliteJSONClass(snomed_json)

    measurement_rules, observation_measurement_rules = \
        create_measurement_and_observation_rules(json_map_directory, s_person_id_mapper, s_encounter_id_mapper, snomed_mapper,
                                                 snomed_code_mapper)

    input_result_csv = os.path.join(input_csv_directory, "source_result.csv")
    output_measurement_csv = os.path.join(output_csv_directory, "measurement_encounter_cdm.csv")

    measurement_runner_obj = generate_mapper_obj(input_result_csv, SourceResultObject(), output_measurement_csv,
                                                 MeasurementObject(),
                                                 measurement_rules, output_class_obj, in_out_map_obj,
                                                 measurement_router_obj)

    output_observation_csv = os.path.join(output_csv_directory, "observation_measurement_encounter_cdm.csv")
    register_to_mapper_obj(input_result_csv, SourceResultObject(), output_observation_csv,
                           ObservationObject(), observation_measurement_rules, output_class_obj, in_out_map_obj)

    measurement_runner_obj.run()

    #### CONDITION / DX ####

    condition_type_name_json = os.path.join(json_map_directory, "CONCEPT_NAME_Condition_Type.json")
    condition_type_name_map = CoderMapperJSONClass(condition_type_name_json)

    condition_claim_type_map = \
        ChainMapper(
            ReplacementMapper({"Primary": "Primary Condition", "Secondary": "Secondary Condition"}),
            condition_type_name_map
        )

    input_condition_csv = os.path.join(input_csv_directory, "source_condition.csv")
    hi_condition_csv_obj = InputClassCSVRealization(input_condition_csv, SourceConditionObject())

    output_condition_csv = os.path.join(output_csv_directory, "condition_occurrence_dx_cdm.csv")
    cdm_condition_csv_obj = OutputClassCSVRealization(output_condition_csv, ConditionOccurrenceObject())

    icd9cm_json = os.path.join(json_map_directory, "ICD9CM_with_parent.json")
    icd10cm_json = os.path.join(json_map_directory, "ICD10CM_with_parent.json")

    def case_mapper_icd9_icd10(input_dict, field="m_condition_code_oid"):
        """Map ICD9 and ICD10 to the CDM vocabularies"""
        coding_system_oid = input_dict[field]
        coding_version = condition_coding_system(coding_system_oid)

        if coding_version == "ICD9CM":
            return 0
        else:
            return 1

    # TODO: condition_type_concept_id
    condition_encounter_mapper = ChainMapper(ConstantMapper({"diagnosis_type_name": "Observation recorded from EHR"}),
                                             condition_type_name_map)

    condition_type_concept_mapper = CascadeMapper(condition_claim_type_map, condition_encounter_mapper)

    def clean_concept_ids(input_dict):

        if "CONCEPT_ID".lower() in input_dict:
            if input_dict["CONCEPT_ID".lower()] == "" or input_dict["CONCEPT_ID".lower()] is None:
                input_dict["CONCEPT_ID".lower()] = 0
        else:
            input_dict["CONCEPT_ID".lower()] = 0

        if "MAPPED_CONCEPT_ID".lower() in input_dict:

            if input_dict["MAPPED_CONCEPT_ID".lower()] == "" or input_dict["MAPPED_CONCEPT_ID".lower()] is None:
                input_dict["MAPPED_CONCEPT_ID".lower()] = 0

        else:
            input_dict["MAPPED_CONCEPT_ID".lower()] = 0

        return input_dict


    ICDMapper = ChainMapper(CaseMapper(case_mapper_icd9_icd10,
                            CodeMapperClassSqliteJSONClass(icd9cm_json, "s_condition_code"),
                            CodeMapperClassSqliteJSONClass(icd10cm_json, "s_condition_code")), PassThroughFunctionMapper(clean_concept_ids))

    s_condition_type_dict = {"Admitting": "52870002", "Final": "89100005", "Preliminary": "148006"}
    condition_status_snomed_mapper = CodeMapperDictClass(s_condition_type_dict, "s_condition_type")
    condition_status_mapper = ChainMapper(condition_status_snomed_mapper, snomed_code_mapper)


    # Required: condition_occurrence_id, person_id, condition_concept_id, condition_start_date
    condition_rules_dx = [(":row_id", "condition_occurrence_id"),
                          ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                          ("s_encounter_id", s_encounter_id_mapper, {"visit_occurrence_id": "visit_occurrence_id"}),
                          (("s_condition_code", "m_condition_code_oid"),
                           ICDMapper,
                           {"CONCEPT_ID".lower(): "condition_source_concept_id", "MAPPED_CONCEPT_ID".lower(): "condition_concept_id"}),
                          ("s_condition_code", "condition_source_value"),
                          ("m_rank", condition_type_concept_mapper, {"CONCEPT_ID".lower(): "condition_type_concept_id"}),
                          ("s_condition_type", condition_status_mapper, {"CONCEPT_ID".lower(): "condition_status_concept_id"}),
                          ("s_condition_type", "condition_status_source_value"),
                          ("s_start_condition_datetime", SplitDateTimeWithTZ(), {"date": "condition_start_date"}),
                          ("s_start_condition_datetime", DateTimeWithTZ(), {"datetime": "condition_start_datetime"}),
                          ("s_end_condition_datetime", SplitDateTimeWithTZ(), {"date": "condition_end_date"}),
                          ("s_end_condition_datetime", DateTimeWithTZ(), {"datetime": "condition_end_datetime"})
                          ]

    condition_rules_dx_class = build_input_output_mapper(condition_rules_dx)

    # ICD9 and ICD10 conditions which map to measurements according to the CDM Vocabulary
    in_out_map_obj.register(SourceConditionObject(), ConditionOccurrenceObject(), condition_rules_dx_class)
    output_directory_obj.register(ConditionOccurrenceObject(), cdm_condition_csv_obj)

    measurement_row_offset = measurement_runner_obj.rows_run
    measurement_rules_dx = [(":row_id", row_map_offset("measurement_id", measurement_row_offset),
                             {"measurement_id": "measurement_id"}),
                            (":row_id", ConstantMapper({"measurement_type_concept_id": 0}),
                             {"measurement_type_concept_id": "measurement_type_concept_id"}),
                            ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                            ("s_encounter_id", s_encounter_id_mapper,
                             {"visit_occurrence_id": "visit_occurrence_id"}),
                            ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                             {"date": "measurement_date", "time": "measurement_time"}),
                            ("s_start_condition_datetime", DateTimeWithTZ(),
                             {"datetime": "measurement_datetime"}),
                            ("s_condition_code", "measurement_source_value"),
                            (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                             {"CONCEPT_ID".lower(): "measurement_source_concept_id",
                              "MAPPED_CONCEPT_ID".lower(): "measurement_concept_id"})]

    measurement_rules_dx_class = build_input_output_mapper(measurement_rules_dx)
    in_out_map_obj.register(SourceConditionObject(), MeasurementObject(), measurement_rules_dx_class)

    # The mapped ICD9 to measurements get mapped to a separate code
    output_measurement_dx_encounter_csv = os.path.join(output_csv_directory, "measurement_dx_cdm.csv")
    output_measurement_dx_encounter_csv_obj = OutputClassCSVRealization(output_measurement_dx_encounter_csv,
                                                                        MeasurementObject())

    output_directory_obj.register(MeasurementObject(), output_measurement_dx_encounter_csv_obj)

    observation_row_offset = measurement_runner_obj.rows_run

    # ICD9 and ICD10 codes which map to observations according to the CDM Vocabulary
    observation_rules_dx = [(":row_id", row_map_offset("observation_id", observation_row_offset),
                             {"observation_id": "observation_id"}),
                            (":row_id", ConstantMapper({"observation_type_concept_id": 0}),
                             {"observation_type_concept_id": "observation_type_concept_id"}),
                            ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                            (("s_encounter_id",),
                             s_encounter_id_mapper,
                             {"visit_occurrence_id": "visit_occurrence_id"}),
                            ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                             {"date": "observation_date", "time": "observation_time"}),
                            ("s_start_condition_datetime", DateTimeWithTZ(),
                             {"datetime": "observation_datetime"}),
                            ("s_condition_code", "observation_source_value"),
                            (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                             {"CONCEPT_ID".lower(): "observation_source_concept_id",
                              "MAPPED_CONCEPT_ID".lower(): "observation_concept_id"}),
                            ("m_rank", condition_claim_type_map,
                             {"CONCEPT_ID".lower(): "condition_type_concept_id"})]

    observation_rules_dx_class = build_input_output_mapper(observation_rules_dx)

    output_observation_dx_encounter_csv = os.path.join(output_csv_directory, "observation_dx_cdm.csv")
    output_observation_dx_encounter_csv_obj = OutputClassCSVRealization(output_observation_dx_encounter_csv,
                                                                        ObservationObject())

    output_directory_obj.register(ObservationObject(), output_observation_dx_encounter_csv_obj)
    in_out_map_obj.register(SourceConditionObject(), ObservationObject(), observation_rules_dx_class)

    procedure_type_json = os.path.join(json_map_directory, "CONCEPT_NAME_Procedure_Type.json")
    procedure_type_mapper = CoderMapperJSONClass(procedure_type_json)

    # ICD9 and ICD10 codes which map to procedures according to the CDM Vocabulary
    # "Procedure recorded as diagnostic code"
    # TODO: Map procedure_type_concept_id

    procedure_rules_dx_encounter = [(":row_id", "procedure_occurrence_id"),
                                    (":row_id",
                                     ChainMapper(ConstantMapper({"name": "Procedure recorded as diagnostic code"}),
                                                 procedure_type_mapper),
                                     {"CONCEPT_ID".lower(): "procedure_type_concept_id"}),
                                    ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                                    ("s_encounter_id", s_encounter_id_mapper,
                                     {"visit_occurrence_id": "visit_occurrence_id"}),
                                    ("s_start_condition_datetime", SplitDateTimeWithTZ(),
                                     {"date": "procedure_date"}),
                                    ("s_start_condition_datetime", DateTimeWithTZ(),
                                     {"datetime": "procedure_datetime"}),
                                    ("s_condition_code", "procedure_source_value"),
                                    (("s_condition_code", "m_condition_code_oid"), ICDMapper,
                                     {"CONCEPT_ID".lower(): "procedure_source_concept_id",
                                      "MAPPED_CONCEPT_ID".lower(): "procedure_concept_id"})]

    procedure_rules_dx_encounter_class = build_input_output_mapper(procedure_rules_dx_encounter)

    output_procedure_dx_encounter_csv = os.path.join(output_csv_directory, "procedure_dx_cdm.csv")
    output_procedure_dx_encounter_csv_obj = OutputClassCSVRealization(output_procedure_dx_encounter_csv,
                                                                      ProcedureOccurrenceObject())

    output_directory_obj.register(ProcedureOccurrenceObject(), output_procedure_dx_encounter_csv_obj)
    in_out_map_obj.register(SourceConditionObject(), ProcedureOccurrenceObject(), procedure_rules_dx_encounter_class)

    def condition_router_obj(input_dict):
        """ICD9 / ICD10 CM contain codes which could either be a procedure, observation, or measurement"""
        coding_system_oid = input_dict["m_condition_code_oid"]
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                if coding_system_oid:
                    result_dict = ICDMapper.map(input_dict)

                    if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict or "DOMAIN_ID".lower() in result_dict:
                        if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict:
                            domain = result_dict["MAPPED_CONCEPT_DOMAIN".lower()]
                        else:
                            domain = result_dict["DOMAIN_ID".lower()]
                    else:
                        domain = ""

                    if result_dict != {}:
                        if domain == "Condition":
                            return ConditionOccurrenceObject()
                        elif domain == "Observation":
                            return ObservationObject()
                        elif domain == "Procedure":
                            return ProcedureOccurrenceObject()
                        elif domain == "Measurement":
                            return MeasurementObject()
                        else:
                            return NoOutputClass()
                    else:
                        return NoOutputClass()
                else:
                    return NoOutputClass()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    condition_runner_obj = RunMapperAgainstSingleInputRealization(hi_condition_csv_obj, in_out_map_obj,
                                                                  output_directory_obj,
                                                                  condition_router_obj)

    condition_runner_obj.run()

    # Update needed offsets
    condition_row_offset = condition_runner_obj.rows_run
    procedure_row_offset = condition_runner_obj.rows_run
    measurement_row_offset += condition_row_offset
    observation_row_offset += condition_row_offset

    procedure_rules_encounter = create_procedure_rules(json_map_directory, s_person_id_mapper, s_encounter_id_mapper,
                                                       procedure_row_offset)
    procedure_rule = procedure_rules_encounter[0]
    procedure_code_map = procedure_rule[1]

    procedure_rules_encounter_class = build_input_output_mapper(procedure_rules_encounter)

    input_proc_csv = os.path.join(input_csv_directory, "source_procedure.csv")
    hi_proc_csv_obj = InputClassCSVRealization(input_proc_csv, SourceProcedureObject())

    in_out_map_obj.register(SourceProcedureObject(), ProcedureOccurrenceObject(), procedure_rules_encounter_class)

    output_proc_encounter_csv = os.path.join(output_csv_directory, "procedure_cdm.csv")
    output_proc_encounter_csv_obj = OutputClassCSVRealization(output_proc_encounter_csv,
                                                              ProcedureOccurrenceObject())

    output_directory_obj.register(ProcedureOccurrenceObject(), output_proc_encounter_csv_obj)

    #### Measurements from Procedures #####
    measurement_rules_proc_encounter = [(":row_id", row_map_offset("measurement_id", measurement_row_offset),
                                         {"measurement_id": "measurement_id"}),
                                        ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                                        (":row_id", ConstantMapper({"measurement_type_concept_id": 0}),
                                         # TODO: Add measurement_type_concept_id
                                         {"measurement_type_concept_id": "measurement_type_concept_id"}),
                                        ("s_encounter_id", s_encounter_id_mapper,
                                         {"visit_occurrence_id": "visit_occurrence_id"}),
                                        ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                                         {"date": "measurement_date", "time": "measurement_time"}),
                                        ("s_start_procedure_datetime", DateTimeWithTZ(),
                                         {"datetime": "measurement_datetime"}),
                                        ("s_procedure_code", "measurement_source_value"),
                                        (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                                         {"CONCEPT_ID".lower(): "measurement_source_concept_id",
                                          "MAPPED_CONCEPT_ID".lower(): "measurement_concept_id"})]

    measurement_rules_proc_encounter_class = build_input_output_mapper(measurement_rules_proc_encounter)

    output_measurement_proc_encounter_csv = os.path.join(output_csv_directory, "measurement_proc_cdm.csv")
    output_measurement_proc_encounter_csv_obj = OutputClassCSVRealization(output_measurement_proc_encounter_csv,
                                                                          MeasurementObject())

    output_directory_obj.register(MeasurementObject(), output_measurement_proc_encounter_csv_obj)

    in_out_map_obj.register(SourceProcedureObject(), MeasurementObject(), measurement_rules_proc_encounter_class)

    #### Observations from Procedures #####
    observation_rules_proc = [(":row_id", row_map_offset("observation_id", observation_row_offset),
                               {"observation_id": "observation_id"}),
                              (":row_id", ConstantMapper({"observation_type_concept_id": 0}),
                               {"observation_type_concept_id": "observation_type_concept_id"}),
                              ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                              ("s_encounter_id",
                               s_encounter_id_mapper,
                               {"visit_occurrence_id": "visit_occurrence_id"}),
                              ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                               {"date": "observation_date", "time": "observation_time"}),
                              ("s_procedure_code", "observation_source_value"),
                              ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "observation_datetime"}),
                              (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                               {"CONCEPT_ID".lower(): "observation_source_concept_id",
                                "MAPPED_CONCEPT_ID".lower(): "observation_concept_id"})]

    observation_rules_proc_class = build_input_output_mapper(observation_rules_proc)
    output_observation_proc_csv = os.path.join(output_csv_directory, "observation_proc_cdm.csv")
    output_observation_proc_csv_obj = OutputClassCSVRealization(output_observation_proc_csv,
                                                                ObservationObject())

    output_directory_obj.register(ObservationObject(), output_observation_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), ObservationObject(), observation_rules_proc_class)

    ##### DrugExposure from Procedures ####
    drug_rules_proc = [(":row_id", "drug_exposure_id"),
                       (":row_id", ConstantMapper({"drug_type_concept_id": 0}),
                        {"drug_type_concept_id": "drug_type_concept_id"}),
                       ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                       ("s_encounter_id", s_encounter_id_mapper,
                        {"visit_occurrence_id": "visit_occurrence_id"}),
                       ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                        {"date": "drug_exposure_start_date"}),
                       ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                        {"date": "drug_exposure_end_date"}),
                       ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "drug_exposure_start_datetime"}),
                       ("s_start_procedure_datetime", DateTimeWithTZ(), {"datetime": "drug_exposure_end_datetime"}),
                       ("s_procedure_code", "drug_source_value"),
                       (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                        {"CONCEPT_ID".lower(): "drug_source_concept_id",
                         "MAPPED_CONCEPT_ID".lower(): "drug_concept_id"})]

    drug_rules_proc_class = build_input_output_mapper(drug_rules_proc)
    output_drug_proc_csv = os.path.join(output_csv_directory, "drug_exposure_proc_cdm.csv")
    output_drug_proc_csv_obj = OutputClassCSVRealization(output_drug_proc_csv,
                                                         DrugExposureObject())

    output_directory_obj.register(DrugExposureObject(), output_drug_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), DrugExposureObject(), drug_rules_proc_class)

    #### Device Exposure from Procedures ####

    device_rules_proc = [(":row_id", "device_exposure_id"),
                         (":row_id", ConstantMapper({"device_type_concept_id": 0}),
                          {"device_type_concept_id": "device_type_concept_id"}),
                         ("s_person_id", s_person_id_mapper, {"person_id": "person_id"}),
                         ("s_encounter_id",
                          s_encounter_id_mapper,
                          {"visit_occurrence_id": "visit_occurrence_id"}),
                         ("s_start_procedure_datetime", SplitDateTimeWithTZ(),
                          {"date": "device_exposure_start_date"}),
                         ("s_start_procedure_datetime", DateTimeWithTZ(),
                          {"datetime": "device_exposure_start_datetime"}),
                         ("s_procedure_code", "device_source_value"),
                         (("s_procedure_code", "m_procedure_code_oid"), procedure_code_map,
                          {"CONCEPT_ID".lower(): "device_source_concept_id",
                           "MAPPED_CONCEPT_ID".lower(): "device_concept_id"})]

    device_rules_proc_class = build_input_output_mapper(device_rules_proc)
    output_device_proc_csv = os.path.join(output_csv_directory, "device_exposure_proc_cdm.csv")
    output_device_proc_csv_obj = OutputClassCSVRealization(output_device_proc_csv,
                                                           DeviceExposureObject())

    output_directory_obj.register(DeviceExposureObject(), output_device_proc_csv_obj)
    in_out_map_obj.register(SourceProcedureObject(), DeviceExposureObject(), device_rules_proc_class)

    def procedure_router_obj(input_dict):
        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):

            if "m_procedure_code_oid" in input_dict:

                if procedure_coding_system(input_dict) in ("ICD9 Procedure Codes", "ICD10 Procedure Codes", "CPT Codes",
                                                           "HCPCS"):
                    result_dict = procedure_code_map.map(input_dict)

                    if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict or "DOMAIN_ID".lower() in result_dict:
                        if "MAPPED_CONCEPT_DOMAIN".lower() in result_dict:
                            domain = result_dict["MAPPED_CONCEPT_DOMAIN".lower()]
                        else:
                            domain = result_dict["DOMAIN_ID".lower()]

                        if domain == "Procedure":
                            return ProcedureOccurrenceObject()
                        elif domain == "Measurement":
                            return MeasurementObject()
                        elif domain == "Observation":
                            return ObservationObject()
                        elif domain == "Drug":
                            return DrugExposureObject()
                        elif domain == "Device":
                            return DeviceExposureObject()
                        else:
                            return NoOutputClass()
                    else:
                        return NoOutputClass()
                else:
                    return NoOutputClass()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()

    procedure_runner_obj = RunMapperAgainstSingleInputRealization(hi_proc_csv_obj, in_out_map_obj,
                                                                  output_directory_obj,
                                                                  procedure_router_obj, post_map_func=drug_post_processing)

    procedure_runner_obj.run()

    drug_row_offset = procedure_runner_obj.rows_run

    #### DRUG EXPOSURE ####
    def drug_exposure_router_obj(input_dict):
        """Route mapping of drug_exposure"""

        if len(s_person_id_mapper.map({"s_person_id": input_dict["s_person_id"]})):
            if input_dict["i_exclude"] != "1":
                return DrugExposureObject()
            else:
                return NoOutputClass()
        else:
            return NoOutputClass()
{'fraction_non_alphanumeric': 0.0863950739778796, 'fraction_numerical': 0.0024951119152537425, 'mean_word_length': 2.3107096275542243, 'pattern_counts': {'":': 202, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 226, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1779309', 'n_tokens_mistral': 19211, 'n_tokens_neox': 18031, 'n_words': 2629}