<commit_before>#!/usr/bin/env python
"""

generate_subject_map.py -  Tool to generate patient-to-research
subject mapping files based on inputs from REDCap projects

"""

__author__      = "CTS-IT team"
__copyright__   = "Copyright 2014, University of Florida"
__license__     = "BSD 3-Clause"
__version__     = "0.1"

import os
import argparse
import logging

from lxml import etree
import appdirs

import gsm_lib
from utils.sftpclient import SFTPClient
from utils.emailsender import EmailProps
from utils.emailsender import EmailSender
from utils.redcap_transactions import redcap_transactions

# This addresses the issues with relative paths
file_dir = os.path.dirname(os.path.realpath(__file__))
goal_dir = os.path.join(file_dir, "../")
proj_root = os.path.abspath(goal_dir) + '/'


def main():
    global configuration_directory

    # obtaining command line arguments for path to config directory
    args = parse_args()
    configuration_directory = os.path.abspath(args['configuration_directory_path'])
    do_keep_gen_files = args['keep']

    # Configure logging
    logger = configure_logging(args['verbose'], args['logfile'])

    conf_file = os.path.join(configuration_directory, 'settings.ini')
    settings = gsm_lib.get_settings(conf_file)
    gsm_lib.read_config(configuration_directory, conf_file, settings)
    site_catalog_file = os.path.join(configuration_directory, settings.site_catalog)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(settings, logger)
    # gets data from the person index for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, logger)
    xml_tree = etree.fromstring(response)

    #XSL Transformation : transforms the person_index data
    transform_xsl = proj_root + "rsm/utils/person_index_transform.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)
    person_index_data = transform(xml_tree)

    # # # retrieve smi.xml from the sftp server
    smi_path = get_smi_and_parse(site_catalog_file, settings, logger)
    try:
        smi_data = etree.parse(smi_path)
    except IOError:
        logger.exception("Could not open file: " + smi_path)
        raise

    #sorting both the xml files.
    gsm_lib.sort_element_tree(smi_data)
    gsm_lib.sort_element_tree(person_index_data)
    #generating the person index dictionary
    person_index_dict = {}
    for item in person_index_data.iter('item'):
        person_index_dict[item.findtext('research_subject_id')] = [
            item.findtext('yob'),
            item.findtext('mrn'),
            item.findtext('facility_code')]
    #iterate through the smi data and generate a
    # new merged xml's for subject_map and subject_map_exceptions
    subjectmap_root = etree.Element("subject_map_records")
    subjectmap_exceptions_root = etree.Element("subject_map_exception_records")
    exceptions = False
    for item in smi_data.iter('item'):
        if item.findtext('research_subject_id') in person_index_dict.keys():
            logger.debug("Processing research_subject_id %s", item.findtext('research_subject_id'))
            if (person_index_dict[item.findtext('research_subject_id')][0] == item.findtext('yob')):
                logger.debug("yob matched for research_subject_id %s", item.findtext('research_subject_id'))
                mrn = etree.SubElement(item, "mrn")
                mrn.text = person_index_dict[item.findtext('research_subject_id')][1]
                facility_code = etree.SubElement(item, "facility_code")
                facility_code.text = person_index_dict[item.findtext('research_subject_id')][2]
                item.remove(item.find('yob'))
                subjectmap_root.append(item)

            else:
                logger.debug("yob not matched for research_subject_id %s", item.findtext('research_subject_id'))
                exception_item = etree.Element("item")
                research_subject_id = etree.SubElement(exception_item, "research_subject_id")
                research_subject_id.text = item.findtext('research_subject_id')
                if (research_subject_id.text is not None):
                    exceptions = True
                pi_yob = etree.SubElement(exception_item, "Person_Index_YOB")
                pi_yob.text = person_index_dict[item.findtext('research_subject_id')][0]
                hcvt_yob = etree.SubElement(exception_item, "HCVTarget_YOB")
                hcvt_yob.text = item.findtext('yob')
                subjectmap_exceptions_root.append(exception_item)

    #Below code transforms the xml files to csv files
    transform_xsl = proj_root + "rsm/utils/xml2csv.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)

    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)
    subject_map_file = tmp_folder + "subject_map.csv"
    logger.info('Using path subject map file path: ' + subject_map_file)

    try:
        subject_map_csv = open(subject_map_file, "w")
        subject_map_csv.write("%s" % '"research_subject_id","start_date","end_date","mrn","facility_code"\n')

        for item in subjectmap_root.iter("item"):
            line = '"{0}","{1}","{2}","{3}","{4}"\n'.format(
                gsm_lib.handle_blanks(item.findtext("research_subject_id")),
                gsm_lib.handle_blanks(item.findtext("start_date")),
                gsm_lib.handle_blanks(item.findtext("end_date")),
                gsm_lib.handle_blanks(item.findtext("mrn")),
                gsm_lib.handle_blanks(item.findtext("facility_code")))
            subject_map_csv.write("%s" % line)

        subject_map_csv.close()
    except IOError:
        logger.exception("Could not open file %s for write", subject_map_file)
        raise

    # remove the smi.xml from the folder because the XSLT process
    # writes data to smi.xml
    try:
        os.remove(smi_path)
    except OSError:
        logger.exception("Could not remove file %s ", smi_path)
        raise

    # send the subject_map.csv to EMR team (sftp server)
    parse_site_details_and_send(site_catalog_file, subject_map_file, 'sftp', settings, logger)
    if do_keep_gen_files:
        logger.info('Keeping the temporary file: ' + subject_map_file)
    else:
        logger.info('Removing the temporary file: ' + subject_map_file)
        os.remove(subject_map_file)

    # send subject_map_exceptions.csv as email attachment
    if exceptions:
        subject_map_exceptions_file = tmp_folder + 'subject_map_exceptions.csv'
        try:
            subject_map_exceptions_csv = open(subject_map_exceptions_file, "w")
        except IOError:
            logger.exception("Could not open file %s for write", subject_map_exceptions_file)
            raise
        subject_map_exceptions_csv.write("%s" % '"research_subject_id","person_index_yob","redcap_yob"\n')
        for item in subjectmap_exceptions_root.iter("item"):
            line = '"{0}","{1}","{2}"\n'.format(
                gsm_lib.handle_blanks(item.find("research_subject_id").text),
                gsm_lib.handle_blanks(item.find("Person_Index_YOB").text),
                gsm_lib.handle_blanks(item.find("HCVTarget_YOB").text))
            subject_map_exceptions_csv.write("%s" % line)
        subject_map_exceptions_csv.close()

        parse_site_details_and_send(site_catalog_file, subject_map_exceptions_file, 'email', settings, logger)
        if do_keep_gen_files:
            logger.info('Keeping the temporary file: ' + subject_map_exceptions_file)
        else:
            logger.info('Removing the temporary file: ' + subject_map_exceptions_file)
            os.remove(subject_map_exceptions_file)


def parse_args():
    """Parses command line arguments"""
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', required=False,
                        dest='configuration_directory_path',
                        default=proj_root + 'config/',
                        help='Specify the path to the configuration directory')

    parser.add_argument('-k', '--keep', required=False,
                        default=False, action='store_true',
                        help='keep files generated during execution')<commit_msg>Add debugging feature to gsm, the -d option<commit_after>#!/usr/bin/env python
"""

generate_subject_map.py -  Tool to generate patient-to-research
subject mapping files based on inputs from REDCap projects

"""

__author__      = "CTS-IT team"
__copyright__   = "Copyright 2014, University of Florida"
__license__     = "BSD 3-Clause"
__version__     = "0.1"

import os
import argparse
import logging

from lxml import etree
import appdirs

import gsm_lib
from utils.sftpclient import SFTPClient
from utils.emailsender import EmailProps
from utils.emailsender import EmailSender
from utils.redcap_transactions import redcap_transactions

# This addresses the issues with relative paths
file_dir = os.path.dirname(os.path.realpath(__file__))
goal_dir = os.path.join(file_dir, "../")
proj_root = os.path.abspath(goal_dir) + '/'


def main():
    global configuration_directory

    # obtaining command line arguments for path to config directory
    args = parse_args()
    configuration_directory = os.path.abspath(args['configuration_directory_path'])
    do_keep_gen_files = args['keep']
    debugging = args['debug']

    # Configure logging
    logger = configure_logging(args['verbose'], args['logfile'])

    conf_file = os.path.join(configuration_directory, 'settings.ini')
    settings = gsm_lib.get_settings(conf_file)
    gsm_lib.read_config(configuration_directory, conf_file, settings)
    site_catalog_file = os.path.join(configuration_directory, settings.site_catalog)

    if debugging:
        if settings.redcap_log_file:
            redcap_log_file = os.path.join(configuration_directory, settings.redcap_log_file)
        else:
            redcap_log_file = 'redcap.log.xml'

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(settings, logger)
    # gets data from the person index for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, logger)

    if debugging:
        try:
            print('Writing REDCap response to file: ')
            with open(redcap_log_file, 'w') as log_file:
                log_file.write(response)
        except:
            print('Unable to write the REDCap response to ' + redcap_log_file)

    try:
        xml_tree = etree.fromstring(response)

        #XSL Transformation : transforms the person_index data
        transform_xsl = proj_root + "rsm/utils/person_index_transform.xsl"
        xslt = etree.parse(transform_xsl)
        transform = etree.XSLT(xslt)
        person_index_data = transform(xml_tree)
    except Exception as xe:
        if debugging:
            print('Unable to transform data returned by REDCap:')
            print('REDCap response logged to ' + redcap_log_file)
            print('Ecountered the following error:')
            print(xe)
            exit()
        else:
            print('Unable to transform data returned by REDCap:')
            exit()

    # # # retrieve smi.xml from the sftp server
    smi_path = get_smi_and_parse(site_catalog_file, settings, logger)
    try:
        smi_data = etree.parse(smi_path)
    except IOError:
        logger.exception("Could not open file: " + smi_path)
        raise

    #sorting both the xml files.
    gsm_lib.sort_element_tree(smi_data)
    gsm_lib.sort_element_tree(person_index_data)
    #generating the person index dictionary
    person_index_dict = {}
    for item in person_index_data.iter('item'):
        person_index_dict[item.findtext('research_subject_id')] = [
            item.findtext('yob'),
            item.findtext('mrn'),
            item.findtext('facility_code')]
    #iterate through the smi data and generate a
    # new merged xml's for subject_map and subject_map_exceptions
    subjectmap_root = etree.Element("subject_map_records")
    subjectmap_exceptions_root = etree.Element("subject_map_exception_records")
    exceptions = False
    for item in smi_data.iter('item'):
        if item.findtext('research_subject_id') in person_index_dict.keys():
            logger.debug("Processing research_subject_id %s", item.findtext('research_subject_id'))
            if (person_index_dict[item.findtext('research_subject_id')][0] == item.findtext('yob')):
                logger.debug("yob matched for research_subject_id %s", item.findtext('research_subject_id'))
                mrn = etree.SubElement(item, "mrn")
                mrn.text = person_index_dict[item.findtext('research_subject_id')][1]
                facility_code = etree.SubElement(item, "facility_code")
                facility_code.text = person_index_dict[item.findtext('research_subject_id')][2]
                item.remove(item.find('yob'))
                subjectmap_root.append(item)

            else:
                logger.debug("yob not matched for research_subject_id %s", item.findtext('research_subject_id'))
                exception_item = etree.Element("item")
                research_subject_id = etree.SubElement(exception_item, "research_subject_id")
                research_subject_id.text = item.findtext('research_subject_id')
                if (research_subject_id.text is not None):
                    exceptions = True
                pi_yob = etree.SubElement(exception_item, "Person_Index_YOB")
                pi_yob.text = person_index_dict[item.findtext('research_subject_id')][0]
                hcvt_yob = etree.SubElement(exception_item, "HCVTarget_YOB")
                hcvt_yob.text = item.findtext('yob')
                subjectmap_exceptions_root.append(exception_item)

    #Below code transforms the xml files to csv files
    transform_xsl = proj_root + "rsm/utils/xml2csv.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)

    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)
    subject_map_file = tmp_folder + "subject_map.csv"
    logger.info('Using path subject map file path: ' + subject_map_file)

    try:
        subject_map_csv = open(subject_map_file, "w")
        subject_map_csv.write("%s" % '"research_subject_id","start_date","end_date","mrn","facility_code"\n')

        for item in subjectmap_root.iter("item"):
            line = '"{0}","{1}","{2}","{3}","{4}"\n'.format(
                gsm_lib.handle_blanks(item.findtext("research_subject_id")),
                gsm_lib.handle_blanks(item.findtext("start_date")),
                gsm_lib.handle_blanks(item.findtext("end_date")),
                gsm_lib.handle_blanks(item.findtext("mrn")),
                gsm_lib.handle_blanks(item.findtext("facility_code")))
            subject_map_csv.write("%s" % line)

        subject_map_csv.close()
    except IOError:
        logger.exception("Could not open file %s for write", subject_map_file)
        raise

    # remove the smi.xml from the folder because the XSLT process
    # writes data to smi.xml
    try:
        os.remove(smi_path)
    except OSError:
        logger.exception("Could not remove file %s ", smi_path)
        raise

    # send the subject_map.csv to EMR team (sftp server)
    parse_site_details_and_send(site_catalog_file, subject_map_file, 'sftp', settings, logger)
    if do_keep_gen_files:
        logger.info('Keeping the temporary file: ' + subject_map_file)
    else:
        logger.info('Removing the temporary file: ' + subject_map_file)
        os.remove(subject_map_file)

    # send subject_map_exceptions.csv as email attachment
    if exceptions:
        subject_map_exceptions_file = tmp_folder + 'subject_map_exceptions.csv'
        try:
            subject_map_exceptions_csv = open(subject_map_exceptions_file, "w")
        except IOError:
            logger.exception("Could not open file %s for write", subject_map_exceptions_file)
            raise
        subject_map_exceptions_csv.write("%s" % '"research_subject_id","person_index_yob","redcap_yob"\n')
        for item in subjectmap_exceptions_root.iter("item"):
            line = '"{0}","{1}","{2}"\n'.format(
                gsm_lib.handle_blanks(item.find("research_subject_id").text),
                gsm_lib.handle_blanks(item.find("Person_Index_YOB").text),
                gsm_lib.handle_blanks(item.find("HCVTarget_YOB").text))
            subject_map_exceptions_csv.write("%s" % line)
        subject_map_exceptions_csv.close()

        parse_site_details_and_send(site_catalog_file, subject_map_exceptions_file, 'email', settings, logger)
        if do_keep_gen_files:
            logger.info('Keeping the temporary file: ' + subject_map_exceptions_file)
        else:
            logger.info('Removing the temporary file: ' + subject_map_exceptions_file)
            os.remove(subject_map_exceptions_file)


def parse_args():
    """Parses command line arguments"""
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', required=False,
                        dest='configuration_directory_path',
                        default=proj_root + 'config/',
                        help='Specify the path to the configuration directory')

    parser.add_argument('-d', '--debug', required=False,
                        default=False, action='store_true',
                        help='enable debugging')

    parser.add_argument('-k', '--keep', required=False,
                        default=False, action='store_true',
                        help='keep files generated during execution')
{'fraction_non_alphanumeric': 0.09954188111532153, 'fraction_numerical': 0.0022623154798936712, 'mean_word_length': 3.072316904652234, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 23}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7291300', 'n_tokens_mistral': 5467, 'n_tokens_neox': 5263, 'n_words': 1040}