<commit_before>#!/usr/bin/env python

##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##

import os
import os.path
import glob
import re
import time
import filecmp

import yaml
import pandas

import redcap


# Truncate age to 2 digits for increased identity protection
def truncate_age(age_in):
    match = re.match('([0-9]*\.[0-9]*)', str(age_in))
    if match:
        return round(float(match.group(1)), 2)
    else:
        return age_in


# "Safe" CSV export - this will catch IO errors from trying to write to a file
# that is currently being read and will retry a number of times before giving up
# This function will also confirm whether the newly created file is different
# from an already existing file of the same name. Only changed files will be
# updated.
def safe_csv_export(df, fname, verbose=False):
    success = False
    retries = 10

    while (not success) and (retries > 0):
        try:
            df.to_csv(fname + '.new', index=False)
            success = True
        except IOError as e:
            print "ERROR: failed to write file", fname, "with errno", e.errno
            if e.errno == 11:
                print "Retrying in 5s..."
                time.sleep(5)
                retries -= 1
            else:
                retries = 0

    if success:
        # Check if new file is equal to old file
        if os.path.exists(fname) and filecmp.cmp(fname, fname + '.new',
                                                 shallow=False):
            # Equal - remove new file
            os.remove(fname + '.new')
        else:
            # Not equal or no old file: put new file in its final place
            os.rename(fname + '.new', fname)

            if verbose:
                print "Updated", fname


# Export selected REDCap data to pipeline/distribution directory
def export(redcap_project, redcap_key, subject_data, visit_age, visit_data,
           arm_code, visit_code, subject_code, subject_datadir,
           forms_this_event, select_exports=None, verbose=False):
    redcap_subject, redcap_event = redcap_key

    # Mark subjects/visits that have QA completed by creating a hidden marker
    # file
    qafile_path = os.path.join(subject_datadir, '.qacomplete')
    if visit_data['mri_qa_completed'] == '1':
        try:
            if not os.path.exists(qafile_path):
                qafile = open(qafile_path, 'w')
                qafile.close()
        except:
            print "ERROR: unable to open QA marker file in", subject_datadir
    else:
        try:
            if os.path.exists(qafile_path):
                os.remove(qafile_path)
        except:
            print "ERROR: unable to remove QA marker file", qafile_path

    # Check if the "measures" subdirectory already exists - this is where all
    # the csv files go. Create it if necessary.
    measures_dir = os.path.join(subject_datadir, 'measures')
    if not os.path.exists(measures_dir):
        os.makedirs(measures_dir)

    # Export demographics (if selected)
    if not select_exports or 'demographics' in select_exports:
        # Create "demographics" file "by hand" - this has some data not (yet)
        # in REDCap.

        # Latino and race coding arrives here as floating point numbers; make
        # int strings from that (cannot use "int()" because it would fail for
        # missing data
        hispanic_code = re.sub('(.0)|(nan)', '', str(subject_data['hispanic']))
        race_code = re.sub('(.0)|(nan)', '', str(subject_data['race']))

        # scanner manufacturer map
        mfg = dict(A='S', B='G', C='G', D='S', E='G')

        # site - first letter for the site id
        site = redcap_subject[0]

        # Site changes are mapped here to the correct case identifier.
        # Get the sibis config
        yml = os.path.join(os.path.expanduser("~"), '.server_config/config.yml')
        with open(yml, 'r') as fi:
            sibis_config = yaml.load(fi).get('operations')
        if not sibis_config:
            raise IOError(
                "Please ensure config.yml file exists at: {}".format(yml))

        # Get a map of subjects that changed sited
        with open(os.path.join(sibis_config, 'special_cases.yml')) as fi:
            site_change_map = yaml.load(fi).get('site_change')
            export_measures_map = site_change_map.get('export_measures')
        if redcap_subject in export_measures_map.iterkeys():
            # Only set for visits in the past
            visit_case_id_map = export_measures_map.get(redcap_subject)
            if visit_code in visit_case_id_map.iterkeys():
                case_id_map = visit_case_id_map.get(visit_code)
                # Use correct case id (NCANDA_S00001)
                subject_code = case_id_map.get('subject')
            else:
                # update with default info for future events
                defaults = visit_case_id_map.get('default')
                site = defaults.get('site')
                # Todo: Test how to swap site

        demographics = [
            ['subject', subject_code],
            ['arm', arm_code],
            ['visit', visit_code],
            ['site', site],
            ['site_label', redcap_subject],
            ['mfg', mfg[redcap_subject[0]]],
            ['sex', redcap_subject[8]],<commit_msg>fix: move yml up higher in exports<commit_after>#!/usr/bin/env python

##
##  Copyright 2016 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms.
##

import os
import os.path
import glob
import re
import time
import filecmp

import yaml
import pandas

import redcap


# Truncate age to 2 digits for increased identity protection
def truncate_age(age_in):
    match = re.match('([0-9]*\.[0-9]*)', str(age_in))
    if match:
        return round(float(match.group(1)), 2)
    else:
        return age_in


# "Safe" CSV export - this will catch IO errors from trying to write to a file
# that is currently being read and will retry a number of times before giving up
# This function will also confirm whether the newly created file is different
# from an already existing file of the same name. Only changed files will be
# updated.
def safe_csv_export(df, fname, verbose=False):
    success = False
    retries = 10

    while (not success) and (retries > 0):
        try:
            df.to_csv(fname + '.new', index=False)
            success = True
        except IOError as e:
            print "ERROR: failed to write file", fname, "with errno", e.errno
            if e.errno == 11:
                print "Retrying in 5s..."
                time.sleep(5)
                retries -= 1
            else:
                retries = 0

    if success:
        # Check if new file is equal to old file
        if os.path.exists(fname) and filecmp.cmp(fname, fname + '.new',
                                                 shallow=False):
            # Equal - remove new file
            os.remove(fname + '.new')
        else:
            # Not equal or no old file: put new file in its final place
            os.rename(fname + '.new', fname)

            if verbose:
                print "Updated", fname


# Export selected REDCap data to pipeline/distribution directory
def export(redcap_project, redcap_key, subject_data, visit_age, visit_data,
           arm_code, visit_code, subject_code, subject_datadir,
           forms_this_event, select_exports=None, verbose=False):
    redcap_subject, redcap_event = redcap_key

    # Site changes are mapped here to the correct case identifier.
    # Get the sibis config
    yml = os.path.join(os.path.expanduser("~"), '.server_config/config.yml')
    with open(yml, 'r') as fi:
        sibis_config = yaml.load(fi).get('operations')
    if not sibis_config:
        raise IOError(
            "Please ensure config.yml file exists at: {}".format(yml))

    # Get a map of subjects that changed sited
    with open(os.path.join(sibis_config, 'special_cases.yml')) as fi:
        site_change_map = yaml.load(fi).get('site_change')
        export_measures_map = site_change_map.get('export_measures')

    # Mark subjects/visits that have QA completed by creating a hidden marker
    # file
    qafile_path = os.path.join(subject_datadir, '.qacomplete')
    if visit_data['mri_qa_completed'] == '1':
        try:
            if not os.path.exists(qafile_path):
                qafile = open(qafile_path, 'w')
                qafile.close()
        except:
            print "ERROR: unable to open QA marker file in", subject_datadir
    else:
        try:
            if os.path.exists(qafile_path):
                os.remove(qafile_path)
        except:
            print "ERROR: unable to remove QA marker file", qafile_path

    # Check if the "measures" subdirectory already exists - this is where all
    # the csv files go. Create it if necessary.
    measures_dir = os.path.join(subject_datadir, 'measures')
    if not os.path.exists(measures_dir):
        os.makedirs(measures_dir)

    # Export demographics (if selected)
    if not select_exports or 'demographics' in select_exports:
        # Create "demographics" file "by hand" - this has some data not (yet)
        # in REDCap.

        # Latino and race coding arrives here as floating point numbers; make
        # int strings from that (cannot use "int()" because it would fail for
        # missing data
        hispanic_code = re.sub('(.0)|(nan)', '', str(subject_data['hispanic']))
        race_code = re.sub('(.0)|(nan)', '', str(subject_data['race']))

        # scanner manufacturer map
        mfg = dict(A='S', B='G', C='G', D='S', E='G')

        # site - first letter for the site id
        site = redcap_subject[0]

        # Handle mapping demographic info for subjects that changed site
        if redcap_subject in export_measures_map.iterkeys():
            # Only set for visits in the past
            visit_case_id_map = export_measures_map.get(redcap_subject)
            if visit_code in visit_case_id_map.iterkeys():
                case_id_map = visit_case_id_map.get(visit_code)
                # Use correct case id (NCANDA_S00001)
                subject_code = case_id_map.get('subject')
            else:
                # update with default info for future events
                defaults = visit_case_id_map.get('default')
                site = defaults.get('site')
                # Todo: Test how to swap site

        demographics = [
            ['subject', subject_code],
            ['arm', arm_code],
            ['visit', visit_code],
            ['site', site],
            ['site_label', redcap_subject],
            ['mfg', mfg[redcap_subject[0]]],
            ['sex', redcap_subject[8]],
{'fraction_non_alphanumeric': 0.09644151258942674, 'fraction_numerical': 0.005760475703800056, 'mean_word_length': 2.222754491017964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '663855', 'n_tokens_mistral': 3212, 'n_tokens_neox': 3036, 'n_words': 1008}