<commit_before>#!/usr/bin/env python

##
##  Copyright 2017 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms
##

import os
import sys
import glob
import pandas

import sibispy
from sibispy import sibislogger as slog
from sibispy import redcap_to_casesdir as r2c

if sys.argv.__len__() > 1 : 
    config_file = sys.argv[1]
else :
    config_file = os.path.join(os.path.dirname(sys.argv[0]), 'data', '.sibis-general-config.yml')

slog.init_log(False, False,'test_redcap_to_casesdir', 'test_redcap_to_casesdir',None)

session = sibispy.Session()
assert(session.configure(config_file=config_file,ordered_config_load_flag = True))

redcap_project = session.connect_server('data_entry', True) 
assert(redcap_project)

red2cas = r2c.redcap_to_casesdir() 
if not red2cas.configure(session,redcap_project.metadata) :
    sys.exit(1)

outdir = "/tmp/test_redcap_to_casedir"
if not os.path.exists(outdir) :
    os.makedirs(outdir)

assert(red2cas.schedule_cluster_job('touch /tmp/blubber.log', 'test_redcap_to_casesdir', log_file= os.path.join(outdir,'cluster-test.log'), verbose = False))

# Test creating data dictionaries
assert(red2cas.create_demographic_datadict(outdir))

#
# Test writing out a case specific demographic file 
# NCANDA SPECIFIC
#
visit_log_fields = ['study_id', 'redcap_data_access_group', 'visit_date',
                    'mri_qa_completed', 'mri_t1_age', 'mri_dti_age',
                    'mri_rsfmri_age','mri_scanner', 'visit_ignore','mri_xnat_sid']
visit_log_redcap = redcap_project.export_records(fields=visit_log_fields,
                                                 event_name='unique',
                                                 export_data_access_groups=True,
                                                 format='df')
# also referred to as key
subject_key = visit_log_redcap.index[0]
subject_red_id =  subject_key[0]
subject_event_id =  subject_key[1]
subject_site_id = str(subject_red_id.split('-')[0])

# also referred to as row 
subject_visit_data = visit_log_redcap.ix[subject_key]
visit_date = str(subject_visit_data['visit_date'])
# subject_red_visit_id =  subject_red_id + "-" + visit_date
subject_xnat_id =  str(subject_visit_data['mri_xnat_sid'])


subject_fields = ['study_id', 'dob',  'exclude', 'enroll_exception',<commit_msg>just added directory where one could see the outcome of cluster<commit_after>#!/usr/bin/env python

##
##  Copyright 2017 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms
##

import os
import sys
import glob
import pandas

import sibispy
from sibispy import sibislogger as slog
from sibispy import redcap_to_casesdir as r2c

if sys.argv.__len__() > 1 : 
    config_file = sys.argv[1]
else :
    config_file = os.path.join(os.path.dirname(sys.argv[0]), 'data', '.sibis-general-config.yml')

slog.init_log(False, False,'test_redcap_to_casesdir', 'test_redcap_to_casesdir',None)

session = sibispy.Session()
assert(session.configure(config_file=config_file,ordered_config_load_flag = True))

redcap_project = session.connect_server('data_entry', True) 
assert(redcap_project)

red2cas = r2c.redcap_to_casesdir() 
if not red2cas.configure(session,redcap_project.metadata) :
    sys.exit(1)

outdir = "/tmp/test_redcap_to_casedir"
if not os.path.exists(outdir) :
    os.makedirs(outdir)

#Kilian: NCANDA
assert(red2cas.schedule_cluster_job('touch /fs/ncanda-share/beta/blubber.log', 'test_redcap_to_casesdir', log_file= os.path.join(outdir,'cluster-test.log'), verbose = False))

# Test creating data dictionaries
assert(red2cas.create_demographic_datadict(outdir))

#
# Test writing out a case specific demographic file 
# NCANDA SPECIFIC
#
visit_log_fields = ['study_id', 'redcap_data_access_group', 'visit_date',
                    'mri_qa_completed', 'mri_t1_age', 'mri_dti_age',
                    'mri_rsfmri_age','mri_scanner', 'visit_ignore','mri_xnat_sid']
visit_log_redcap = redcap_project.export_records(fields=visit_log_fields,
                                                 event_name='unique',
                                                 export_data_access_groups=True,
                                                 format='df')
# also referred to as key
subject_key = visit_log_redcap.index[0]
subject_red_id =  subject_key[0]
subject_event_id =  subject_key[1]
subject_site_id = str(subject_red_id.split('-')[0])

# also referred to as row 
subject_visit_data = visit_log_redcap.ix[subject_key]
visit_date = str(subject_visit_data['visit_date'])
# subject_red_visit_id =  subject_red_id + "-" + visit_date
subject_xnat_id =  str(subject_visit_data['mri_xnat_sid'])


subject_fields = ['study_id', 'dob',  'exclude', 'enroll_exception',
{'fraction_non_alphanumeric': 0.10704521556256572, 'fraction_numerical': 0.007991587802313354, 'mean_word_length': 4.147186147186147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2342223', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1587, 'n_words': 297}