<commit_before>#!/usr/bin/env python

##
##  Copyright 2017 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms
##

import os
import sys
import glob
import pandas

import sibispy
from sibispy import sibislogger as slog
from sibispy import redcap_to_casesdir as r2c
import pwd 

if sys.argv.__len__() > 1 : 
    config_file = sys.argv[1]
else :
    config_file = os.path.join(os.path.dirname(sys.argv[0]), 'data', '.sibis-general-config.yml')

slog.init_log(False, False,'test_redcap_to_casesdir', 'test_redcap_to_casesdir',None)

session = sibispy.Session()
assert(session.configure(config_file=config_file,ordered_config_load_flag = True))

redcap_project = session.connect_server('data_entry', True) 
assert(redcap_project)

red2cas = r2c.redcap_to_casesdir() 
if not red2cas.configure(session,redcap_project.metadata) :
    sys.exit(1)

outdir = "/tmp/test_redcap_to_casedir"
if not os.path.exists(outdir) :
    os.makedirs(outdir)

#Kilian: NCANDA
cluster_submit_log = os.path.join(outdir,'cluster-submit.log')
cluster_job_log = os.path.join(session.get_beta_dir(),'cluster-job.log')
if os.path.exists(cluster_job_log):
     os.remove(cluster_job_log)

cluster_test_file = os.path.join(session.get_cases_dir(),"test_redcap_to_casesdir.cluster")
user_id =  os.getuid()
user_name = pwd.getpwuid(user_id )[ 0 ] 
cmd = 'echo "Submitting User:' + user_name + '(' + str(user_id) + ')"; echo "Executing User: ${LOGNAME}(${UID})"; touch ' + str(cluster_test_file) + '; rm -f '  + str(cluster_test_file) 
assert(red2cas.schedule_cluster_job(cmd, "test_redcap_to_casesdir", submit_log=cluster_submit_log, job_log=cluster_job_log, verbose = False))
print "Please check " +  cluster_job_log + " if cluster job was successfully executed !"<commit_msg>Added note that it only has to work on backend<commit_after>#!/usr/bin/env python

##
##  Copyright 2017 SRI International
##  See COPYING file distributed along with the package for the copyright and license terms
##

import os
import sys
import glob
import pandas

import sibispy
from sibispy import sibislogger as slog
from sibispy import redcap_to_casesdir as r2c
import pwd 

if sys.argv.__len__() > 1 : 
    config_file = sys.argv[1]
else :
    config_file = os.path.join(os.path.dirname(sys.argv[0]), 'data', '.sibis-general-config.yml')

slog.init_log(False, False,'test_redcap_to_casesdir', 'test_redcap_to_casesdir',None)

session = sibispy.Session()
assert(session.configure(config_file=config_file,ordered_config_load_flag = True))

redcap_project = session.connect_server('data_entry', True) 
assert(redcap_project)

red2cas = r2c.redcap_to_casesdir() 
if not red2cas.configure(session,redcap_project.metadata) :
    sys.exit(1)

outdir = "/tmp/test_redcap_to_casedir"
if not os.path.exists(outdir) :
    os.makedirs(outdir)

#Kilian: NCANDA
cluster_submit_log = os.path.join(outdir,'cluster-submit.log')
cluster_job_log = os.path.join(session.get_beta_dir(),'cluster-job.log')
if os.path.exists(cluster_job_log):
     os.remove(cluster_job_log)

cluster_test_file = os.path.join(session.get_cases_dir(),"test_redcap_to_casesdir.cluster")
user_id =  os.getuid()
user_name = pwd.getpwuid(user_id )[ 0 ] 
cmd = 'echo "Submitting User:' + user_name + '(' + str(user_id) + ')"; echo "Executing User: ${LOGNAME}(${UID})"; touch ' + str(cluster_test_file) + '; rm -f '  + str(cluster_test_file) 
print "== Only works on backend ==" 
assert(red2cas.schedule_cluster_job(cmd, "test_redcap_to_casesdir", submit_log=cluster_submit_log, job_log=cluster_job_log, verbose = False))
print "Please check " +  cluster_job_log + " if cluster job was successfully executed !"
{'fraction_non_alphanumeric': 0.12425068119891008, 'fraction_numerical': 0.007629427792915531, 'mean_word_length': 5.926415094339623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6323300', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1310, 'n_words': 262}