<commit_before>#!/usr/bin/env python

##
##  See COPYING file distributed along with the ncanda-data-integration package
##  for the copyright and license terms
##

import os
import glob
import stat
import sys
import imp
from sibispy import sibislogger as slog

fields_list = dict()
functions = dict()
output_form = dict()
instrument_list = []

module_dir = os.path.dirname(os.path.abspath(__file__))

instruments = [ os.path.basename( d ) for d in glob.glob( os.path.join( module_dir, '*' ) ) if stat.S_ISDIR( os.stat( d ).st_mode ) and os.path.exists( os.path.join( d, '__init__.py' ) ) ]

sys.path.append( os.path.abspath( os.path.dirname( __file__ ) ) )

for i in instruments:
#    try:
        module_found = imp.find_module( i, [module_dir] )
        module = imp.load_module( i, module_found[0], module_found[1], module_found[2] )

        instrument_list.append( i )
        fields_list[i] =  module.input_fields
        functions[i] = module.compute_scores
        output_form[i] = module.output_form

def compute_scores(instrument,input_data,demographics):
    try:
        scoresDF = functions[instrument](input_data, demographics)   
    except Exception as e:
        error = "ERROR: scoring failed for instrument", instrument
        slog.info(instrument + "-" + hashlib.sha1(str(e)).hexdigest()[0:6], error, exception=str(e))
        return pandas.DataFrame()

    # remove nan entries as they corrupt data ingest (REDCAP cannot handle it correctly) and superfluous zeros
    # this gave an error as it only works for float values to replace 
    return scoresDF.astype(object).fillna('')   
    # return scoresDF(lambda x: '' if math.isnan(x) else x )<commit_msg>Failed when record was empty<commit_after>#!/usr/bin/env python

##
##  See COPYING file distributed along with the ncanda-data-integration package
##  for the copyright and license terms
##

import os
import glob
import stat
import sys
import imp
from sibispy import sibislogger as slog

fields_list = dict()
functions = dict()
output_form = dict()
instrument_list = []

module_dir = os.path.dirname(os.path.abspath(__file__))

instruments = [ os.path.basename( d ) for d in glob.glob( os.path.join( module_dir, '*' ) ) if stat.S_ISDIR( os.stat( d ).st_mode ) and os.path.exists( os.path.join( d, '__init__.py' ) ) ]

sys.path.append( os.path.abspath( os.path.dirname( __file__ ) ) )

for i in instruments:
#    try:
        module_found = imp.find_module( i, [module_dir] )
        module = imp.load_module( i, module_found[0], module_found[1], module_found[2] )

        instrument_list.append( i )
        fields_list[i] =  module.input_fields
        functions[i] = module.compute_scores
        output_form[i] = module.output_form

def compute_scores(instrument,input_data,demographics):
    try:
        scoresDF = functions[instrument](input_data, demographics)   
    except Exception as e:
        error = "ERROR: scoring failed for instrument", instrument
        slog.info(instrument + "-" + hashlib.sha1(str(e)).hexdigest()[0:6], error, exception=str(e))
        return pandas.DataFrame()

    # remove nan entries as they corrupt data ingest (REDCAP cannot handle it correctly) and superfluous zeros
    # this gave an error as it only works for float values to replace
    if len(scoresDF) :
        # Only execute it not empty 
        return scoresDF.astype(object).fillna('')   
            
    return scoresDF

    # return scoresDF(lambda x: '' if math.isnan(x) else x )
{'fraction_non_alphanumeric': 0.11181556195965418, 'fraction_numerical': 0.00345821325648415, 'mean_word_length': 3.51953125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3203990', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1088, 'n_words': 313}