<commit_before>#! /usr/bin/env python
import argparse
import textwrap
import os.path as osp
import json
import os
import os.path as osp

class InputFileMissing(Exception):
    pass

class ALFAHelper(object):
    '''Returns filenames from existing items in the ALFA repository
    or suggests new ones according to the Axon ontology,
    generates commands based on various pipelines steps (parse_command)
    and identifies the last performed step in the processing of a subject'''
    def __init__(self, directory='/home/grg/data/ALFA_DWI', jsonfile='/home/grg/git/alfa/alfa_dwi_pipeline_io_aug2016.json'):
        from brainvisa import axon
        axon.initializeProcesses()
        import neuroHierarchy
        self.__db = neuroHierarchy.databases._databases[directory]
        self.args_types = json.load(open(jsonfile))

    def find_diskitem(self, subject, axontype='Any Type', fmt=None):
        res = self.__db.findDiskItem(exactType=True, **{'_type': axontype, 'subject':subject})
        if res is None:
            if fmt is None:
                raise Exception('No diskitem found. Provide a format to generate a filename for a new item.')
            else:
                res = self.__db.findOrCreateDiskItem(**{'_type': axontype, 'subject':subject, '_format': fmt})
        return res

    def get_ALFA_types(self):
        import neuroProcesses
        return list(set([each.type.name for each in self.__db.findDiskItems(**{'_type': 'Any Type'})]))

    def parse_command(self, subject, name, jsonfile='/home/grg/git/alfa/alfa_dwi_pipeline_aug2016.json'):
        '''Types starting with @ indicate that the corresponding files must exist before execution (ReadDiskItems). \n
        Types starting with > indicate that the corresponding files must exist after execution (WriteDiskItems). \n
        Strings starting with # designate hard-coded filenames.
        Types starting with ! indicate that the filenames will be returned without extension.
        '''
        j = json.load(open(jsonfile))
        types = self.args_types[name]
        dsk = []
        for each in types:
            t = each.strip('@#!>')
            d = t
            if not each.startswith('#'):
                if 'DTIFIT' in t:
                    fmt = 'Directory'
                elif 'Bvec' in t:
                    fmt = 'Bvec file'
                elif 'stats' in t:
                    fmt = 'CSV file'
                else:
                    fmt = 'gz compressed NIFTI-1 image'
                d = self.find_diskitem(subject, t, fmt=fmt).fullPath()
                if each.startswith('@'):
                    if not osp.isfile(d) and not osp.isdir(d):
                        raise InputFileMissing('%s not found (type %s) while declared as input (or remove the leading @)'%(d, t))
                if each.startswith('!'):
                    d = self.find_diskitem(subject, t, fmt=fmt).fullPath()
                    d = d[:d.index('.')]
            dsk.append(d)
        res = parse_command(j[name], dsk)

        return res

    def current_stage(self, subject):
        steps = ['denoising', 'eddycorrect', 'extractb0', 'fslbet.25', 'fslfast', 'denoise_t1', 'ants_t1', 'dilate', 'ants_dwi', 'ants_aal', 'rotcorr', 'dtifit', 'warp', 'warp_md', 'warp_md2MNI', 'roistats', 'roistats_mni']

        def __sort_input_first__(a):
            inp = []
            out = []
            oth = []
            for each in a:
                if each.startswith('@'):
                    inp.append(each)
                elif each.startswith('>'):<commit_msg>Few more commandlines<commit_after>#! /usr/bin/env python
import argparse
import textwrap
import os.path as osp
import json
import os
import os.path as osp

class InputFileMissing(Exception):
    pass

class ALFAHelper(object):
    '''Returns filenames from existing items in the ALFA repository
    or suggests new ones according to the Axon ontology,
    generates commands based on various pipelines steps (parse_command)
    and identifies the last performed step in the processing of a subject'''
    def __init__(self, directory='/home/grg/data/ALFA_DWI', jsonfile='/home/grg/git/alfa/alfa_dwi_pipeline_io_aug2016.json'):
        from brainvisa import axon
        axon.initializeProcesses()
        import neuroHierarchy
        self.__db = neuroHierarchy.databases._databases[directory]
        self.args_types = json.load(open(jsonfile))

    def find_diskitem(self, subject, axontype='Any Type', fmt=None):
        res = self.__db.findDiskItem(exactType=True, **{'_type': axontype, 'subject':subject})
        if res is None:
            if fmt is None:
                raise Exception('No diskitem found. Provide a format to generate a filename for a new item.')
            else:
                res = self.__db.findOrCreateDiskItem(**{'_type': axontype, 'subject':subject, '_format': fmt})
        return res

    def get_ALFA_types(self):
        import neuroProcesses
        return list(set([each.type.name for each in self.__db.findDiskItems(**{'_type': 'Any Type'})]))

    def parse_command(self, subject, name, jsonfile='/home/grg/git/alfa/alfa_dwi_pipeline_aug2016.json'):
        '''Types starting with @ indicate that the corresponding files must exist before execution (ReadDiskItems). \n
        Types starting with > indicate that the corresponding files must exist after execution (WriteDiskItems). \n
        Strings starting with # designate hard-coded filenames.
        Types starting with ! indicate that the filenames will be returned without extension.
        '''
        j = json.load(open(jsonfile))
        types = self.args_types[name]
        dsk = []
        for each in types:
            t = each.strip('@#!>')
            d = t
            if not each.startswith('#'):
                if 'DTIFIT' in t:
                    fmt = 'Directory'
                elif 'Bvec' in t:
                    fmt = 'Bvec file'
                elif 'stats' in t:
                    fmt = 'CSV file'
                else:
                    fmt = 'gz compressed NIFTI-1 image'
                d = self.find_diskitem(subject, t, fmt=fmt).fullPath()
                if each.startswith('@'):
                    if not osp.isfile(d) and not osp.isdir(d):
                        raise InputFileMissing('%s not found (type %s) while declared as input (or remove the leading @)'%(d, t))
                if each.startswith('!'):
                    d = self.find_diskitem(subject, t, fmt=fmt).fullPath()
                    d = d[:d.index('.')]
            dsk.append(d)
        res = parse_command(j[name], dsk)

        return res

    def current_stage(self, subject):
        steps = ['denoising', 'eddycorrect', 'extractb0', 'fslbet.25', 'fslfast', 'denoise_t1', 'ants_t1', 'dilate', 'ants_dwi', 'ants_aal', 'rotcorr', 'dtifit', 'warp_AAL', 'warp_AAL2DWI', 'warp_md', 'warp_md2MNI', 'roistats', 'roistats_mni']

        def __sort_input_first__(a):
            inp = []
            out = []
            oth = []
            for each in a:
                if each.startswith('@'):
                    inp.append(each)
                elif each.startswith('>'):
{'fraction_non_alphanumeric': 0.11149972020145495, 'fraction_numerical': 0.004336877448237269, 'mean_word_length': 2.223174030658251, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '859224', 'n_tokens_mistral': 2152, 'n_tokens_neox': 2052, 'n_words': 606}