<commit_before># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
""" Set of interfaces that allow interaction with data. Currently
    available interfaces are:

    DataSource: Generic nifti to named Nifti interface
    DataSink: Generic named output from interfaces to data store

    To come :
    XNATSource, XNATSink

"""
from copy import deepcopy
import glob
import os
import shutil

from enthought.traits.trait_errors import TraitError

from nipype.interfaces.base import (Interface, CommandLine, Bunch,
                                    InterfaceResult, Interface,
                                    TraitedSpec, traits, File, Directory,
                                    BaseInterface,
                                    OutputMultiPath, DynamicTraitedSpec,
                                    BaseTraitedSpec, Undefined)
from nipype.utils.misc import isdefined
from nipype.utils.filemanip import (copyfile, list_to_filename,
                                    filename_to_list, FileNotFoundError)

def add_traits(base, names, trait_type=None):
    """ Add traits to a traited class.

    All traits are set to Undefined by default
    """
    if trait_type is None:
        trait_type = traits.Any
    undefined_traits = {}
    for key in names:
        base.add_trait(key, trait_type)
        undefined_traits[key] = Undefined
    base.trait_set(trait_change_notify=False, **undefined_traits)
    # access each trait
    for key in names:
        value = getattr(base, key)
    return base

class IOBase(BaseInterface):

    def _run_interface(self, runtime):
        runtime.returncode = 0
        return runtime

    def _list_outputs(self):
        raise NotImplementedError

    def _outputs(self):
        return self._add_output_traits(super(IOBase, self)._outputs())
    
    def _add_output_traits(self, base):
        return base
    
class DataSinkInputSpec(DynamicTraitedSpec):
    base_directory = Directory( 
        desc='Path to the base directory consisting of subject data.')
    container = traits.Str(desc = 'Folder within basedirectory in which to store output')
    parameterization = traits.Bool(True, usedefault=True,
                                   desc='store output in parameterized structure')
    strip_dir = Directory(desc='path to strip out of filename')
    _outputs = traits.Dict(traits.Str, value={}, usedefault=True)
    
    def __setattr__(self, key, value):
        if key not in self.copyable_trait_names():
            self._outputs[key] = value
        else:
            super(DataSinkInputSpec, self).__setattr__(key, value)
    
class DataSink(IOBase):
    """ Generic datasink module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """
    input_spec = DataSinkInputSpec

    def _get_dst(self, src):
        #print 'isrc', src
        path, fname = os.path.split(src)
        if self.inputs.parameterization:
            dst = path
            if isdefined(self.inputs.strip_dir):
                dst = dst.replace(self.inputs.strip_dir,'')
            folders = [folder for folder in dst.split(os.path.sep) if folder.startswith('_')]
            dst = os.path.sep.join(folders)
            if fname:
                dst = os.path.join(dst,fname)
        else:
            if fname:
                dst = fname
            else:
                dst = path.split(os.path.sep)[-1]
        if dst[0] == os.path.sep:
            dst = dst[1:]
        #print 'dst', dst
        return dst
        
    def _list_outputs(self):
        """Execute this module.
        """
        outdir = self.inputs.base_directory
        if not isdefined(outdir):
            outdir = '.'
        outdir = os.path.abspath(outdir)
        if isdefined(self.inputs.container):
            outdir = os.path.join(outdir, self.inputs.container)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        for key,files in self.inputs._outputs.items():
            files = filename_to_list(files)
            outfiles = []
            tempoutdir = outdir
            for d in key.split('.'):
                if d[0] == '@':
                    continue
                tempoutdir = os.path.join(tempoutdir,d)
            if not os.path.exists(tempoutdir):
                #print 'tmpoutdir', tempoutdir
                os.makedirs(tempoutdir)
            for src in filename_to_list(files):
                #print 'src',src
                src = os.path.abspath(src)
                if os.path.isfile(src):
                    dst = self._get_dst(src)
                    dst = os.path.join(tempoutdir, dst)
                    path,_ = os.path.split(dst)
                    if not os.path.exists(path):
                        #print 'path',path
                        os.makedirs(path)
                    #print 'copyfile',src, dst
                    copyfile(src, dst, copy=True)
                elif os.path.isdir(src):
                    dst = self._get_dst(os.path.join(src,''))
                    dst = os.path.join(tempoutdir, dst)
                    path,_ = os.path.split(dst)
                    if os.path.exists(path):
                        #print "removing: ", path
                        shutil.rmtree(path)
                    print "copydir", src, dst
                    shutil.copytree(src, dst)
        return None


class DataGrabberInputSpec(DynamicTraitedSpec): #InterfaceInputSpec):
    base_directory = Directory(exists=True,
            desc='Path to the base directory consisting of subject data.')
    template = traits.Str(mandatory=True,
             desc='Layout used to get files. relative to base directory if defined')
    template_args = traits.Dict(traits.Str,
                                traits.List(traits.List),
                                value=dict(outfiles=[]), usedefault=True,<commit_msg>fixed bug in datasink to prevent directory deletion<commit_after># emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
""" Set of interfaces that allow interaction with data. Currently
    available interfaces are:

    DataSource: Generic nifti to named Nifti interface
    DataSink: Generic named output from interfaces to data store

    To come :
    XNATSource, XNATSink

"""
from copy import deepcopy
import glob
import os
import shutil

from enthought.traits.trait_errors import TraitError

from nipype.interfaces.base import (Interface, CommandLine, Bunch,
                                    InterfaceResult, Interface,
                                    TraitedSpec, traits, File, Directory,
                                    BaseInterface,
                                    OutputMultiPath, DynamicTraitedSpec,
                                    BaseTraitedSpec, Undefined)
from nipype.utils.misc import isdefined
from nipype.utils.filemanip import (copyfile, list_to_filename,
                                    filename_to_list, FileNotFoundError)

import logging
iflogger = logging.getLogger('interface')

def add_traits(base, names, trait_type=None):
    """ Add traits to a traited class.

    All traits are set to Undefined by default
    """
    if trait_type is None:
        trait_type = traits.Any
    undefined_traits = {}
    for key in names:
        base.add_trait(key, trait_type)
        undefined_traits[key] = Undefined
    base.trait_set(trait_change_notify=False, **undefined_traits)
    # access each trait
    for key in names:
        value = getattr(base, key)
    return base

class IOBase(BaseInterface):

    def _run_interface(self, runtime):
        runtime.returncode = 0
        return runtime

    def _list_outputs(self):
        raise NotImplementedError

    def _outputs(self):
        return self._add_output_traits(super(IOBase, self)._outputs())
    
    def _add_output_traits(self, base):
        return base
    
class DataSinkInputSpec(DynamicTraitedSpec):
    base_directory = Directory( 
        desc='Path to the base directory consisting of subject data.')
    container = traits.Str(desc = 'Folder within basedirectory in which to store output')
    parameterization = traits.Bool(True, usedefault=True,
                                   desc='store output in parameterized structure')
    strip_dir = Directory(desc='path to strip out of filename')
    _outputs = traits.Dict(traits.Str, value={}, usedefault=True)
    
    def __setattr__(self, key, value):
        if key not in self.copyable_trait_names():
            self._outputs[key] = value
        else:
            super(DataSinkInputSpec, self).__setattr__(key, value)
    
class DataSink(IOBase):
    """ Generic datasink module that takes a directory containing a
        list of nifti files and provides a set of structured output
        fields.
    """
    input_spec = DataSinkInputSpec

    def _get_dst(self, src):
        #print 'isrc', src
        path, fname = os.path.split(src)
        if self.inputs.parameterization:
            dst = path
            if isdefined(self.inputs.strip_dir):
                dst = dst.replace(self.inputs.strip_dir,'')
            folders = [folder for folder in dst.split(os.path.sep) if folder.startswith('_')]
            dst = os.path.sep.join(folders)
            if fname:
                dst = os.path.join(dst,fname)
        else:
            if fname:
                dst = fname
            else:
                dst = path.split(os.path.sep)[-1]
        if dst[0] == os.path.sep:
            dst = dst[1:]
        #print 'dst', dst
        return dst
        
    def _list_outputs(self):
        """Execute this module.
        """
        outdir = self.inputs.base_directory
        if not isdefined(outdir):
            outdir = '.'
        outdir = os.path.abspath(outdir)
        if isdefined(self.inputs.container):
            outdir = os.path.join(outdir, self.inputs.container)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        for key,files in self.inputs._outputs.items():
            files = filename_to_list(files)
            outfiles = []
            tempoutdir = outdir
            for d in key.split('.'):
                if d[0] == '@':
                    continue
                tempoutdir = os.path.join(tempoutdir,d)
            if not os.path.exists(tempoutdir):
                #print 'tmpoutdir', tempoutdir
                os.makedirs(tempoutdir)
            for src in filename_to_list(files):
                #print 'src',src
                src = os.path.abspath(src)
                if os.path.isfile(src):
                    dst = self._get_dst(src)
                    dst = os.path.join(tempoutdir, dst)
                    path,_ = os.path.split(dst)
                    if not os.path.exists(path):
                        #print 'path',path
                        os.makedirs(path)
                    #print 'copyfile',src, dst
                    copyfile(src, dst, copy=True)
                elif os.path.isdir(src):
                    dst = self._get_dst(os.path.join(src,''))
                    dst = os.path.join(tempoutdir, dst)
                    if os.path.exists(dst):
                        iflogger.debug("removing: %s"%dst)
                        shutil.rmtree(dst)
                    iflogger.debug("copydir: %s %s"%(src, dst))
                    shutil.copytree(src, dst)
        return None


class DataGrabberInputSpec(DynamicTraitedSpec): #InterfaceInputSpec):
    base_directory = Directory(exists=True,
            desc='Path to the base directory consisting of subject data.')
    template = traits.Str(mandatory=True,
             desc='Layout used to get files. relative to base directory if defined')
    template_args = traits.Dict(traits.Str,
                                traits.List(traits.List),
                                value=dict(outfiles=[]), usedefault=True,
{'fraction_non_alphanumeric': 0.0908179463614337, 'fraction_numerical': 0.0015038850363438883, 'mean_word_length': 1.8231132075471699, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4590461', 'n_tokens_mistral': 3375, 'n_tokens_neox': 3248, 'n_words': 849}