<commit_before>import os.path
import os

from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermover import mover_utils
from pilot.api import data

# logger
baseLogger = core_utils.setup_logger()


# plugin for preparator based on Pilot2.0 Data API
# Pilot 2.0 should be deployed as library
# default self.basePath came from preparator section of configuration file

class PilotmoverPreparator(PluginBase):
    """
    Praparator bring files from remote ATLAS/Rucio storage to local facility. 
    """

    # constructor
    def __init__(self, **kwarg):
        PluginBase.__init__(self, **kwarg)

    # check status
    def check_status(self, jobspec):
        return True, ''

    # trigger preparation
    def trigger_preparation(self, jobspec):
        # make logger
        tmpLog = core_utils.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID))
        tmpLog.debug('start')        
       
        # check that jobspec.computingSite is defined
        if jobspec.computingSite is None:
            # not found
            tmpLog.error('jobspec.computingSite is not defined')
            return False, 'jobspec.computingSite is not defined'
        else:
            tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
        # get input files
        files = []
        inFiles = jobspec.get_input_file_attributes()
        # set path to each file
        for inLFN, inFile in inFiles.iteritems():
            # check if path exists if not create it.
            if not os.access(self.basePath, os.F_OK):
                os.makedirs(self.basePath)
            files.append({'scope': inFile['scope'],
                          'name': inLFN,
                          'destination': self.basePath})
        tmpLog.debug('files[] {0}'.format(files))
        data_client = data.StageInClient(site=jobspec.computingSite)
        result = data_client.transfer(files)
        tmpLog.debug('pilot.api data.StageInClient.transfer(files) result: {0}'.format(result))
        # loop over each file check result all must be true for entire result to be true
        allChecked = True
        ErrMsg = 'These files failed to download : '
        for answer in result:
            if answer['errno'] != 0:
                allChecked = False
                ErrMsg = ErrMsg + (" %s " % answer['name'])
        # return
        tmpLog.debug('stop')
        if allChecked:
            return True, ''
        else:
            return False, ErrMsg

    #set local path for downloading
    #def setBasePath(self, BasePath):
    #    self.__basePath = BasePath
    #    return self.__basePath

    # resolve input file paths
    def resolve_input_paths(self, jobspec):
        # get input files
        inFiles = jobspec.get_input_file_attributes()
        # set path to each file
        for inLFN, inFile in inFiles.iteritems():
            inFile['path'] = self.basePath
        # set
        jobspec.set_input_file_paths(inFiles)
        return True, ''<commit_msg>full local path to file fix<commit_after>import os.path
import os

from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermover import mover_utils
from pilot.api import data

# logger
baseLogger = core_utils.setup_logger()


# plugin for preparator based on Pilot2.0 Data API
# Pilot 2.0 should be deployed as library
# default self.basePath came from preparator section of configuration file

class PilotmoverPreparator(PluginBase):
    """
    Praparator bring files from remote ATLAS/Rucio storage to local facility. 
    """

    # constructor
    def __init__(self, **kwarg):
        PluginBase.__init__(self, **kwarg)

    # check status
    def check_status(self, jobspec):
        return True, ''

    # trigger preparation
    def trigger_preparation(self, jobspec):
        # make logger
        tmpLog = core_utils.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID))
        tmpLog.debug('start')        
       
        # check that jobspec.computingSite is defined
        if jobspec.computingSite is None:
            # not found
            tmpLog.error('jobspec.computingSite is not defined')
            return False, 'jobspec.computingSite is not defined'
        else:
            tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
        # get input files
        files = []
        inFiles = jobspec.get_input_file_attributes()
        # set path to each file
        for inLFN, inFile in inFiles.iteritems():
            # check if path exists if not create it.
            if not os.access(self.basePath, os.F_OK):
                os.makedirs(self.basePath)
            files.append({'scope': inFile['scope'],
                          'name': inLFN,
                          'destination': self.basePath})
        tmpLog.debug('files[] {0}'.format(files))
        data_client = data.StageInClient(site=jobspec.computingSite)
        result = data_client.transfer(files)
        tmpLog.debug('pilot.api data.StageInClient.transfer(files) result: {0}'.format(result))
        # loop over each file check result all must be true for entire result to be true
        allChecked = True
        ErrMsg = 'These files failed to download : '
        for answer in result:
            if answer['errno'] != 0:
                allChecked = False
                ErrMsg = ErrMsg + (" %s " % answer['name'])
        # return
        tmpLog.debug('stop')
        if allChecked:
            return True, ''
        else:
            return False, ErrMsg

    #set local path for downloading
    #def setBasePath(self, BasePath):
    #    self.__basePath = BasePath
    #    return self.__basePath

    # resolve input file paths
    def resolve_input_paths(self, jobspec):
        # get input files
        inFiles = jobspec.get_input_file_attributes()
        # set path to each file
        for inLFN, inFile in inFiles.iteritems():
            inFile['path'] = self.basePath + inLFN if self.basePath[-1:] == '/' else self.basePath + '/' + inLFN
        # set
        jobspec.set_input_file_paths(inFiles)
        return True, ''
{'fraction_non_alphanumeric': 0.08944, 'fraction_numerical': 0.00304, 'mean_word_length': 2.355340848094471, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5105315', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1738, 'n_words': 489}