<commit_before>import re
import sys

from pandajedi.jedicore import Interaction
from PostProcessorBase import PostProcessorBase

from pandaserver.dataservice import DataServiceUtils
from pandaserver.taskbuffer import EventServiceUtils


# post processor for ATLAS production
class AtlasProdPostProcessor (PostProcessorBase):

    # constructor
    def __init__(self,taskBufferIF,ddmIF):
        PostProcessorBase.__init__(self,taskBufferIF,ddmIF)


    # main
    def doPostProcess(self,taskSpec,tmpLog):
        # pre-check
        try:
            tmpStat = self.doPreCheck(taskSpec,tmpLog)
            if tmpStat:
                return self.SC_SUCCEEDED
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doPreCheck failed with {0}:{1}'.format(errtype.__name__,errvalue))
            return self.SC_FATAL
        # get DDM I/F
        ddmIF = self.ddmIF.getInterface(taskSpec.vo)
        # loop over all datasets
        for datasetSpec in taskSpec.datasetSpecList:
            # skip pseudo output datasets
            if datasetSpec.type in ['output'] and datasetSpec.isPseudo():
                continue
            try:
                # remove wrong files
                if datasetSpec.type in ['output']:
                    # get successful files
                    okFiles = self.taskBufferIF.getSuccessfulFiles_JEDI(datasetSpec.jediTaskID,datasetSpec.datasetID)
                    if okFiles == None:
                        tmpLog.warning('failed to get successful files for {0}'.format(datasetSpec.datasetName))
                        return self.SC_FAILED
                    # get files in dataset
                    ddmFiles = ddmIF.getFilesInDataset(datasetSpec.datasetName,skipDuplicate=False,ignoreUnknown=True)
                    tmpLog.debug('datasetID={0}:Name={1} has {2} files in DB, {3} files in DDM'.format(datasetSpec.datasetID,
                                                                                                      datasetSpec.datasetName,
                                                                                                      len(okFiles),len(ddmFiles)))
                    # check all files
                    toDelete = []
                    for tmpGUID,attMap in ddmFiles.iteritems():
                        if attMap['lfn'] not in okFiles:
                            did = {'scope':attMap['scope'], 'name':attMap['lfn']}
                            toDelete.append(did)
                            tmpLog.debug('delete {0} from {1}'.format(attMap['lfn'],datasetSpec.datasetName))
                    # delete
                    if toDelete != []:
                        ddmIF.deleteFilesFromDataset(datasetSpec.datasetName,toDelete)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to remove wrong files with {0}:{1}'.format(errtype.__name__,errvalue))
                return self.SC_FAILED
            try:
                # freeze output and log datasets
                if datasetSpec.type in ['output','log','trn_log']:
                    tmpLog.info('freezing datasetID={0}:Name={1}'.format(datasetSpec.datasetID,datasetSpec.datasetName))
                    ddmIF.freezeDataset(datasetSpec.datasetName,ignoreUnknown=True)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to freeze datasets with {0}:{1}'.format(errtype.__name__,errvalue))
                return self.SC_FAILED
            try:
                # delete transient datasets
                if datasetSpec.type in ['trn_output']:
                    tmpLog.debug('deleting datasetID={0}:Name={1}'.format(datasetSpec.datasetID,datasetSpec.datasetName))
                    retStr = ddmIF.deleteDataset(datasetSpec.datasetName,False,ignoreUnknown=True)
                    tmpLog.info(retStr)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to delete datasets with {0}:{1}'.format(errtype.__name__,errvalue))
        # check duplication
        if self.getFinalTaskStatus(taskSpec) in ['finished','done']:
            nDup = self.taskBufferIF.checkDuplication_JEDI(taskSpec.jediTaskID)
            tmpLog.debug('checked duplication with {0}'.format(nDup))
            if nDup > 0:
                errStr = 'paused since {0} duplication found'.format(nDup)
                taskSpec.oldStatus = self.getFinalTaskStatus(taskSpec)
                taskSpec.status = 'paused'
                taskSpec.setErrDiag(errStr)
                tmpLog.debug(errStr)
        # delete ES datasets
        if taskSpec.registerEsFiles():
            try:
                targetName = EventServiceUtils.getEsDatasetName(taskSpec.jediTaskID)
                tmpLog.debug('deleting ES dataset name={0}'.format(targetName))
                retStr = ddmIF.deleteDataset(targetName,False,ignoreUnknown=True)
                tmpLog.debug(retStr)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to delete ES dataset with {0}:{1}'.format(errtype.__name__,errvalue))
        try:
            self.doBasicPostProcess(taskSpec,tmpLog)
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doBasicPostProcess failed with {0}:{1}'.format(errtype.__name__,errvalue))
            return self.SC_FATAL
        return self.SC_SUCCEEDED


<commit_msg>skip duplication check for Test tasks<commit_after>import re
import sys

from pandajedi.jedicore import Interaction
from PostProcessorBase import PostProcessorBase

from pandaserver.dataservice import DataServiceUtils
from pandaserver.taskbuffer import EventServiceUtils


# post processor for ATLAS production
class AtlasProdPostProcessor (PostProcessorBase):

    # constructor
    def __init__(self,taskBufferIF,ddmIF):
        PostProcessorBase.__init__(self,taskBufferIF,ddmIF)


    # main
    def doPostProcess(self,taskSpec,tmpLog):
        # pre-check
        try:
            tmpStat = self.doPreCheck(taskSpec,tmpLog)
            if tmpStat:
                return self.SC_SUCCEEDED
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doPreCheck failed with {0}:{1}'.format(errtype.__name__,errvalue))
            return self.SC_FATAL
        # get DDM I/F
        ddmIF = self.ddmIF.getInterface(taskSpec.vo)
        # loop over all datasets
        for datasetSpec in taskSpec.datasetSpecList:
            # skip pseudo output datasets
            if datasetSpec.type in ['output'] and datasetSpec.isPseudo():
                continue
            try:
                # remove wrong files
                if datasetSpec.type in ['output']:
                    # get successful files
                    okFiles = self.taskBufferIF.getSuccessfulFiles_JEDI(datasetSpec.jediTaskID,datasetSpec.datasetID)
                    if okFiles == None:
                        tmpLog.warning('failed to get successful files for {0}'.format(datasetSpec.datasetName))
                        return self.SC_FAILED
                    # get files in dataset
                    ddmFiles = ddmIF.getFilesInDataset(datasetSpec.datasetName,skipDuplicate=False,ignoreUnknown=True)
                    tmpLog.debug('datasetID={0}:Name={1} has {2} files in DB, {3} files in DDM'.format(datasetSpec.datasetID,
                                                                                                      datasetSpec.datasetName,
                                                                                                      len(okFiles),len(ddmFiles)))
                    # check all files
                    toDelete = []
                    for tmpGUID,attMap in ddmFiles.iteritems():
                        if attMap['lfn'] not in okFiles:
                            did = {'scope':attMap['scope'], 'name':attMap['lfn']}
                            toDelete.append(did)
                            tmpLog.debug('delete {0} from {1}'.format(attMap['lfn'],datasetSpec.datasetName))
                    # delete
                    if toDelete != []:
                        ddmIF.deleteFilesFromDataset(datasetSpec.datasetName,toDelete)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to remove wrong files with {0}:{1}'.format(errtype.__name__,errvalue))
                return self.SC_FAILED
            try:
                # freeze output and log datasets
                if datasetSpec.type in ['output','log','trn_log']:
                    tmpLog.info('freezing datasetID={0}:Name={1}'.format(datasetSpec.datasetID,datasetSpec.datasetName))
                    ddmIF.freezeDataset(datasetSpec.datasetName,ignoreUnknown=True)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to freeze datasets with {0}:{1}'.format(errtype.__name__,errvalue))
                return self.SC_FAILED
            try:
                # delete transient datasets
                if datasetSpec.type in ['trn_output']:
                    tmpLog.debug('deleting datasetID={0}:Name={1}'.format(datasetSpec.datasetID,datasetSpec.datasetName))
                    retStr = ddmIF.deleteDataset(datasetSpec.datasetName,False,ignoreUnknown=True)
                    tmpLog.info(retStr)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to delete datasets with {0}:{1}'.format(errtype.__name__,errvalue))
        # check duplication
        if self.getFinalTaskStatus(taskSpec) in ['finished','done'] and taskSpec.gshare != 'Test':
            nDup = self.taskBufferIF.checkDuplication_JEDI(taskSpec.jediTaskID)
            tmpLog.debug('checked duplication with {0}'.format(nDup))
            if nDup > 0:
                errStr = 'paused since {0} duplication found'.format(nDup)
                taskSpec.oldStatus = self.getFinalTaskStatus(taskSpec)
                taskSpec.status = 'paused'
                taskSpec.setErrDiag(errStr)
                tmpLog.debug(errStr)
        # delete ES datasets
        if taskSpec.registerEsFiles():
            try:
                targetName = EventServiceUtils.getEsDatasetName(taskSpec.jediTaskID)
                tmpLog.debug('deleting ES dataset name={0}'.format(targetName))
                retStr = ddmIF.deleteDataset(targetName,False,ignoreUnknown=True)
                tmpLog.debug(retStr)
            except:
                errtype,errvalue = sys.exc_info()[:2]
                tmpLog.warning('failed to delete ES dataset with {0}:{1}'.format(errtype.__name__,errvalue))
        try:
            self.doBasicPostProcess(taskSpec,tmpLog)
        except:
            errtype,errvalue = sys.exc_info()[:2]
            tmpLog.error('doBasicPostProcess failed with {0}:{1}'.format(errtype.__name__,errvalue))
            return self.SC_FATAL
        return self.SC_SUCCEEDED



{'fraction_non_alphanumeric': 0.09225058844830708, 'fraction_numerical': 0.005975013579576317, 'mean_word_length': 1.8217113665389528, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249', 'n_tokens_mistral': 2944, 'n_tokens_neox': 2835, 'n_words': 542}