<commit_before>import json

from django.db.models import Q

from atlas.prodtask.models import ActionStaging, ActionDefault, DatasetStaging, StepAction, TTask
from datetime import timedelta

from atlas.prodtask.ddm_api import DDM
from atlas.prodtask.models import  StepExecution, InputRequestList, ProductionTask, ProductionDataset

from decimal import Decimal
from datetime import datetime
from rest_framework.decorators import api_view
from rest_framework.response import Response
import logging
# import os
from django.utils import timezone

from django.contrib.auth.decorators import login_required

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render

from atlas.prodtask.task_actions import _do_deft_action

_logger = logging.getLogger('prodtaskwebui')


def test_step_action(step_action_ids):
    pass

def check_staging_task(step_action_ids, check_archive=False):
    ddm = DDM()
    config = ActionDefault.objects.get(name='stage_creation').get_config()
    delay = config['delay']
    max_waite_time = config['max_waite_time']
    rule = config['rule']
    for waiting_step in step_action_ids:
        try:
            check_tasks_for_prestage(waiting_step, ddm, rule, delay, max_waite_time, check_archive)
        except Exception as e:
            _logger.error("Check replicas problem %s" % str(e))
            waiting_step = StepAction.objects.get(id=waiting_step)
            waiting_step.status = 'active'
            waiting_step.save()
    pass


class ResourceQueue(object):
    def __init__(self, resource_name):
        self.queue = []
        self.total_queued = 0
        self.resource_name = resource_name
        pass

    def priorities_queue(self):

        return

    def running_level(self):
        return 0

    def get_limits(self):
        self.minimum_level = 0
        self.maximum_level = 0
        self.continious_percentage = 0.0
        pass

    def do_submission(self):
        pass

    def print_queue(self):
        pass

    def find_submission(self):
        running_level = self.running_level()
        _logger.info("Find queued to submit {resource}: {maximum}/{minimum}/{percent} - running/queued "
                     "- {running}/{queued}".format(resource=self.resource_name,maximum=self.maximum_level,minimum=self.minimum_level,
                                                   percent=self.continious_percentage,running=running_level,queued=self.total_queued))
        if (float(running_level) / float(self.maximum_level)) < (self.continious_percentage / 100.0):
            self.priorities_queue()
            to_submit_list = []
            to_submit_value = 0
            required_submission = self.maximum_level - running_level
            if required_submission + self.minimum_level > self.total_queued:
                to_submit_list = self.queue
                to_submit_value = self.total_queued
            else:
                i = 0
                while (to_submit_value < required_submission):
                    if i > len(self.queue):
                        break
                    to_submit_list.append(self.queue[i])
                    to_submit_value += self.queue[i]['value']
                    i += 1
                if (i == (len(self.queue) - 1)) and (self.queue[-1]['value'] < self.minimum_level):
                    to_submit_list.append(self.queue[-1])
                    to_submit_value += self.queue[-1]['value']
            if ((running_level+to_submit_value) < self.minimum_level) and self.to_wait(to_submit_list):
                return []
            return to_submit_list

    def to_wait(self, submission_list):
        return False


class TestResource(ResourceQueue):

    def __init__(self, resource_name, test_queue):
        super().__init__(resource_name)
        self.__get_test_queue(test_queue)
        self.running_queue = [x for x in test_queue if x['status']=='running']
        self.get_limits()

    def __get_test_queue(self,test_queue):
        self.total_queued = 0
        for x in test_queue:
            if x['status'] == 'queued':
                self.total_queued += x['total']
                x['value'] =  x['total']
                self.queue.append(x)


    def running_level(self):
        running_level  = 0
        for x in self.running_queue :
            running_level += x['total'] - x['running']
        return running_level

    def priorities_queue(self):
        self.queue.sort(key=lambda x: x['value'])


    def get_limits(self):
        self.minimum_level = 5
        self.maximum_level = 100
        self.continious_percentage = 40

    def __submit(self, submission_list):
        ids = [x['id'] for x in submission_list]
        for x in self.queue:
            if x['id'] in ids:
                x['status'] = 'running'
                print(x)


    def do_submission(self):
        self.__submit(self.find_submission())




class TapeResource(ResourceQueue):

    def __init__(self, resource_name, ddm, test=False):
        super().__init__(resource_name)
        self.ddm = ddm
        self.__get_tape_queue()
        self.get_limits()
        self.is_test = test


    def __get_tape_queue(self):
        queued_staging_request = DatasetStaging.objects.filter(source=self.resource_name,status='queued').values()
        self.total_queued = 0
        for x in queued_staging_request:
            dataset_stagings = ActionStaging.objects.filter(dataset_stage=x['id'])
            priority = 0
            for dataset_staging in dataset_stagings:
                if dataset_staging.step_action.status == 'active':
                    task = ProductionTask.objects.get(id=dataset_staging.task)
                    priority = task.current_priority
                    if not priority:
                        priority = task.priority
            x['value'] = x['total_files']
            x['priority'] = priority
            self.total_queued += x['value']
            self.queue.append(x)

    def running_level(self):
        staing_requests =  DatasetStaging.objects.filter(source=self.resource_name,status__in=['staging']).values()
        running_level  = 0
        for x in staing_requests:
            running_level += x['total_files'] - x['staged_files']
        return running_level


    def get_limits(self):
        limits_config = ActionDefault.objects.get(type='Tape',name=self.resource_name)
        self.minimum_level = limits_config.get_config('minimum_level')
        self.maximum_level = limits_config.get_config('maximum_level')
        self.continious_percentage =limits_config.get_config('continious_percentage')


    def priorities_queue(self):
        self.queue.sort(key=lambda x:-x['priority'])

    def __submit(self, submission_list):
        #print(submission_list)
        if submission_list:
            total = sum([x['value'] for x in submission_list ])
            _logger.info("Submit rules for {resource}: {to_submit}, {total}".format(resource=self.resource_name,to_submit=len(submission_list),total=total))
            for x in submission_list:
                dataset_stagings = ActionStaging.objects.filter(dataset_stage=x['id'])
                rule = None
                lifetime = None
                source_replica = None
                for dataset_staging in dataset_stagings:
                    if dataset_staging.step_action.status == 'active':
                        rule = dataset_staging.step_action.get_config('rule')
                        lifetime = dataset_staging.step_action.get_config('lifetime')
                        source_replica =  dataset_staging.step_action.get_config('source_replica')
                if rule and lifetime:

                    #print(x['dataset'], rule)
                    try:
                        existed_rule = self.ddm.dataset_active_rule_by_rse(x['dataset'], rule)
                        if not self.is_test:
                            if not existed_rule:
                                if not source_replica:
                                    _logger.info("Submit rule for {resource}: {dataset} {rule}".format(resource=self.resource_name,
                                                                                                       dataset=x['dataset'],
                                                                                                       rule=rule))
                                    self.ddm.add_replication_rule(x['dataset'], rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                                            activity='Staging', notify='P')
                                else:
                                    _logger.info("Submit rule for {resource}: {dataset} {rule} {source}".format(resource=self.resource_name,
                                                                                                       dataset=x['dataset'],
                                                                                                       rule=rule, source=source_replica))
                                    self.ddm.add_replication_rule(x['dataset'], rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                                            activity='Staging', notify='P', source_replica_expression=source_replica)
                            staging =  DatasetStaging.objects.get(id=x['id'])
                            staging.status = 'staging'
                            staging.save()
                        else:
                            print(x['dataset'], rule, source_replica)
                    except Exception as e:
                        _logger.error("Problem during submission {resource}: {error}".format(resource=self.resource_name,
                                                                   error=str(e)))
                else:
                    _logger.error("Problem during submission for dataset {dataset_staging}".format(dataset_staging=x['id']))


    def to_wait(self, submission_list):
        oldest = min(submission_list,key=lambda x:x['start_time'])['start_time']
        return (timezone.now() - oldest) < timedelta(days=1)

    def do_submission(self):
        self.__submit(self.find_submission())


    def print_queue(self):
        self.priorities_queue()
        for x in self.queue:
            print(x)

class TapeResourceProcessed(TapeResource):
    def __init__(self, resource_name, ddm, running_level_processed, test=False):
        super().__init__(resource_name, ddm, test)
        self.running_level_processed = running_level_processed

    def running_level(self):
        return self.running_level_processed

class TestTapeResource(TapeResource):
    def __init__(self, resource_name, ddm, limits):
        self.limits = limits
        super().__init__(resource_name, ddm)


    def get_limits(self):
        limits_config = ActionDefault.objects.get(type='Tape',name=self.resource_name)
        self.minimum_level = limits_config.get_config('minimum_level')
        self.maximum_level = limits_config.get_config('maximum_level')
        self.continious_percentage =limits_config.get_config('continious_percentage')
        if self.limits.get('minimum_level'):
            self.minimum_level = self.limits['minimum_level']
        if self.limits.get('maximum_level'):
            self.maximum_level = self.limits['maximum_level']
        if self.limits.get('continious_percentage'):
            self.continious_percentage = self.limits['continious_percentage']


def start_stagind_task(task):
    #Send resume command
    if(task.status in ['staging','waiting','paused']):
        _logger.info('Resume task after pre stage %s ' % (str(task.id)))
        _do_deft_action('mborodin',int(task.id),'resume_task')
    pass

def send_use_archive_task(task):
    #Send resume command
    _do_deft_action('mborodin',int(task.id),'change_split_rule','UZ','1')


def perfom_dataset_stage(input_dataset, ddm, rule, lifetime, replicas=None):
    try:
        _logger.info('%s should be pre staged rule %s  '%(input_dataset,rule))
        rse = ddm.dataset_active_rule_by_rse(input_dataset, rule)
        if rse:
            return rse['id']
        if not replicas:
            ddm.add_replication_rule(input_dataset, rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                    activity='Staging', notify='P')
        else:
            ddm.add_replication_rule(input_dataset, rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                    activity='Staging', notify='P',source_replica_expression=replicas)

        return True
    except Exception as e:
        _logger.error("Can't create rule %s" % str(e))
        return False

def create_staging_action(input_dataset,task,ddm,rule,config,replicas=None,source=None,lifetime=None):
    step = task.step
    waiting_parameters_from_step = None
    if step.get_task_config('PDAParams'):
        try:
            waiting_parameters_from_step = _parse_action_options(step.get_task_config('PDAParams'))
            if waiting_parameters_from_step.get('level', None):
                level = int(waiting_parameters_from_step.get('level'))
                if level > 100:
                    level = 100
                elif level < -1:
                    level = 0
        except Exception as e:
            _logger.error(" %s" % str(e))
    if not lifetime:
        lifetime = config['lifetime']
    if not DatasetStaging.objects.filter(dataset=input_dataset,status__in=DatasetStaging.ACTIVE_STATUS).exists():
        if DatasetStaging.objects.filter(dataset=input_dataset,status='done').exists():
            dataset_staging = DatasetStaging.objects.filter(dataset=input_dataset,status='done').first()
            dataset_staging.update_time = None
            dataset_staging.rse = None
            dataset_staging.source = None
            dataset_staging.end_time = None
        else:
            dataset_staging = DatasetStaging()
            dataset_staging.dataset = input_dataset
        dataset_staging.total_files = ddm.dataset_metadata(input_dataset)['length']
        dataset_staging.staged_files = 0
        dataset_staging.status = 'queued'
        if source:
            dataset_staging.source = source
        dataset_staging.start_time = timezone.now()
        dataset_staging.save()


    else:
        dataset_staging = DatasetStaging.objects.filter(dataset=input_dataset,
                                                        status__in=DatasetStaging.ACTIVE_STATUS).last()
    if dataset_staging.status == 'queued' and waiting_parameters_from_step and waiting_parameters_from_step.get('nowait', False):
        perfom_dataset_stage(input_dataset, ddm, rule, lifetime, replicas)
        dataset_staging.status = 'staging'
        dataset_staging.start_time = timezone.now()
        dataset_staging.save()
    # Create new action
    if StepAction.objects.filter(step=task.step_id,action = 6,status__in=['active','executing']).exists():
        action_step = StepAction.objects.filter(step=task.step_id,action = 6,status__in=['active','executing']).last()
    else:
        action_step = StepAction()
        action_step.action = 6
        #todo add config
        action_step.create_time = timezone.now()
        action_step.set_config({'delay':config['delay']})
        action_step.step = step.id
        level = None
        if not level:
            level = config['level']
        if level == -1:
            if dataset_staging.total_files:
                if dataset_staging.total_files < 1000:
                    level = 90
                else:
                    level = 95
            else:
                level = 95
        action_step.set_config({'level':level})
        action_step.set_config({'lifetime':lifetime})
        action_step.set_config({'rule': rule})
        if replicas:
            action_step.set_config({'source_replica': replicas})
        if source:
            action_step.set_config({'tape': source})
        action_step.execution_time = timezone.now() + timedelta(hours=config['delay'])
        action_step.attempt = 0
        action_step.status = 'active'

        action_step.request = task.request
        action_step.save()
    action_dataset = ActionStaging()
    action_dataset.task = task.id
    action_dataset.dataset_stage = dataset_staging
    action_dataset.step_action = action_step
    action_dataset.save()

def submit_all_tapes():
    ddm = DDM()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active'):
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                resource_tape = TapeResource(tape.name,ddm)
                resource_tape.do_submission()

def submit_all_tapes_processed():
    ddm = DDM()
    active_staged = find_active_staged()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active'):
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                active_for_tape = [active_staged[x] for x in active_staged.keys() if
                                   active_staged[x]['tape'] == tape.name]
                total_submitted = sum([x['total']- x['value'] for x in active_for_tape])
                resource_tape = TapeResourceProcessed(tape.name,ddm,total_submitted)
                resource_tape.do_submission()


def test_tape_processed(tape_name, test):
    ddm = DDM()
    active_staged = find_active_staged()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active') and tape.name==tape_name:
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                active_for_tape = [active_staged[x] for x in active_staged.keys() if
                                   active_staged[x]['tape'] == tape.name]
                total_submitted = sum([x['total']- x['value'] for x in active_for_tape])
                resource_tape = TapeResourceProcessed(tape.name,ddm,total_submitted,test)
                resource_tape.print_queue()

def create_prestage(task,ddm,rule, input_dataset,config, special=False):
    #check that's only Tape replica
    replicas = ddm.full_replicas_per_type(input_dataset)
    if (len(replicas['data']) > 0):
        start_stagind_task(task)
        return True
    else:
        #No data replica - create a rule
        source_replicas = None
        input = None
        if special:
            special_datasets = list(InputRequestList.objects.filter(~Q(is_hide=True),request=29036).values_list('dataset',flat=True))
            if ddm.rucio_convention(input_dataset)[1] in special_datasets:
                rule, source_replicas, input = 'CERN-PROD_DATADISK', 'CERN-PROD_RAW',  'CERN-PROD_RAW'
            else:
                rule, source_replicas, input = ddm.get_replica_pre_stage_rule(input_dataset)
            if input == 'INFN-T1_DATATAPE':
                source_replicas = 'INFN-T1_DATATAPE'
        else:
            if replicas['tape']:
                input = [x['rse'] for x in replicas['tape']]
                if len(input) == 1:
                    input = input[0]
        create_staging_action(input_dataset,task,ddm,rule,config,source_replicas,input)


def _parse_action_options(option_string):
    project_mode_dict = dict()
    for option in option_string.replace(' ', '').split(";"):
        if not option:
            continue
        if not '=' in option:
            raise Exception('The  option \"{0}\" has invalid format. '.format(option) +
                            'Expected format is \"optionName=optionValue\"')
        project_mode_dict.update({option.split("=")[0].lower(): option.split("=")[1]})
    return project_mode_dict


def create_follow_prestage_action(task):
    config =  ActionDefault.objects.get(name='active_archive_staging').get_config()
    action_step = StepAction()
    action_step.action = 9
    action_step.create_time = timezone.now()
    action_step.set_config({'delay': config['delay']})
    action_step.set_config({'task': task.id})
    step = task.step
    action_step.step = step.id
    action_step.execution_time = timezone.now() + timedelta(hours=config['delay'])
    action_step.attempt = 0
    action_step.status = 'active'
    action_step.request = task.request
    action_step.save()

def check_tasks_for_prestage(action_step_id, ddm, rule, delay, max_waite_time, check_archive=False):
    action_step = StepAction.objects.get(id=action_step_id)
    action_step.attempt += 1
    step = StepExecution.objects.get(id=action_step.step)
    special = False
    idds = False
    if step.get_task_config('PDAParams'):
        try:
            waiting_parameters_from_step = _parse_action_options(step.get_task_config('PDAParams'))
            if waiting_parameters_from_step.get('special'):
                special = True
            if waiting_parameters_from_step.get('idds'):
                idds = True
        except Exception as e:
            _logger.error(" %s" % str(e))
    production_request = step.request
    tasks = ProductionTask.objects.filter(step=step)
    current_time = timezone.now()
    #todo add timeout
    if (not tasks) and (production_request.cstatus != 'approved'):
        action_step.status = 'failed'
        action_step.message = 'No tasks were defined'
        action_step.done_time = current_time
        action_step.save()
        step.remove_project_mode('toStaging')
        step.save()
        return
    finish_action = True
    fail_action = False
    for task in tasks:
        if (task.status in ['staging','waiting']) and (not ActionStaging.objects.filter(task=task.id).exists()):
            try:
                if check_archive:
                    config = ActionDefault.objects.get(name='active_archive_staging').get_config()
                else:
                    ActionDefault.objects.get(name='active_staging').get_config()
                if idds:
                    config['level'] = 1
                if check_archive:
                    input_dataset = find_archive_dataset(task.input_dataset,ddm)
                    if input_dataset:
                        send_use_archive_task(task)
                        create_prestage(task, ddm, rule, input_dataset, config, special)
                        create_follow_prestage_action(task)
                    else:
                        fail_action = True
                else:
                    create_prestage(task,ddm,rule,task.input_dataset,config,special)
            except Exception as e:
                _logger.error("Check replicas problem %s" % str(e))
                finish_action = False
    if finish_action and (production_request.cstatus != 'approved'):
        action_step.status = 'done'
        action_step.message = 'All task checked'
        action_step.done_time = current_time
        step.remove_project_mode('toStaging')
        step.save()
    else:
        action_step.execution_time = current_time + timedelta(hours=delay)
        action_step.status = 'active'<commit_msg>Fix rule creation<commit_after>import json

from django.db.models import Q

from atlas.prodtask.models import ActionStaging, ActionDefault, DatasetStaging, StepAction, TTask
from datetime import timedelta

from atlas.prodtask.ddm_api import DDM
from atlas.prodtask.models import  StepExecution, InputRequestList, ProductionTask, ProductionDataset

from decimal import Decimal
from datetime import datetime
from rest_framework.decorators import api_view
from rest_framework.response import Response
import logging
# import os
from django.utils import timezone

from django.contrib.auth.decorators import login_required

from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render

from atlas.prodtask.task_actions import _do_deft_action

_logger = logging.getLogger('prodtaskwebui')


def test_step_action(step_action_ids):
    pass

def check_staging_task(step_action_ids, check_archive=False):
    ddm = DDM()
    config = ActionDefault.objects.get(name='stage_creation').get_config()
    delay = config['delay']
    max_waite_time = config['max_waite_time']
    rule = config['rule']
    for waiting_step in step_action_ids:
        try:
            check_tasks_for_prestage(waiting_step, ddm, rule, delay, max_waite_time, check_archive)
        except Exception as e:
            _logger.error("Check replicas problem %s" % str(e))
            waiting_step = StepAction.objects.get(id=waiting_step)
            waiting_step.status = 'active'
            waiting_step.save()
    pass


class ResourceQueue(object):
    def __init__(self, resource_name):
        self.queue = []
        self.total_queued = 0
        self.resource_name = resource_name
        pass

    def priorities_queue(self):

        return

    def running_level(self):
        return 0

    def get_limits(self):
        self.minimum_level = 0
        self.maximum_level = 0
        self.continious_percentage = 0.0
        pass

    def do_submission(self):
        pass

    def print_queue(self):
        pass

    def find_submission(self):
        running_level = self.running_level()
        _logger.info("Find queued to submit {resource}: {maximum}/{minimum}/{percent} - running/queued "
                     "- {running}/{queued}".format(resource=self.resource_name,maximum=self.maximum_level,minimum=self.minimum_level,
                                                   percent=self.continious_percentage,running=running_level,queued=self.total_queued))
        if (float(running_level) / float(self.maximum_level)) < (self.continious_percentage / 100.0):
            self.priorities_queue()
            to_submit_list = []
            to_submit_value = 0
            required_submission = self.maximum_level - running_level
            if required_submission + self.minimum_level > self.total_queued:
                to_submit_list = self.queue
                to_submit_value = self.total_queued
            else:
                i = 0
                while (to_submit_value < required_submission):
                    if i > len(self.queue):
                        break
                    to_submit_list.append(self.queue[i])
                    to_submit_value += self.queue[i]['value']
                    i += 1
                if (i == (len(self.queue) - 1)) and (self.queue[-1]['value'] < self.minimum_level):
                    to_submit_list.append(self.queue[-1])
                    to_submit_value += self.queue[-1]['value']
            if ((running_level+to_submit_value) < self.minimum_level) and self.to_wait(to_submit_list):
                return []
            return to_submit_list

    def to_wait(self, submission_list):
        return False


class TestResource(ResourceQueue):

    def __init__(self, resource_name, test_queue):
        super().__init__(resource_name)
        self.__get_test_queue(test_queue)
        self.running_queue = [x for x in test_queue if x['status']=='running']
        self.get_limits()

    def __get_test_queue(self,test_queue):
        self.total_queued = 0
        for x in test_queue:
            if x['status'] == 'queued':
                self.total_queued += x['total']
                x['value'] =  x['total']
                self.queue.append(x)


    def running_level(self):
        running_level  = 0
        for x in self.running_queue :
            running_level += x['total'] - x['running']
        return running_level

    def priorities_queue(self):
        self.queue.sort(key=lambda x: x['value'])


    def get_limits(self):
        self.minimum_level = 5
        self.maximum_level = 100
        self.continious_percentage = 40

    def __submit(self, submission_list):
        ids = [x['id'] for x in submission_list]
        for x in self.queue:
            if x['id'] in ids:
                x['status'] = 'running'
                print(x)


    def do_submission(self):
        self.__submit(self.find_submission())




class TapeResource(ResourceQueue):

    def __init__(self, resource_name, ddm, test=False):
        super().__init__(resource_name)
        self.ddm = ddm
        self.__get_tape_queue()
        self.get_limits()
        self.is_test = test


    def __get_tape_queue(self):
        queued_staging_request = DatasetStaging.objects.filter(source=self.resource_name,status='queued').values()
        self.total_queued = 0
        for x in queued_staging_request:
            dataset_stagings = ActionStaging.objects.filter(dataset_stage=x['id'])
            priority = 0
            for dataset_staging in dataset_stagings:
                if dataset_staging.step_action.status == 'active':
                    task = ProductionTask.objects.get(id=dataset_staging.task)
                    priority = task.current_priority
                    if not priority:
                        priority = task.priority
            x['value'] = x['total_files']
            x['priority'] = priority
            self.total_queued += x['value']
            self.queue.append(x)

    def running_level(self):
        staing_requests =  DatasetStaging.objects.filter(source=self.resource_name,status__in=['staging']).values()
        running_level  = 0
        for x in staing_requests:
            running_level += x['total_files'] - x['staged_files']
        return running_level


    def get_limits(self):
        limits_config = ActionDefault.objects.get(type='Tape',name=self.resource_name)
        self.minimum_level = limits_config.get_config('minimum_level')
        self.maximum_level = limits_config.get_config('maximum_level')
        self.continious_percentage =limits_config.get_config('continious_percentage')


    def priorities_queue(self):
        self.queue.sort(key=lambda x:-x['priority'])

    def __submit(self, submission_list):
        #print(submission_list)
        if submission_list:
            total = sum([x['value'] for x in submission_list ])
            _logger.info("Submit rules for {resource}: {to_submit}, {total}".format(resource=self.resource_name,to_submit=len(submission_list),total=total))
            for x in submission_list:
                dataset_stagings = ActionStaging.objects.filter(dataset_stage=x['id'])
                rule = None
                lifetime = None
                source_replica = None
                for dataset_staging in dataset_stagings:
                    if dataset_staging.step_action.status == 'active':
                        rule = dataset_staging.step_action.get_config('rule')
                        lifetime = dataset_staging.step_action.get_config('lifetime')
                        source_replica =  dataset_staging.step_action.get_config('source_replica')
                if rule and lifetime:

                    #print(x['dataset'], rule)
                    try:
                        existed_rule = self.ddm.dataset_active_rule_by_rse(x['dataset'], rule)
                        if not self.is_test:
                            if not existed_rule:
                                if not source_replica:
                                    _logger.info("Submit rule for {resource}: {dataset} {rule}".format(resource=self.resource_name,
                                                                                                       dataset=x['dataset'],
                                                                                                       rule=rule))
                                    self.ddm.add_replication_rule(x['dataset'], rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                                            activity='Staging', notify='P')
                                else:
                                    _logger.info("Submit rule for {resource}: {dataset} {rule} {source}".format(resource=self.resource_name,
                                                                                                       dataset=x['dataset'],
                                                                                                       rule=rule, source=source_replica))
                                    self.ddm.add_replication_rule(x['dataset'], rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                                            activity='Staging', notify='P', source_replica_expression=source_replica)
                            staging =  DatasetStaging.objects.get(id=x['id'])
                            staging.status = 'staging'
                            staging.save()
                        else:
                            print(x['dataset'], rule, source_replica)
                    except Exception as e:
                        _logger.error("Problem during submission {resource}: {error}".format(resource=self.resource_name,
                                                                   error=str(e)))
                else:
                    _logger.error("Problem during submission for dataset {dataset_staging}".format(dataset_staging=x['id']))


    def to_wait(self, submission_list):
        oldest = min(submission_list,key=lambda x:x['start_time'])['start_time']
        return (timezone.now() - oldest) < timedelta(days=1)

    def do_submission(self):
        self.__submit(self.find_submission())


    def print_queue(self):
        self.priorities_queue()
        for x in self.queue:
            print(x)

class TapeResourceProcessed(TapeResource):
    def __init__(self, resource_name, ddm, running_level_processed, test=False):
        super().__init__(resource_name, ddm, test)
        self.running_level_processed = running_level_processed

    def running_level(self):
        return self.running_level_processed

class TestTapeResource(TapeResource):
    def __init__(self, resource_name, ddm, limits):
        self.limits = limits
        super().__init__(resource_name, ddm)


    def get_limits(self):
        limits_config = ActionDefault.objects.get(type='Tape',name=self.resource_name)
        self.minimum_level = limits_config.get_config('minimum_level')
        self.maximum_level = limits_config.get_config('maximum_level')
        self.continious_percentage =limits_config.get_config('continious_percentage')
        if self.limits.get('minimum_level'):
            self.minimum_level = self.limits['minimum_level']
        if self.limits.get('maximum_level'):
            self.maximum_level = self.limits['maximum_level']
        if self.limits.get('continious_percentage'):
            self.continious_percentage = self.limits['continious_percentage']


def start_stagind_task(task):
    #Send resume command
    if(task.status in ['staging','waiting','paused']):
        _logger.info('Resume task after pre stage %s ' % (str(task.id)))
        _do_deft_action('mborodin',int(task.id),'resume_task')
    pass

def send_use_archive_task(task):
    #Send resume command
    _do_deft_action('mborodin',int(task.id),'change_split_rule','UZ','1')


def perfom_dataset_stage(input_dataset, ddm, rule, lifetime, replicas=None):
    try:
        _logger.info('%s should be pre staged rule %s  '%(input_dataset,rule))
        rse = ddm.dataset_active_rule_by_rse(input_dataset, rule)
        if rse:
            return rse['id']
        if not replicas:
            ddm.add_replication_rule(input_dataset, rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                    activity='Staging', notify='P')
        else:
            ddm.add_replication_rule(input_dataset, rule, copies=1, lifetime=lifetime*86400, weight='freespace',
                                    activity='Staging', notify='P',source_replica_expression=replicas)

        return True
    except Exception as e:
        _logger.error("Can't create rule %s" % str(e))
        return False

def create_staging_action(input_dataset,task,ddm,rule,config,replicas=None,source=None,lifetime=None):
    step = task.step
    waiting_parameters_from_step = None
    if step.get_task_config('PDAParams'):
        try:
            waiting_parameters_from_step = _parse_action_options(step.get_task_config('PDAParams'))
            if waiting_parameters_from_step.get('level', None):
                level = int(waiting_parameters_from_step.get('level'))
                if level > 100:
                    level = 100
                elif level < -1:
                    level = 0
        except Exception as e:
            _logger.error(" %s" % str(e))
    if not lifetime:
        lifetime = config['lifetime']
    if not DatasetStaging.objects.filter(dataset=input_dataset,status__in=DatasetStaging.ACTIVE_STATUS).exists():
        if DatasetStaging.objects.filter(dataset=input_dataset,status='done').exists():
            dataset_staging = DatasetStaging.objects.filter(dataset=input_dataset,status='done').first()
            dataset_staging.update_time = None
            dataset_staging.rse = None
            dataset_staging.source = None
            dataset_staging.end_time = None
        else:
            dataset_staging = DatasetStaging()
            dataset_staging.dataset = input_dataset
        dataset_staging.total_files = ddm.dataset_metadata(input_dataset)['length']
        dataset_staging.staged_files = 0
        dataset_staging.status = 'queued'
        if source:
            dataset_staging.source = source
        dataset_staging.start_time = timezone.now()
        dataset_staging.save()


    else:
        dataset_staging = DatasetStaging.objects.filter(dataset=input_dataset,
                                                        status__in=DatasetStaging.ACTIVE_STATUS).last()
    if dataset_staging.status == 'queued' and waiting_parameters_from_step and waiting_parameters_from_step.get('nowait', False):
        perfom_dataset_stage(input_dataset, ddm, rule, lifetime, replicas)
        dataset_staging.status = 'staging'
        dataset_staging.start_time = timezone.now()
        dataset_staging.save()
    # Create new action
    if StepAction.objects.filter(step=task.step_id,action = 6,status__in=['active','executing']).exists():
        action_step = StepAction.objects.filter(step=task.step_id,action = 6,status__in=['active','executing']).last()
    else:
        action_step = StepAction()
        action_step.action = 6
        #todo add config
        action_step.create_time = timezone.now()
        action_step.set_config({'delay':config['delay']})
        action_step.step = step.id
        level = None
        if not level:
            level = config['level']
        if level == -1:
            if dataset_staging.total_files:
                if dataset_staging.total_files < 1000:
                    level = 90
                else:
                    level = 95
            else:
                level = 95
        action_step.set_config({'level':level})
        action_step.set_config({'lifetime':lifetime})
        action_step.set_config({'rule': rule})
        if replicas:
            action_step.set_config({'source_replica': replicas})
        if source:
            action_step.set_config({'tape': source})
        action_step.execution_time = timezone.now() + timedelta(hours=config['delay'])
        action_step.attempt = 0
        action_step.status = 'active'

        action_step.request = task.request
        action_step.save()
    action_dataset = ActionStaging()
    action_dataset.task = task.id
    action_dataset.dataset_stage = dataset_staging
    action_dataset.step_action = action_step
    action_dataset.save()

def submit_all_tapes():
    ddm = DDM()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active'):
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                resource_tape = TapeResource(tape.name,ddm)
                resource_tape.do_submission()

def submit_all_tapes_processed():
    ddm = DDM()
    active_staged = find_active_staged()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active'):
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                active_for_tape = [active_staged[x] for x in active_staged.keys() if
                                   active_staged[x]['tape'] == tape.name]
                total_submitted = sum([x['total']- x['value'] for x in active_for_tape])
                resource_tape = TapeResourceProcessed(tape.name,ddm,total_submitted)
                resource_tape.do_submission()


def test_tape_processed(tape_name, test):
    ddm = DDM()
    active_staged = find_active_staged()
    for tape in ActionDefault.objects.filter(type='Tape'):
        if tape.get_config('active') and tape.name==tape_name:
            if DatasetStaging.objects.filter(source=tape.name,status='queued').exists():
                active_for_tape = [active_staged[x] for x in active_staged.keys() if
                                   active_staged[x]['tape'] == tape.name]
                total_submitted = sum([x['total']- x['value'] for x in active_for_tape])
                resource_tape = TapeResourceProcessed(tape.name,ddm,total_submitted,test)
                resource_tape.print_queue()

def create_prestage(task,ddm,rule, input_dataset,config, special=False):
    #check that's only Tape replica
    replicas = ddm.full_replicas_per_type(input_dataset)
    if (len(replicas['data']) > 0):
        start_stagind_task(task)
        return True
    else:
        #No data replica - create a rule
        source_replicas = None
        input = None
        if special:
            special_datasets = list(InputRequestList.objects.filter(~Q(is_hide=True),request=29036).values_list('dataset',flat=True))
            if ddm.rucio_convention(input_dataset)[1] in special_datasets:
                rule, source_replicas, input = 'CERN-PROD_DATADISK', 'CERN-PROD_RAW',  'CERN-PROD_RAW'
            else:
                rule, source_replicas, input = ddm.get_replica_pre_stage_rule(input_dataset)
            if input == 'INFN-T1_DATATAPE':
                source_replicas = 'INFN-T1_DATATAPE'
        else:
            if replicas['tape']:
                input = [x['rse'] for x in replicas['tape']]
                if len(input) == 1:
                    input = input[0]
        create_staging_action(input_dataset,task,ddm,rule,config,source_replicas,input)


def _parse_action_options(option_string):
    project_mode_dict = dict()
    for option in option_string.replace(' ', '').split(";"):
        if not option:
            continue
        if not '=' in option:
            raise Exception('The  option \"{0}\" has invalid format. '.format(option) +
                            'Expected format is \"optionName=optionValue\"')
        project_mode_dict.update({option.split("=")[0].lower(): option.split("=")[1]})
    return project_mode_dict


def create_follow_prestage_action(task):
    config =  ActionDefault.objects.get(name='active_archive_staging').get_config()
    action_step = StepAction()
    action_step.action = 9
    action_step.create_time = timezone.now()
    action_step.set_config({'delay': config['delay']})
    action_step.set_config({'task': task.id})
    step = task.step
    action_step.step = step.id
    action_step.execution_time = timezone.now() + timedelta(hours=config['delay'])
    action_step.attempt = 0
    action_step.status = 'active'
    action_step.request = task.request
    action_step.save()

def check_tasks_for_prestage(action_step_id, ddm, rule, delay, max_waite_time, check_archive=False):
    action_step = StepAction.objects.get(id=action_step_id)
    action_step.attempt += 1
    step = StepExecution.objects.get(id=action_step.step)
    special = False
    idds = False
    if step.get_task_config('PDAParams'):
        try:
            waiting_parameters_from_step = _parse_action_options(step.get_task_config('PDAParams'))
            if waiting_parameters_from_step.get('special'):
                special = True
            if waiting_parameters_from_step.get('idds'):
                idds = True
        except Exception as e:
            _logger.error(" %s" % str(e))
    production_request = step.request
    tasks = ProductionTask.objects.filter(step=step)
    current_time = timezone.now()
    #todo add timeout
    if (not tasks) and (production_request.cstatus != 'approved'):
        action_step.status = 'failed'
        action_step.message = 'No tasks were defined'
        action_step.done_time = current_time
        action_step.save()
        step.remove_project_mode('toStaging')
        step.save()
        return
    finish_action = True
    fail_action = False
    for task in tasks:
        if (task.status in ['staging','waiting']) and (not ActionStaging.objects.filter(task=task.id).exists()):
            try:
                if check_archive:
                    config = ActionDefault.objects.get(name='active_archive_staging').get_config()
                else:
                    config = ActionDefault.objects.get(name='active_staging').get_config()
                if idds:
                    config['level'] = 1
                if check_archive:
                    input_dataset = find_archive_dataset(task.input_dataset,ddm)
                    if input_dataset:
                        send_use_archive_task(task)
                        create_prestage(task, ddm, rule, input_dataset, config, special)
                        create_follow_prestage_action(task)
                    else:
                        fail_action = True
                else:
                    create_prestage(task,ddm,rule,task.input_dataset,config,special)
            except Exception as e:
                _logger.error("Check replicas problem %s" % str(e))
                finish_action = False
    if finish_action and (production_request.cstatus != 'approved'):
        action_step.status = 'done'
        action_step.message = 'All task checked'
        action_step.done_time = current_time
        step.remove_project_mode('toStaging')
        step.save()
    else:
        action_step.execution_time = current_time + timedelta(hours=delay)
        action_step.status = 'active'
{'fraction_non_alphanumeric': 0.09974690116165877, 'fraction_numerical': 0.004153416834317606, 'mean_word_length': 2.240658955485454, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 72, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476064', 'n_tokens_mistral': 13359, 'n_tokens_neox': 12858, 'n_words': 2346}