<commit_before>/*
 *  Copyright (C) 2010 reuillon
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.openmole.core.implementation.mole;

import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.collections15.BidiMap;
import org.apache.commons.collections15.MultiMap;
import org.apache.commons.collections15.bidimap.DualHashBidiMap;
import org.apache.commons.collections15.multimap.MultiHashMap;
import org.openmole.commons.aspect.eventdispatcher.BeforeObjectModified;

import org.openmole.commons.exception.InternalProcessingError;
import org.openmole.commons.exception.UserBadDataError;
import org.openmole.core.model.execution.IEnvironment;
import org.openmole.core.model.job.IMoleJob;
import org.openmole.core.model.capsule.IGenericTaskCapsule;
import org.openmole.core.implementation.internal.Activator;
import org.openmole.core.implementation.job.Context;
import org.openmole.core.implementation.job.Job;
import org.openmole.core.implementation.job.MoleJob;
import org.openmole.core.implementation.job.MoleJobId;
import org.openmole.core.model.execution.IMoleJobCategory;
import org.openmole.core.model.execution.IMoleJobGroupingStrategy;
import org.openmole.core.model.mole.ISubMoleExecution;
import org.openmole.core.model.job.IContext;
import org.openmole.core.model.job.IJob;
import org.openmole.core.model.job.IMoleJobId;
import org.openmole.core.model.job.ITicket;
import org.openmole.core.model.mole.IMoleExecution;
import org.openmole.core.model.mole.ILocalCommunication;
import org.openmole.commons.aspect.eventdispatcher.IObjectChangedSynchronousListener;
import org.openmole.commons.tools.service.Priority;
import org.openmole.core.implementation.execution.JobRegistry;
import org.openmole.core.implementation.execution.local.LocalExecutionEnvironment;
import org.openmole.core.implementation.job.SynchronizedContext;
import org.openmole.core.model.mole.IEnvironmentSelection;
import org.openmole.core.model.mole.IMole;
import org.openmole.core.model.mole.IMoleJobGrouping;
import scala.Tuple2;
import scala.Tuple3;

public class MoleExecution implements IMoleExecution {

    class MoleExecutionAdapterForMoleJobOutputTransitionPerformed implements IObjectChangedSynchronousListener<IMoleJob> {

        @Override
        public void objectChanged(IMoleJob job) throws InternalProcessingError, UserBadDataError {
            jobOutputTransitionsPerformed(job);
        }

    }

    class MoleExecutionAdapterForMoleJob implements IObjectChangedSynchronousListener<IMoleJob> {

        @Override
        public void objectChanged(IMoleJob job) throws InternalProcessingError, UserBadDataError {
            switch (job.getState()) {
                case FAILED:
                    jobFailed(job);
                    break;
            }
        }
    }

    class MoleExecutionAdapterForSubMoleExecution implements IObjectChangedSynchronousListener<ISubMoleExecution> {

        @Override
        public void objectChanged(ISubMoleExecution obj) throws InternalProcessingError, UserBadDataError {
            submitGroups(obj);
        }
    }

    private static final Logger LOGGER = Logger.getLogger(MoleExecution.class.getName());
    
    
    final IMole mole;
    final IMoleJobGrouping moleJobGrouping;

    final BlockingQueue<Tuple2<IJob, IEnvironment>> jobs = new LinkedBlockingQueue<Tuple2<IJob, IEnvironment>>();
    final Map<IMoleJob, Tuple2<ISubMoleExecution, ITicket>> inProgress = Collections.synchronizedMap(new TreeMap<IMoleJob, Tuple2<ISubMoleExecution, ITicket>>()); 
    
    final AtomicLong ticketNumber = new AtomicLong();
    final AtomicLong currentJobId = new AtomicLong();

    final ILocalCommunication localCommunication;
    final IEnvironmentSelection environmentSelectionStrategy;
    final IEnvironment defaultEnvironment;
    
    final BidiMap<Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>, Job> categorizer = new DualHashBidiMap<Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>, Job>();
    final MultiMap<ISubMoleExecution, Job> jobsGrouping = new MultiHashMap<ISubMoleExecution, Job>();

    final MoleExecutionAdapterForMoleJob moleExecutionAdapterForMoleJob = new MoleExecutionAdapterForMoleJob();
    final MoleExecutionAdapterForSubMoleExecution moleExecutionAdapterForSubMoleExecution = new MoleExecutionAdapterForSubMoleExecution();
    final MoleExecutionAdapterForMoleJobOutputTransitionPerformed moleJobOutputTransitionPerformed = new MoleExecutionAdapterForMoleJobOutputTransitionPerformed();

    transient Thread submiter;

    public MoleExecution(IMole mole) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), FixedEnvironmentSelection.EMPTY_SELECTION, MoleJobGrouping.EMPTY_GROUPING);
    }
    
    public MoleExecution(IMole mole, IEnvironmentSelection environmentSelectionStrategy) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), environmentSelectionStrategy, MoleJobGrouping.EMPTY_GROUPING);
    }

    public MoleExecution(IMole mole, IEnvironmentSelection environmentSelectionStrategy, IMoleJobGrouping moleJobGrouping) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), environmentSelectionStrategy, moleJobGrouping);
    }
    
    public MoleExecution(IMole mole, IContext global, IContext context) throws InternalProcessingError, UserBadDataError {
        this(mole, global, context, FixedEnvironmentSelection.EMPTY_SELECTION, MoleJobGrouping.EMPTY_GROUPING);
    }
    
    public MoleExecution(IMole mole, IContext global, IContext context, IEnvironmentSelection environmentSelectionStrategy, IMoleJobGrouping moleJobGrouping) throws InternalProcessingError, UserBadDataError {
        this.mole = mole;
        this.moleJobGrouping = moleJobGrouping;
        this.localCommunication = new LocalCommunication();
        this.environmentSelectionStrategy = environmentSelectionStrategy;
        this.defaultEnvironment = LocalExecutionEnvironment.getInstance();

        ITicket ticket = nextTicket(createRootTicket());

        IGenericTaskCapsule root = mole.getRoot();
        if (root == null) {
            throw new UserBadDataError("First task of workflow hasn't been set.");
        }

        submit(root, global, context, ticket, new SubMoleExecution());
    }

    @Override
    public synchronized void submit(IGenericTaskCapsule capsule, IContext global, IContext context, ITicket ticket, ISubMoleExecution subMole) throws InternalProcessingError, UserBadDataError {
        IMoleJob job = capsule.toJob(global, context, nextJobId());
        submit(job, capsule, subMole, ticket);
    }
   
    public synchronized void submit(IMoleJob moleJob, IGenericTaskCapsule capsule, ISubMoleExecution subMole, ITicket ticket) throws InternalProcessingError, UserBadDataError {
        Activator.getEventDispatcher().objectChanged(this, oneJobSubmitted, new Object[]{moleJob});

        ExecutionInfoRegistry.GetInstance().register(moleJob, this);
        Activator.getEventDispatcher().registerListener(moleJob, Priority.HIGH.getValue(), moleExecutionAdapterForMoleJob, MoleJob.StateChanged);
        Activator.getEventDispatcher().registerListener(moleJob, Priority.NORMAL.getValue(), moleJobOutputTransitionPerformed, MoleJob.TransitionPerformed);

        inProgress.put(moleJob, new Tuple2<ISubMoleExecution, ITicket>(subMole, ticket));
        subMole.incNbJobInProgress();

        IMoleJobGroupingStrategy strategy = moleJobGrouping.getMoleJobGroupingStrategy(capsule);

        if (strategy != null) {
            IMoleJobCategory category = strategy.getCategory(moleJob.getContext());

            Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory> key = new Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>(subMole, capsule, category);
            Job job = categorizer.get(key);
            if (job == null) {
                job = new Job();

                categorizer.put(key, job);
                jobsGrouping.put(subMole, job);

                if (!Activator.getEventDispatcher().isRegistred(subMole, moleExecutionAdapterForSubMoleExecution, ISubMoleExecution.allJobsWaitingInGroup)) {
                    Activator.getEventDispatcher().registerListener(subMole, Priority.NORMAL.getValue(), moleExecutionAdapterForSubMoleExecution, ISubMoleExecution.allJobsWaitingInGroup);
                }
            }

            job.addMoleJob(moleJob);
            subMole.incNbJobWaitingInGroup();
        } else {
            Job job = new Job();
            job.addMoleJob(moleJob);
            submit(job, capsule);
        }

        LOGGER.log(Level.FINER, "A new job has been successfully submitted:{0}", moleJob);
    }

    private void submit(Job job, IGenericTaskCapsule capsule) {
        JobRegistry.getInstance().register(job, this);
        IEnvironment environment = environmentSelectionStrategy.selectEnvironment(capsule);
        environment = environment!=null?environment:defaultEnvironment;
        jobs.add(new Tuple2<IJob, IEnvironment>(job, environment));
    }

    synchronized void submitGroups(ISubMoleExecution subMoleExecution) {
        Iterable<Job> jobs = jobsGrouping.remove(subMoleExecution);

        LOGGER.finer("Submit a group");
        for (Job job : jobs) {
            Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory> info = categorizer.removeValue(job);
            subMoleExecution.decNbJobWaitingInGroup(job.size());
            submit(job, info._2());
        }
    }


    class Submiter implements Runnable {

        boolean stop = false;

        @Override
        public void run() {
            while (!stop) {
                Tuple2<IJob, IEnvironment> p;
                try {
                    p = jobs.take();
                } catch (InterruptedException e) {
                    LOGGER.log(Level.FINE, "Scheduling interrupted", e);
                    return;
                }
                LOGGER.log(Level.FINER, "New job taken:{0}", p._1());
                try {
                    p._2().submit(p._1());
                } catch (UserBadDataError e) {
                    LOGGER.log(Level.SEVERE, "Error durring scheduling", e);
                } catch (InternalProcessingError e) {
                    LOGGER.log(Level.SEVERE, "Error durring scheduling", e);

                } catch (Throwable t) {<commit_msg>Remove log.<commit_after>/*
 *  Copyright (C) 2010 reuillon
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.openmole.core.implementation.mole;

import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.collections15.BidiMap;
import org.apache.commons.collections15.MultiMap;
import org.apache.commons.collections15.bidimap.DualHashBidiMap;
import org.apache.commons.collections15.multimap.MultiHashMap;
import org.openmole.commons.aspect.eventdispatcher.BeforeObjectModified;

import org.openmole.commons.exception.InternalProcessingError;
import org.openmole.commons.exception.UserBadDataError;
import org.openmole.core.model.execution.IEnvironment;
import org.openmole.core.model.job.IMoleJob;
import org.openmole.core.model.capsule.IGenericTaskCapsule;
import org.openmole.core.implementation.internal.Activator;
import org.openmole.core.implementation.job.Context;
import org.openmole.core.implementation.job.Job;
import org.openmole.core.implementation.job.MoleJob;
import org.openmole.core.implementation.job.MoleJobId;
import org.openmole.core.model.execution.IMoleJobCategory;
import org.openmole.core.model.execution.IMoleJobGroupingStrategy;
import org.openmole.core.model.mole.ISubMoleExecution;
import org.openmole.core.model.job.IContext;
import org.openmole.core.model.job.IJob;
import org.openmole.core.model.job.IMoleJobId;
import org.openmole.core.model.job.ITicket;
import org.openmole.core.model.mole.IMoleExecution;
import org.openmole.core.model.mole.ILocalCommunication;
import org.openmole.commons.aspect.eventdispatcher.IObjectChangedSynchronousListener;
import org.openmole.commons.tools.service.Priority;
import org.openmole.core.implementation.execution.JobRegistry;
import org.openmole.core.implementation.execution.local.LocalExecutionEnvironment;
import org.openmole.core.implementation.job.SynchronizedContext;
import org.openmole.core.model.mole.IEnvironmentSelection;
import org.openmole.core.model.mole.IMole;
import org.openmole.core.model.mole.IMoleJobGrouping;
import scala.Tuple2;
import scala.Tuple3;

public class MoleExecution implements IMoleExecution {

    class MoleExecutionAdapterForMoleJobOutputTransitionPerformed implements IObjectChangedSynchronousListener<IMoleJob> {

        @Override
        public void objectChanged(IMoleJob job) throws InternalProcessingError, UserBadDataError {
            jobOutputTransitionsPerformed(job);
        }

    }

    class MoleExecutionAdapterForMoleJob implements IObjectChangedSynchronousListener<IMoleJob> {

        @Override
        public void objectChanged(IMoleJob job) throws InternalProcessingError, UserBadDataError {
            switch (job.getState()) {
                case FAILED:
                    jobFailed(job);
                    break;
            }
        }
    }

    class MoleExecutionAdapterForSubMoleExecution implements IObjectChangedSynchronousListener<ISubMoleExecution> {

        @Override
        public void objectChanged(ISubMoleExecution obj) throws InternalProcessingError, UserBadDataError {
            submitGroups(obj);
        }
    }

    private static final Logger LOGGER = Logger.getLogger(MoleExecution.class.getName());
    
    
    final IMole mole;
    final IMoleJobGrouping moleJobGrouping;

    final BlockingQueue<Tuple2<IJob, IEnvironment>> jobs = new LinkedBlockingQueue<Tuple2<IJob, IEnvironment>>();
    final Map<IMoleJob, Tuple2<ISubMoleExecution, ITicket>> inProgress = Collections.synchronizedMap(new TreeMap<IMoleJob, Tuple2<ISubMoleExecution, ITicket>>()); 
    
    final AtomicLong ticketNumber = new AtomicLong();
    final AtomicLong currentJobId = new AtomicLong();

    final ILocalCommunication localCommunication;
    final IEnvironmentSelection environmentSelectionStrategy;
    final IEnvironment defaultEnvironment;
    
    final BidiMap<Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>, Job> categorizer = new DualHashBidiMap<Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>, Job>();
    final MultiMap<ISubMoleExecution, Job> jobsGrouping = new MultiHashMap<ISubMoleExecution, Job>();

    final MoleExecutionAdapterForMoleJob moleExecutionAdapterForMoleJob = new MoleExecutionAdapterForMoleJob();
    final MoleExecutionAdapterForSubMoleExecution moleExecutionAdapterForSubMoleExecution = new MoleExecutionAdapterForSubMoleExecution();
    final MoleExecutionAdapterForMoleJobOutputTransitionPerformed moleJobOutputTransitionPerformed = new MoleExecutionAdapterForMoleJobOutputTransitionPerformed();

    transient Thread submiter;

    public MoleExecution(IMole mole) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), FixedEnvironmentSelection.EMPTY_SELECTION, MoleJobGrouping.EMPTY_GROUPING);
    }
    
    public MoleExecution(IMole mole, IEnvironmentSelection environmentSelectionStrategy) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), environmentSelectionStrategy, MoleJobGrouping.EMPTY_GROUPING);
    }

    public MoleExecution(IMole mole, IEnvironmentSelection environmentSelectionStrategy, IMoleJobGrouping moleJobGrouping) throws InternalProcessingError, UserBadDataError {
        this(mole, new SynchronizedContext(), new Context(), environmentSelectionStrategy, moleJobGrouping);
    }
    
    public MoleExecution(IMole mole, IContext global, IContext context) throws InternalProcessingError, UserBadDataError {
        this(mole, global, context, FixedEnvironmentSelection.EMPTY_SELECTION, MoleJobGrouping.EMPTY_GROUPING);
    }
    
    public MoleExecution(IMole mole, IContext global, IContext context, IEnvironmentSelection environmentSelectionStrategy, IMoleJobGrouping moleJobGrouping) throws InternalProcessingError, UserBadDataError {
        this.mole = mole;
        this.moleJobGrouping = moleJobGrouping;
        this.localCommunication = new LocalCommunication();
        this.environmentSelectionStrategy = environmentSelectionStrategy;
        this.defaultEnvironment = LocalExecutionEnvironment.getInstance();

        ITicket ticket = nextTicket(createRootTicket());

        IGenericTaskCapsule root = mole.getRoot();
        if (root == null) {
            throw new UserBadDataError("First task of workflow hasn't been set.");
        }

        submit(root, global, context, ticket, new SubMoleExecution());
    }

    @Override
    public synchronized void submit(IGenericTaskCapsule capsule, IContext global, IContext context, ITicket ticket, ISubMoleExecution subMole) throws InternalProcessingError, UserBadDataError {
        IMoleJob job = capsule.toJob(global, context, nextJobId());
        submit(job, capsule, subMole, ticket);
    }
   
    public synchronized void submit(IMoleJob moleJob, IGenericTaskCapsule capsule, ISubMoleExecution subMole, ITicket ticket) throws InternalProcessingError, UserBadDataError {
        Activator.getEventDispatcher().objectChanged(this, oneJobSubmitted, new Object[]{moleJob});

        ExecutionInfoRegistry.GetInstance().register(moleJob, this);
        Activator.getEventDispatcher().registerListener(moleJob, Priority.HIGH.getValue(), moleExecutionAdapterForMoleJob, MoleJob.StateChanged);
        Activator.getEventDispatcher().registerListener(moleJob, Priority.NORMAL.getValue(), moleJobOutputTransitionPerformed, MoleJob.TransitionPerformed);

        inProgress.put(moleJob, new Tuple2<ISubMoleExecution, ITicket>(subMole, ticket));
        subMole.incNbJobInProgress();

        IMoleJobGroupingStrategy strategy = moleJobGrouping.getMoleJobGroupingStrategy(capsule);

        if (strategy != null) {
            IMoleJobCategory category = strategy.getCategory(moleJob.getContext());

            Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory> key = new Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory>(subMole, capsule, category);
            Job job = categorizer.get(key);
            if (job == null) {
                job = new Job();

                categorizer.put(key, job);
                jobsGrouping.put(subMole, job);

                if (!Activator.getEventDispatcher().isRegistred(subMole, moleExecutionAdapterForSubMoleExecution, ISubMoleExecution.allJobsWaitingInGroup)) {
                    Activator.getEventDispatcher().registerListener(subMole, Priority.NORMAL.getValue(), moleExecutionAdapterForSubMoleExecution, ISubMoleExecution.allJobsWaitingInGroup);
                }
            }

            job.addMoleJob(moleJob);
            subMole.incNbJobWaitingInGroup();
        } else {
            Job job = new Job();
            job.addMoleJob(moleJob);
            submit(job, capsule);
        }
    }

    private void submit(Job job, IGenericTaskCapsule capsule) {
        JobRegistry.getInstance().register(job, this);
        IEnvironment environment = environmentSelectionStrategy.selectEnvironment(capsule);
        environment = environment!=null?environment:defaultEnvironment;
        jobs.add(new Tuple2<IJob, IEnvironment>(job, environment));
    }

    synchronized void submitGroups(ISubMoleExecution subMoleExecution) {
        Iterable<Job> jobs = jobsGrouping.remove(subMoleExecution);

        for (Job job : jobs) {
            Tuple3<ISubMoleExecution, IGenericTaskCapsule, IMoleJobCategory> info = categorizer.removeValue(job);
            subMoleExecution.decNbJobWaitingInGroup(job.size());
            submit(job, info._2());
        }
    }


    class Submiter implements Runnable {

        boolean stop = false;

        @Override
        public void run() {
            while (!stop) {
                Tuple2<IJob, IEnvironment> p;
                try {
                    p = jobs.take();
                } catch (InterruptedException e) {
                    LOGGER.log(Level.FINE, "Scheduling interrupted", e);
                    return;
                }
                LOGGER.log(Level.FINER, "New job taken:{0}", p._1());
                try {
                    p._2().submit(p._1());
                } catch (UserBadDataError e) {
                    LOGGER.log(Level.SEVERE, "Error durring scheduling", e);
                } catch (InternalProcessingError e) {
                    LOGGER.log(Level.SEVERE, "Error durring scheduling", e);

                } catch (Throwable t) {
{'fraction_non_alphanumeric': 0.08312020460358056, 'fraction_numerical': 0.0028662139518476058, 'mean_word_length': 3.8866623572505925, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5942004', 'n_tokens_mistral': 6347, 'n_tokens_neox': 6153, 'n_words': 1285}