<commit_before>/**
 * Copyright 2011-2016 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.yaess.core;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.asakusafw.yaess.core.util.PropertiesUtil;

/**
 * A script describes each jobflow structure.
 * @since 0.2.3
 * @version 0.8.0
 */
public final class FlowScript {

    static final Logger LOG = LoggerFactory.getLogger(FlowScript.class);

    /**
     * A configuration key prefix of each flow.
     */
    public static final String KEY_FLOW_PREFIX = "flow.";

    /**
     * A configuration key name of IDs.
     */
    public static final String KEY_ID = "id";

    /**
     * A configuration key name of blockers' ID.
     */
    public static final String KEY_BLOCKERS = "blockerIds";

    /**
     * A configuration key name of {@link ExecutionScript#getKind() script kind}.
     */
    public static final String KEY_KIND = "kind";

    /**
     * A configuration key name of {@link HadoopScript#getClassName() class name}.
     */
    public static final String KEY_CLASS_NAME = "class";

    /**
     * A configuration key name of {@link CommandScript#getProfileName() profile name}.
     */
    public static final String KEY_PROFILE = "profile";

    /**
     * A configuration key name of {@link CommandScript#getModuleName() module name}.
     */
    public static final String KEY_MODULE = "module";

    /**
     * A configuration key name of {@link ExecutionScript#getSupportedExtensions() supported extensions}.
     * @since 0.8.0
     */
    public static final String KEY_SUPPORTED_EXTENSIONS = "extensions";

    /**
     * A configuration key name of enabled {@link ExecutionScript#getKind() script kinds}.
     * @since 0.8.0
     */
    public static final String KEY_ENABLED_SCRIPT_KINDS = "enables";

    /**
     * The configuration key prefix of {@link ExecutionScript#getEnvironmentVariables() environment variables}.
     */
    public static final String KEY_ENV_PREFIX = "env.";

    /**
     * The configuration key prefix of {@link CommandScript#getCommandLineTokens() command line tokens}.
     * This must follows each command token with lexicographic order.
     */
    private static final String KEY_COMMAND_PREFIX = "command.";

    /**
     * The configuration key prefix of {@link HadoopScript#getHadoopProperties() extra Hadoop properties}.
     */
    private static final String KEY_PROP_PREFIX = "prop.";

    private static final Comparator<ExecutionScript> SCRIPT_COMPARATOR = new Comparator<ExecutionScript>() {
        @Override
        public int compare(ExecutionScript o1, ExecutionScript o2) {
            return o1.getId().compareTo(o2.getId());
        }
    };

    private final String id;

    private final Set<String> blockerIds;

    private final Map<ExecutionPhase, Set<ExecutionScript>> scripts;

    private final Set<ExecutionScript.Kind> enables;

    /**
     * Creates a new instance.
     * @param id the flow ID
     * @param blockerIds the predecessors' flow ID
     * @param scripts the execution scripts for each {@link ExecutionPhase}
     * @throws IllegalArgumentException if some parameters were {@code null}
     * @deprecated Use {@link #FlowScript(String, Set, Map, Set)} instead
     */
    @Deprecated
    public FlowScript(
            String id,
            Set<String> blockerIds,
            Map<ExecutionPhase, ? extends Collection<? extends ExecutionScript>> scripts) {
        this(id, blockerIds, scripts, Collections.<ExecutionScript.Kind>emptySet());
    }

    /**
     * Creates a new instance.
     * @param id the flow ID
     * @param blockerIds the predecessors' flow ID
     * @param scripts the execution scripts for each {@link ExecutionPhase}
     * @param enables the enabled script kinds
     * @throws IllegalArgumentException if some parameters were {@code null}
     * @since 0.8.0
     */
    public FlowScript(
            String id,
            Set<String> blockerIds,
            Map<ExecutionPhase, ? extends Collection<? extends ExecutionScript>> scripts,
            Set<ExecutionScript.Kind> enables) {
        if (id == null) {
            throw new IllegalArgumentException("id must not be null"); //$NON-NLS-1$
        }
        if (id.indexOf('.') >= 0) {
            throw new IllegalArgumentException("id must not contain dot"); //$NON-NLS-1$
        }
        if (blockerIds == null) {
            throw new IllegalArgumentException("blockerIds must not be null"); //$NON-NLS-1$
        }
        if (scripts == null) {
            throw new IllegalArgumentException("scripts must not be null"); //$NON-NLS-1$
        }
        if (enables == null) {
            throw new IllegalArgumentException("enables must not be null"); //$NON-NLS-1$
        }
        this.id = id;
        this.blockerIds = Collections.unmodifiableSet(new LinkedHashSet<>(blockerIds));
        EnumMap<ExecutionPhase, Set<ExecutionScript>> map = new EnumMap<>(ExecutionPhase.class);
        for (ExecutionPhase phase : ExecutionPhase.values()) {
            if (scripts.containsKey(phase)) {
                TreeSet<ExecutionScript> set = new TreeSet<>(SCRIPT_COMPARATOR);
                set.addAll(scripts.get(phase));
                if (set.size() != scripts.get(phase).size()) {
                    throw new IllegalArgumentException(MessageFormat.format(
                            "{0}@{1} contains duplicated IDs in scripts",
                            id,
                            phase));
                }
                for (ExecutionScript script : set) {
                    if (enables.contains(script.getKind()) == false) {
                        throw new IllegalArgumentException(MessageFormat.format(
                                "script kind \"{3}\" is not supported in this flow: {0}",
                                id,
                                script.getKind().getSymbol()));
                    }
                }
                map.put(phase, Collections.unmodifiableSet(set));
            } else {
                map.put(phase, Collections.<ExecutionScript>emptySet());
            }
        }
        this.scripts = Collections.unmodifiableMap(map);<commit_msg>All execution script kinds should be enabled in default.<commit_after>/**
 * Copyright 2011-2016 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.yaess.core;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.asakusafw.yaess.core.util.PropertiesUtil;

/**
 * A script describes each jobflow structure.
 * @since 0.2.3
 * @version 0.8.0
 */
public final class FlowScript {

    static final Logger LOG = LoggerFactory.getLogger(FlowScript.class);

    /**
     * A configuration key prefix of each flow.
     */
    public static final String KEY_FLOW_PREFIX = "flow.";

    /**
     * A configuration key name of IDs.
     */
    public static final String KEY_ID = "id";

    /**
     * A configuration key name of blockers' ID.
     */
    public static final String KEY_BLOCKERS = "blockerIds";

    /**
     * A configuration key name of {@link ExecutionScript#getKind() script kind}.
     */
    public static final String KEY_KIND = "kind";

    /**
     * A configuration key name of {@link HadoopScript#getClassName() class name}.
     */
    public static final String KEY_CLASS_NAME = "class";

    /**
     * A configuration key name of {@link CommandScript#getProfileName() profile name}.
     */
    public static final String KEY_PROFILE = "profile";

    /**
     * A configuration key name of {@link CommandScript#getModuleName() module name}.
     */
    public static final String KEY_MODULE = "module";

    /**
     * A configuration key name of {@link ExecutionScript#getSupportedExtensions() supported extensions}.
     * @since 0.8.0
     */
    public static final String KEY_SUPPORTED_EXTENSIONS = "extensions";

    /**
     * A configuration key name of enabled {@link ExecutionScript#getKind() script kinds}.
     * @since 0.8.0
     */
    public static final String KEY_ENABLED_SCRIPT_KINDS = "enables";

    /**
     * The configuration key prefix of {@link ExecutionScript#getEnvironmentVariables() environment variables}.
     */
    public static final String KEY_ENV_PREFIX = "env.";

    /**
     * The configuration key prefix of {@link CommandScript#getCommandLineTokens() command line tokens}.
     * This must follows each command token with lexicographic order.
     */
    private static final String KEY_COMMAND_PREFIX = "command.";

    /**
     * The configuration key prefix of {@link HadoopScript#getHadoopProperties() extra Hadoop properties}.
     */
    private static final String KEY_PROP_PREFIX = "prop.";

    private static final Comparator<ExecutionScript> SCRIPT_COMPARATOR = new Comparator<ExecutionScript>() {
        @Override
        public int compare(ExecutionScript o1, ExecutionScript o2) {
            return o1.getId().compareTo(o2.getId());
        }
    };

    private final String id;

    private final Set<String> blockerIds;

    private final Map<ExecutionPhase, Set<ExecutionScript>> scripts;

    private final Set<ExecutionScript.Kind> enables;

    /**
     * Creates a new instance.
     * @param id the flow ID
     * @param blockerIds the predecessors' flow ID
     * @param scripts the execution scripts for each {@link ExecutionPhase}
     * @throws IllegalArgumentException if some parameters were {@code null}
     * @deprecated Use {@link #FlowScript(String, Set, Map, Set)} instead
     */
    @Deprecated
    public FlowScript(
            String id,
            Set<String> blockerIds,
            Map<ExecutionPhase, ? extends Collection<? extends ExecutionScript>> scripts) {
        this(id, blockerIds, scripts, EnumSet.allOf(ExecutionScript.Kind.class));
    }

    /**
     * Creates a new instance.
     * @param id the flow ID
     * @param blockerIds the predecessors' flow ID
     * @param scripts the execution scripts for each {@link ExecutionPhase}
     * @param enables the enabled script kinds
     * @throws IllegalArgumentException if some parameters were {@code null}
     * @since 0.8.0
     */
    public FlowScript(
            String id,
            Set<String> blockerIds,
            Map<ExecutionPhase, ? extends Collection<? extends ExecutionScript>> scripts,
            Set<ExecutionScript.Kind> enables) {
        if (id == null) {
            throw new IllegalArgumentException("id must not be null"); //$NON-NLS-1$
        }
        if (id.indexOf('.') >= 0) {
            throw new IllegalArgumentException("id must not contain dot"); //$NON-NLS-1$
        }
        if (blockerIds == null) {
            throw new IllegalArgumentException("blockerIds must not be null"); //$NON-NLS-1$
        }
        if (scripts == null) {
            throw new IllegalArgumentException("scripts must not be null"); //$NON-NLS-1$
        }
        if (enables == null) {
            throw new IllegalArgumentException("enables must not be null"); //$NON-NLS-1$
        }
        this.id = id;
        this.blockerIds = Collections.unmodifiableSet(new LinkedHashSet<>(blockerIds));
        EnumMap<ExecutionPhase, Set<ExecutionScript>> map = new EnumMap<>(ExecutionPhase.class);
        for (ExecutionPhase phase : ExecutionPhase.values()) {
            if (scripts.containsKey(phase)) {
                TreeSet<ExecutionScript> set = new TreeSet<>(SCRIPT_COMPARATOR);
                set.addAll(scripts.get(phase));
                if (set.size() != scripts.get(phase).size()) {
                    throw new IllegalArgumentException(MessageFormat.format(
                            "{0}@{1} contains duplicated IDs in scripts",
                            id,
                            phase));
                }
                for (ExecutionScript script : set) {
                    if (enables.contains(script.getKind()) == false) {
                        throw new IllegalArgumentException(MessageFormat.format(
                                "script kind \"{1}\" is not supported in this flow: {0}",
                                id,
                                script.getKind().getSymbol()));
                    }
                }
                map.put(phase, Collections.unmodifiableSet(set));
            } else {
                map.put(phase, Collections.<ExecutionScript>emptySet());
            }
        }
        this.scripts = Collections.unmodifiableMap(map);
{'fraction_non_alphanumeric': 0.10077519379844961, 'fraction_numerical': 0.005952380952380952, 'mean_word_length': 2.502787878787879, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1093749', 'n_tokens_mistral': 3836, 'n_tokens_neox': 3717, 'n_words': 1201}