<commit_before>/*
 * ProActive Parallel Suite(TM):
 * The Open Source library for parallel and distributed
 * Workflows & Scheduling, Orchestration, Cloud Automation
 * and Big Data Analysis on Enterprise Grids & Clouds.
 *
 * Copyright (c) 2007 - 2017 ActiveEon
 * Contact: <EMAIL>
 *
 * This library is free software: you can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation: version 3 of
 * the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * If needed, contact us to obtain a release under GPL Version 2 or 3
 * or a different license than the AGPL.
 */
package org.ow2.proactive.catalog.util.parser;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.ow2.proactive.catalog.repository.entity.KeyValueLabelMetadataEntity;
import org.ow2.proactive.catalog.service.exception.ParsingObjectException;
import org.ow2.proactive.catalog.util.SeparatorUtility;
import org.ow2.proactive.scheduler.common.exception.JobCreationException;
import org.ow2.proactive.scheduler.common.job.Job;
import org.ow2.proactive.scheduler.common.job.JobVariable;
import org.ow2.proactive.scheduler.common.job.TaskFlowJob;
import org.ow2.proactive.scheduler.common.job.factories.JobFactory;
import org.ow2.proactive.scheduler.common.task.ScriptTask;
import org.ow2.proactive.scheduler.common.task.Task;
import org.ow2.proactive.scheduler.common.task.TaskVariable;
import org.ow2.proactive.scheduler.core.properties.PASchedulerProperties;
import org.springframework.stereotype.Component;

import lombok.NoArgsConstructor;
import lombok.extern.log4j.Log4j2;


/**
 * Parse a ProActive XML workflow (whatever the schema version is)
 * in order to extract some specific values (job name, project name, generic
 * information and variables).
 *
 * @author ActiveEon Team
 */
@Log4j2
@NoArgsConstructor
@Component
public final class WorkflowParser extends AbstractCatalogObjectParser {

    private static final String JOB_NAME_KEY = "name";

    private static final String PROJECT_NAME_KEY = "project_name";

    private static final String JOB_AND_PROJECT_LABEL = "job_information";

    public static final String ATTRIBUTE_GENERIC_INFORMATION_LABEL = "generic_information";

    public static final String ATTRIBUTE_VARIABLE_LABEL = "variable";

    private static final String ATTRIBUTE_VARIABLE_MODEL_LABEL = "variable_model";

    public static final String ATTRIBUTE_DEPENDS_ON_LABEL = "depends_on";

    private static final String CATALOG_OBJECT_MODEL = "PA:CATALOG_OBJECT";

    public static final String LATEST_VERSION = "latest";

    public static final String DEPENDS_ON_SEPARATOR = "/";

    private static final String CATALOG_OBJECT_MODEL_REGEXP = "^([^/]+/[^/]+)(/[^/][0-9]{12})?$";

    // e.g ${PA_CATALOG_REST_URL}/buckets/notification-tools/resources/Web_Validation_Script/revisions/1555683355837/raw
    private static final String SCRIPT_URL_REGEX = "^/buckets/([^/]+)/resources/([^/]+)(/revisions/([^/][0-9]{12}))?/raw";

    private static final Pattern URL_PATTERN = Pattern.compile(SCRIPT_URL_REGEX);

    private static final Pattern PATTERN = Pattern.compile(CATALOG_OBJECT_MODEL_REGEXP);

    public static final String JOB_DESCRIPTION_KEY = "description";

    private static final String JOB_VISUALIZATION_KEY = "visualization";

    SeparatorUtility separatorUtility = new SeparatorUtility();

    @Override
    List<KeyValueLabelMetadataEntity> getMetadataKeyValues(InputStream inputStream) {
        TaskFlowJob job;
        try {
            job = (TaskFlowJob) JobFactory.getFactory().createJob(inputStream);
        } catch (JobCreationException e) {
            throw new ParsingObjectException(e.getMessage(), e);
        }
        Set keyValueMapBuilder = new LinkedHashSet();

        addProjectNameIfNotNullAndNotEmpty(keyValueMapBuilder, job);
        addJobNameIfNotNull(keyValueMapBuilder, job);
        job.getUnresolvedGenericInformation()
           .forEach((name, value) -> addGenericInformationIfNotNull(keyValueMapBuilder, name, value));
        job.getUnresolvedVariables()
           .values()
           .forEach(jobVariable -> addVariableIfNotNullAndModelIfNotEmpty(keyValueMapBuilder, jobVariable));
        job.getTasks()
           .forEach(task -> task.getVariables()
                                .values()
                                .forEach(taskVariable -> addDependsOnIfCatalogObjectModelExistOnTaskVariable(keyValueMapBuilder,
                                                                                                             taskVariable)));
        job.getTasks().forEach(task -> addDependsOnIfScriptUrlExistInEachTaskScripts(keyValueMapBuilder, task));
        addJobDescriptionIfNotNullAndNotEmpty(keyValueMapBuilder, job);
        addJobVizualisationIfNotNullAndNotEmpty(keyValueMapBuilder, job);

        return new ArrayList<>(keyValueMapBuilder);
    }
<commit_msg>WorkflowParser : disable global variable handling<commit_after>/*
 * ProActive Parallel Suite(TM):
 * The Open Source library for parallel and distributed
 * Workflows & Scheduling, Orchestration, Cloud Automation
 * and Big Data Analysis on Enterprise Grids & Clouds.
 *
 * Copyright (c) 2007 - 2017 ActiveEon
 * Contact: <EMAIL>
 *
 * This library is free software: you can redistribute it and/or
 * modify it under the terms of the GNU Affero General Public License
 * as published by the Free Software Foundation: version 3 of
 * the License.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 *
 * If needed, contact us to obtain a release under GPL Version 2 or 3
 * or a different license than the AGPL.
 */
package org.ow2.proactive.catalog.util.parser;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.ow2.proactive.catalog.repository.entity.KeyValueLabelMetadataEntity;
import org.ow2.proactive.catalog.service.exception.ParsingObjectException;
import org.ow2.proactive.catalog.util.SeparatorUtility;
import org.ow2.proactive.scheduler.common.exception.JobCreationException;
import org.ow2.proactive.scheduler.common.job.Job;
import org.ow2.proactive.scheduler.common.job.JobVariable;
import org.ow2.proactive.scheduler.common.job.TaskFlowJob;
import org.ow2.proactive.scheduler.common.job.factories.JobFactory;
import org.ow2.proactive.scheduler.common.task.ScriptTask;
import org.ow2.proactive.scheduler.common.task.Task;
import org.ow2.proactive.scheduler.common.task.TaskVariable;
import org.ow2.proactive.scheduler.core.properties.PASchedulerProperties;
import org.springframework.stereotype.Component;

import lombok.NoArgsConstructor;
import lombok.extern.log4j.Log4j2;


/**
 * Parse a ProActive XML workflow (whatever the schema version is)
 * in order to extract some specific values (job name, project name, generic
 * information and variables).
 *
 * @author ActiveEon Team
 */
@Log4j2
@NoArgsConstructor
@Component
public final class WorkflowParser extends AbstractCatalogObjectParser {

    private static final String JOB_NAME_KEY = "name";

    private static final String PROJECT_NAME_KEY = "project_name";

    private static final String JOB_AND_PROJECT_LABEL = "job_information";

    public static final String ATTRIBUTE_GENERIC_INFORMATION_LABEL = "generic_information";

    public static final String ATTRIBUTE_VARIABLE_LABEL = "variable";

    private static final String ATTRIBUTE_VARIABLE_MODEL_LABEL = "variable_model";

    public static final String ATTRIBUTE_DEPENDS_ON_LABEL = "depends_on";

    private static final String CATALOG_OBJECT_MODEL = "PA:CATALOG_OBJECT";

    public static final String LATEST_VERSION = "latest";

    public static final String DEPENDS_ON_SEPARATOR = "/";

    private static final String CATALOG_OBJECT_MODEL_REGEXP = "^([^/]+/[^/]+)(/[^/][0-9]{12})?$";

    // e.g ${PA_CATALOG_REST_URL}/buckets/notification-tools/resources/Web_Validation_Script/revisions/1555683355837/raw
    private static final String SCRIPT_URL_REGEX = "^/buckets/([^/]+)/resources/([^/]+)(/revisions/([^/][0-9]{12}))?/raw";

    private static final Pattern URL_PATTERN = Pattern.compile(SCRIPT_URL_REGEX);

    private static final Pattern PATTERN = Pattern.compile(CATALOG_OBJECT_MODEL_REGEXP);

    public static final String JOB_DESCRIPTION_KEY = "description";

    private static final String JOB_VISUALIZATION_KEY = "visualization";

    SeparatorUtility separatorUtility = new SeparatorUtility();

    @Override
    List<KeyValueLabelMetadataEntity> getMetadataKeyValues(InputStream inputStream) {
        TaskFlowJob job;
        try {
            job = (TaskFlowJob) JobFactory.getFactory(false).createJob(inputStream);
        } catch (JobCreationException e) {
            throw new ParsingObjectException(e.getMessage(), e);
        }
        Set keyValueMapBuilder = new LinkedHashSet();

        addProjectNameIfNotNullAndNotEmpty(keyValueMapBuilder, job);
        addJobNameIfNotNull(keyValueMapBuilder, job);
        job.getUnresolvedGenericInformation()
           .forEach((name, value) -> addGenericInformationIfNotNull(keyValueMapBuilder, name, value));
        job.getUnresolvedVariables()
           .values()
           .forEach(jobVariable -> addVariableIfNotNullAndModelIfNotEmpty(keyValueMapBuilder, jobVariable));
        job.getTasks()
           .forEach(task -> task.getVariables()
                                .values()
                                .forEach(taskVariable -> addDependsOnIfCatalogObjectModelExistOnTaskVariable(keyValueMapBuilder,
                                                                                                             taskVariable)));
        job.getTasks().forEach(task -> addDependsOnIfScriptUrlExistInEachTaskScripts(keyValueMapBuilder, task));
        addJobDescriptionIfNotNullAndNotEmpty(keyValueMapBuilder, job);
        addJobVizualisationIfNotNullAndNotEmpty(keyValueMapBuilder, job);

        return new ArrayList<>(keyValueMapBuilder);
    }

{'fraction_non_alphanumeric': 0.08868281264067705, 'fraction_numerical': 0.009003331232556045, 'mean_word_length': 3.9150442477876104, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7393293', 'n_tokens_mistral': 3180, 'n_tokens_neox': 3067, 'n_words': 784}