<commit_before>/*
 * Copyright 2007-2014 University Of Southern California
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.isi.pegasus.planner.catalog.replica.impl;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.dataformat.yaml.JacksonYAMLParseException;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import edu.isi.pegasus.common.util.Boolean;
import edu.isi.pegasus.common.util.Currently;
import edu.isi.pegasus.common.util.Escape;
import edu.isi.pegasus.common.util.VariableExpander;
import edu.isi.pegasus.planner.catalog.CatalogException;
import edu.isi.pegasus.planner.catalog.ReplicaCatalog;
import edu.isi.pegasus.planner.catalog.replica.ReplicaCatalogEntry;
import edu.isi.pegasus.planner.catalog.replica.ReplicaCatalogException;
import edu.isi.pegasus.planner.catalog.replica.classes.ReplicaCatalogJsonDeserializer;
import edu.isi.pegasus.planner.catalog.replica.classes.ReplicaCatalogKeywords;
import edu.isi.pegasus.planner.classes.ReplicaLocation;
import edu.isi.pegasus.planner.common.PegasusJsonSerializer;
import edu.isi.pegasus.planner.common.PegasusProperties;
import edu.isi.pegasus.planner.common.VariableExpansionReader;
import edu.isi.pegasus.planner.namespace.Metadata;
import edu.isi.pegasus.planner.parser.YAMLSchemaValidationResult;
import edu.isi.pegasus.planner.parser.YAMLSchemaValidator;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * This class implements a replica catalog on top of a simple file with regular expression based
 * entries, which contains two or more columns. It is neither transactionally safe, nor advised to
 * use for production purposes in any way. Multiple concurrent instances <b>will clobber</b> each
 * other!
 *
 * <p>The site attribute should be specified whenever possible. The attribute key for the site
 * attribute is "site". For the shell planner, its value will always be "local".
 *
 * <p>The class is permissive in what inputs it accepts. The LFN may or may not be quoted. If it
 * contains linear whitespace, quotes, backslash or an equality sign, it must be quoted and escaped.
 * Ditto for the PFN.
 *
 * <p>A sample replica catalog description is indicated below.
 *
 * <p>
 *
 * <pre>
 * pegasus: "5.0"
 * replicas:
 * # matches "f.a"
 * - lfn: "f.a"
 *   pfn: "file:///Volumes/data/input/f.a"
 *   site: "local"
 *
 * # matches faa, f.a, f0a, etc.
 * - lfn: "f.a"
 *   pfn: "file:///Volumes/data/input/f.a"
 *   site: "local"
 *   regex: true
 * </pre>
 *
 * <p>The class is strict when producing (storing) results. The LFN and PFN are only quoted and
 * escaped, if necessary. The attribute values are always quoted and escaped.
 *
 * @author <NAME>
 * @version $Revision: 5402 $
 */
@JsonDeserialize(using = YAML.CallbackJsonDeserializer.class)
@JsonSerialize(using = YAML.JsonSerializer.class)
public class YAML implements ReplicaCatalog {
    /** The default transformation Catalog version to which this maps to */
    public static final String DEFAULT_REPLICA_CATALOG_VERSION = "5.0";

    /**
     * The name of the key that disables writing back to the cache file. Designates a static file.
     * i.e. read only
     */
    public static final String READ_ONLY_KEY = "read.only";

    /** The "not-so-official" location URL of the Replica Catalog Schema. */
    public static final String SCHEMA_URI = "http://pegasus.isi.edu/schema/rc-5.0.yml";

    /** File object of the schema.. */
    private final File SCHEMA_FILE;

    /**
     * Records the quoting mode for LFNs and PFNs. If false, only quote as necessary. If true,
     * always quote all LFNs and PFNs.
     */
    protected boolean mQuote = false;

    /** Records the name of the on-disk representation. */
    protected String mFilename = null;

    /** Maintains a memory slurp of the file representation. */
    /** Maintains a memory slurp of the file representation. */
    protected Map<String, ReplicaLocation> mLFN = null;

    protected Map<String, ReplicaLocation> mLFNRegex = null;

    protected Map<String, Pattern> mLFNPattern = null;

    /** A boolean indicating whether the catalog is read only or not. */
    boolean m_readonly;

    /** Handle to pegasus variable expander */
    private VariableExpander mVariableExpander;

    /** The version for the Replica Catalog */
    private String mVersion;

    /**
     * Default empty constructor creates an object that is not yet connected to any database. You
     * must use support methods to connect before this instance becomes usable.
     *
     * @see #connect(Properties)
     */
    public YAML() {
        // make connection defunc
        mLFN = null;
        mLFNRegex = null;
        mLFNPattern = null;
        mFilename = null;
        m_readonly = false;
        mVariableExpander = new VariableExpander();
        mVersion = YAML.DEFAULT_REPLICA_CATALOG_VERSION;

        PegasusProperties props = PegasusProperties.getInstance();
        File schemaDir = props.getSchemaDir();
        File yamlSchemaDir = new File(schemaDir, "yaml");
        SCHEMA_FILE = new File(yamlSchemaDir, new File(SCHEMA_URI).getName());
    }

    /**
     * Reads the on-disk map file into memory.
     *
     * @param filename is the name of the file to read.
     * @return true, if the in-memory data structures appear sound.
     */
    public boolean connect(String filename) {
        // sanity check
        if (filename == null) {
            return false;
        }
        mFilename = filename;
        mLFN = new LinkedHashMap<String, ReplicaLocation>();
        mLFNRegex = new LinkedHashMap<String, ReplicaLocation>();
        mLFNPattern = new LinkedHashMap<String, Pattern>();

        // first attempt to validate
        if (validate(new File(filename), SCHEMA_FILE)) {<commit_msg>PM-1523 we should only attempt to valid rc file if it exists<commit_after>/*
 * Copyright 2007-2014 University Of Southern California
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.isi.pegasus.planner.catalog.replica.impl;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.ObjectCodec;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.dataformat.yaml.JacksonYAMLParseException;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
import edu.isi.pegasus.common.util.Boolean;
import edu.isi.pegasus.common.util.Currently;
import edu.isi.pegasus.common.util.Escape;
import edu.isi.pegasus.common.util.VariableExpander;
import edu.isi.pegasus.planner.catalog.CatalogException;
import edu.isi.pegasus.planner.catalog.ReplicaCatalog;
import edu.isi.pegasus.planner.catalog.replica.ReplicaCatalogEntry;
import edu.isi.pegasus.planner.catalog.replica.ReplicaCatalogException;
import edu.isi.pegasus.planner.catalog.replica.classes.ReplicaCatalogJsonDeserializer;
import edu.isi.pegasus.planner.catalog.replica.classes.ReplicaCatalogKeywords;
import edu.isi.pegasus.planner.classes.ReplicaLocation;
import edu.isi.pegasus.planner.common.PegasusJsonSerializer;
import edu.isi.pegasus.planner.common.PegasusProperties;
import edu.isi.pegasus.planner.common.VariableExpansionReader;
import edu.isi.pegasus.planner.namespace.Metadata;
import edu.isi.pegasus.planner.parser.YAMLSchemaValidationResult;
import edu.isi.pegasus.planner.parser.YAMLSchemaValidator;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * This class implements a replica catalog on top of a simple file with regular expression based
 * entries, which contains two or more columns. It is neither transactionally safe, nor advised to
 * use for production purposes in any way. Multiple concurrent instances <b>will clobber</b> each
 * other!
 *
 * <p>The site attribute should be specified whenever possible. The attribute key for the site
 * attribute is "site". For the shell planner, its value will always be "local".
 *
 * <p>The class is permissive in what inputs it accepts. The LFN may or may not be quoted. If it
 * contains linear whitespace, quotes, backslash or an equality sign, it must be quoted and escaped.
 * Ditto for the PFN.
 *
 * <p>A sample replica catalog description is indicated below.
 *
 * <p>
 *
 * <pre>
 * pegasus: "5.0"
 * replicas:
 * # matches "f.a"
 * - lfn: "f.a"
 *   pfn: "file:///Volumes/data/input/f.a"
 *   site: "local"
 *
 * # matches faa, f.a, f0a, etc.
 * - lfn: "f.a"
 *   pfn: "file:///Volumes/data/input/f.a"
 *   site: "local"
 *   regex: true
 * </pre>
 *
 * <p>The class is strict when producing (storing) results. The LFN and PFN are only quoted and
 * escaped, if necessary. The attribute values are always quoted and escaped.
 *
 * @author <NAME>
 * @version $Revision: 5402 $
 */
@JsonDeserialize(using = YAML.CallbackJsonDeserializer.class)
@JsonSerialize(using = YAML.JsonSerializer.class)
public class YAML implements ReplicaCatalog {
    /** The default transformation Catalog version to which this maps to */
    public static final String DEFAULT_REPLICA_CATALOG_VERSION = "5.0";

    /**
     * The name of the key that disables writing back to the cache file. Designates a static file.
     * i.e. read only
     */
    public static final String READ_ONLY_KEY = "read.only";

    /** The "not-so-official" location URL of the Replica Catalog Schema. */
    public static final String SCHEMA_URI = "http://pegasus.isi.edu/schema/rc-5.0.yml";

    /** File object of the schema.. */
    private final File SCHEMA_FILE;

    /**
     * Records the quoting mode for LFNs and PFNs. If false, only quote as necessary. If true,
     * always quote all LFNs and PFNs.
     */
    protected boolean mQuote = false;

    /** Records the name of the on-disk representation. */
    protected String mFilename = null;

    /** Maintains a memory slurp of the file representation. */
    /** Maintains a memory slurp of the file representation. */
    protected Map<String, ReplicaLocation> mLFN = null;

    protected Map<String, ReplicaLocation> mLFNRegex = null;

    protected Map<String, Pattern> mLFNPattern = null;

    /** A boolean indicating whether the catalog is read only or not. */
    boolean m_readonly;

    /** Handle to pegasus variable expander */
    private VariableExpander mVariableExpander;

    /** The version for the Replica Catalog */
    private String mVersion;

    /**
     * Default empty constructor creates an object that is not yet connected to any database. You
     * must use support methods to connect before this instance becomes usable.
     *
     * @see #connect(Properties)
     */
    public YAML() {
        // make connection defunc
        mLFN = null;
        mLFNRegex = null;
        mLFNPattern = null;
        mFilename = null;
        m_readonly = false;
        mVariableExpander = new VariableExpander();
        mVersion = YAML.DEFAULT_REPLICA_CATALOG_VERSION;

        PegasusProperties props = PegasusProperties.getInstance();
        File schemaDir = props.getSchemaDir();
        File yamlSchemaDir = new File(schemaDir, "yaml");
        SCHEMA_FILE = new File(yamlSchemaDir, new File(SCHEMA_URI).getName());
    }

    /**
     * Reads the on-disk map file into memory.
     *
     * @param filename is the name of the file to read.
     * @return true, if the in-memory data structures appear sound.
     */
    public boolean connect(String filename) {
        // sanity check
        if (filename == null) {
            return false;
        }
        mFilename = filename;
        mLFN = new LinkedHashMap<String, ReplicaLocation>();
        mLFNRegex = new LinkedHashMap<String, ReplicaLocation>();
        mLFNPattern = new LinkedHashMap<String, Pattern>();

        File replicaFile = new File(filename);
        // first attempt to validate only if it exists
        if (replicaFile.exists() && validate(replicaFile, SCHEMA_FILE)) {
{'fraction_non_alphanumeric': 0.09644770787854733, 'fraction_numerical': 0.003307534563736191, 'mean_word_length': 3.715533374922021, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1460207', 'n_tokens_mistral': 4619, 'n_tokens_neox': 4373, 'n_words': 1355}