<commit_before><?xml version="1.0"?>
<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
    <!-- Composite definition (document root) -->
    <xsd:element name="composite" type="COMPOSITE" />

    <xsd:complexType name="COMPOSITE">
        <xsd:sequence>
            <xsd:element name="variables"   type="VARIABLES"   minOccurs="0" />
            <xsd:element name="data"        type="DATA"        minOccurs="1" />
            <xsd:element name="conversions" type="CONVERSIONS" minOccurs="0" />
        </xsd:sequence>
    </xsd:complexType>

    <!-- Variables element definitions -->
    <xsd:complexType name="VARIABLES">
        <xsd:choice minOccurs="0" maxOccurs="unbounded">
            <xsd:element name="scan"     type="SCAN"     minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="input"    type="INPUT"    minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="constant" type="CONSTANT" minOccurs="0" maxOccurs="unbounded" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="SCAN">
        <xsd:attribute name="oid"        type="VAR_OID" use="required" />
        <xsd:attribute name="oid_suffix" type="VAR_NAME" use="required" />
        <xsd:attribute name="poll_value" type="VAR_NAME" use="required" />
    </xsd:complexType>

    <xsd:complexType name="INPUT">
        <xsd:attribute name="name"     type="VAR_NAME" use="required" />
        <xsd:attribute name="value"    type="VAR_NAME" use="required" />
        <xsd:attribute name="required" type="INT_BOOL" />
    </xsd:complexType>

    <xsd:complexType name="CONSTANT">
        <xsd:attribute name="name"  type="VAR_NAME"   use="required" />
        <xsd:attribute name="value" type="xsd:string" use="required" />
    </xsd:complexType>

    <!-- Data element definitions -->
    <xsd:complexType name="DATA">
        <xsd:choice minOccurs="2" maxOccurs="unbounded">
            <xsd:element name="meta"  type="META"   maxOccurs="unbounded" />
            <xsd:element name="value" type="VALUE"  maxOccurs="unbounded" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="META">
        <xsd:attribute name="name"          type="VAR_NAME"   use="required" />
        <xsd:attribute name="source"        type="VAR_SOURCE" use="required" />
        <xsd:attribute name="require_match" type="xsd:string" />
        <xsd:attribute name="invert_match"  type="INT_BOOL" />
    </xsd:complexType>

    <xsd:complexType name="VALUE">
        <xsd:attribute name="name"          type="VAR_NAME"   use="required" />
        <xsd:attribute name="source"        type="VAR_SOURCE" use="required" />
        <xsd:attribute name="require_match" type="xsd:string" />
        <xsd:attribute name="invert_match"  type="INT_BOOL" />
	    <xsd:attribute name="type"          type="VAR_TYPE" />
    </xsd:complexType>

    <!-- Conversions element definitions -->
    <xsd:complexType name="CONVERSIONS">
        <xsd:choice maxOccurs="unbounded">
            <xsd:element name="function" type="FUNCTION" minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="replace"  type="REPLACE"  minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="match"    type="MATCH"    minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="drop"     type="DROP"     minOccurs="0" maxOccurs="1" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="FUNCTION">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="definition" use="required">
            <xsd:simpleType>
                <xsd:restriction base="xsd:string">
                    <!-- "${}" is required in the definition -->
                    <xsd:pattern  value=".*$\{.*\}.*" />
                </xsd:restriction>
            </xsd:simpleType>
        </xsd:attribute>
    </xsd:complexType>

    <xsd:complexType name="CONVERSION_DATA">
        <xsd:attribute name="name" type="VAR_NAME" use="required" />
    </xsd:complexType>

    <xsd:complexType name="REPLACE">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="this" type="xsd:string" use="required" />
      	<xsd:attribute name="with" type="xsd:string" use="required" />
    </xsd:complexType>

    <xsd:complexType name="MATCH">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="pattern" type="xsd:string" use="required" />
        <xsd:attribute name="exclude" type="INT_BOOL" />
        <xsd:attribute name="drop" type="xsd:string" />
    </xsd:complexType>

    <xsd:complexType name="DROP">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
    </xsd:complexType>

    <!-- Global simpleType Definitions -->

    <!-- Validates Variable Names -->
    <xsd:simpleType name="VAR_NAME">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]+|[a-zA-Z]+[a-zA-Z0-9]*" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Variable OIDs -->
    <xsd:simpleType name="VAR_OID">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="\.?([0-9]+\.|[a-zA-Z][a-zA-Z0-9_-]*[a-zA-Z0-9]\.|[a-zA-Z][a-zA-Z0-9]*\.)+([0-9]+|[a-zA-Z][a-zA-Z0-9_-]*[a-zA-Z0-9]|[a-zA-Z][a-zA-Z0-9]*)(\.\*{1,2})?" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Variable OIDs -->
    <xsd:simpleType name="VAR_SOURCE">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="\.?([0-9]+\.|[a-zA-Z][a-zA-Z0-9_-]*[a-zA-Z0-9]\.|[a-zA-Z][a-zA-Z0-9]*\.)+([0-9]+|[a-zA-Z][a-zA-Z0-9_-]*[a-zA-Z0-9]|[a-zA-Z][a-zA-Z0-9]*)(\.\*)?|[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]+|[a-zA-Z]+[a-zA-Z0-9]*" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Data Value Types -->
    <xsd:simpleType name="VAR_TYPE">
        <xsd:restriction base="xsd:string">
            <xsd:enumeration value="rate" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates 1/0 Booleans -->
    <xsd:simpleType name="INT_BOOL">
        <xsd:restriction base="xsd:integer">
            <xsd:enumeration value="0" />
            <xsd:enumeration value="1" />
        </xsd:restriction>
    </xsd:simpleType>

    <xsd:simpleType name="REGEX_TYPE">
        <xsd:restriction base="xsd:string">
            <xsd:enumeration value="match" />
            <xsd:enumeration value="replace" />
        </xsd:restriction>
    </xsd:simpleType>
</xsd:schema><commit_msg>SCTASK0079363 Updated XSD for composites to align with new multi-variable scans<commit_after><?xml version="1.0"?>
<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
    <!-- Composite definition (document root) -->
    <xsd:element name="composite" type="COMPOSITE" />

    <xsd:complexType name="COMPOSITE">
        <xsd:sequence>
            <xsd:element name="variables"   type="VARIABLES"   minOccurs="0" />
            <xsd:element name="data"        type="DATA"        minOccurs="1" />
            <xsd:element name="conversions" type="CONVERSIONS" minOccurs="0" />
        </xsd:sequence>
    </xsd:complexType>

    <!-- Variables element definitions -->
    <xsd:complexType name="VARIABLES">
        <xsd:choice minOccurs="0" maxOccurs="unbounded">
            <xsd:element name="scan"     type="SCAN"     minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="input"    type="INPUT"    minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="constant" type="CONSTANT" minOccurs="0" maxOccurs="unbounded" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="SCAN">
        <xsd:sequence minOccurs="0" maxOccurs="unbounded">
            <xsd:element name="match" type="SCAN_MATCH" maxOccurs="unbounded"/>
        </xsd:sequence>
        <xsd:attribute name="oid"        type="VAR_OID" use="required" />
        <xsd:attribute name="oid_suffix" type="VAR_NAME"/>
        <xsd:attribute name="poll_value" type="VAR_NAME"/>
        <xsd:attribute name="value"      type="VAR_NAME"/>
    </xsd:complexType>

    <xsd:complexType name="INPUT">
        <xsd:attribute name="name"     type="VAR_NAME" use="required" />
        <xsd:attribute name="value"    type="VAR_NAME" use="required" />
        <xsd:attribute name="required" type="INT_BOOL" />
    </xsd:complexType>

    <xsd:complexType name="CONSTANT">
        <xsd:attribute name="name"  type="VAR_NAME"   use="required" />
        <xsd:attribute name="value" type="xsd:string" use="required" />
    </xsd:complexType>


    <xsd:complexType name="SCAN_MATCH">
        <xsd:attribute name="name" type="VAR_NAME" use="required" />
    </xsd:complexType>


    <!-- Data element definitions -->
    <xsd:complexType name="DATA">
        <xsd:choice minOccurs="2" maxOccurs="unbounded">
            <xsd:element name="meta"  type="META"   maxOccurs="unbounded" />
            <xsd:element name="value" type="VALUE"  maxOccurs="unbounded" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="META">
        <xsd:attribute name="name"          type="VAR_NAME"   use="required" />
        <xsd:attribute name="source"        type="VAR_SOURCE" use="required" />
        <xsd:attribute name="require_match" type="xsd:string" />
        <xsd:attribute name="invert_match"  type="INT_BOOL" />
    </xsd:complexType>

    <xsd:complexType name="VALUE">
        <xsd:attribute name="name"          type="VAR_NAME"   use="required" />
        <xsd:attribute name="source"        type="VAR_SOURCE" use="required" />
        <xsd:attribute name="require_match" type="xsd:string" />
        <xsd:attribute name="invert_match"  type="INT_BOOL" />
	    <xsd:attribute name="type"          type="VAR_TYPE" />
    </xsd:complexType>

    <!-- Conversions element definitions -->
    <xsd:complexType name="CONVERSIONS">
        <xsd:choice maxOccurs="unbounded">
            <xsd:element name="function" type="FUNCTION" minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="replace"  type="REPLACE"  minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="match"    type="MATCH"    minOccurs="0" maxOccurs="unbounded" />
            <xsd:element name="drop"     type="DROP"     minOccurs="0" maxOccurs="1" />
        </xsd:choice>
    </xsd:complexType>

    <xsd:complexType name="FUNCTION">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="definition" use="required">
            <xsd:simpleType>
                <xsd:restriction base="xsd:string">
                    <!-- "${}" is required in the definition -->
                    <xsd:pattern  value=".*$\{.*\}.*" />
                </xsd:restriction>
            </xsd:simpleType>
        </xsd:attribute>
    </xsd:complexType>

    <xsd:complexType name="CONVERSION_DATA">
        <xsd:attribute name="name" type="VAR_NAME" use="required" />
    </xsd:complexType>

    <xsd:complexType name="REPLACE">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="this" type="xsd:string" use="required" />
      	<xsd:attribute name="with" type="xsd:string" use="required" />
    </xsd:complexType>

    <xsd:complexType name="MATCH">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
        <xsd:attribute name="pattern" type="xsd:string" use="required" />
        <xsd:attribute name="exclude" type="INT_BOOL" />
        <xsd:attribute name="drop" type="xsd:string" />
    </xsd:complexType>

    <xsd:complexType name="DROP">
        <xsd:sequence>
            <xsd:element name="data" type="CONVERSION_DATA" maxOccurs="unbounded" />
        </xsd:sequence>
    </xsd:complexType>

    <!-- Global simpleType Definitions -->

    <!-- Validates Variable Names -->
    <xsd:simpleType name="VAR_NAME">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]+|[a-zA-Z]+[a-zA-Z0-9]*" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Variable OIDs -->
    <!--    
        Sometimes starts with a period
        \.?

        At least one or more from the following:
        (
            Any integer, followed by a period
            \d+\.
    
            | OR

            Any valid var name that includes an underscore or hyphen
            Must start with alpha chars and end with an alphanumeric
            At least 2 char long
            Followed by a period
            \w[\w\d_-]*[\w\d]\.
    
            | OR

            Any valid combo of alphanum chars starting with an alpha
            Followed by a period
            \w[\w\d]*\.

            | OR

            An asterisk
            Followed by a period
            \*\.
        )+

        Ending with exactly one of the following:
        (
            Either one or two asterisks
            \*{1,2}

            | OR

            Any integer
            \d+

            | OR

            Any valid var name that includes an underscore or hyphen
            Must start with alpha chars and end with an alphanumeric
            At least 2 char long
            Followed by a period
            \w[\w\d_-]*[\w\d]\.

            | OR

            Any valid combo of alphanum chars starting with an alpha
            Followed by a period
            \w[\w\d]*\.
        )
    -->
    <xsd:simpleType name="VAR_OID">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="\.?(\d+\.|\w[\w\d_-]*[\w\d]\.|\w[\w\d]*\.|\*\.)+(\*{1,2}|\d+|\w[\w\d_-]*[\w\d]|\w[\w\d]*)" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Variable OIDs -->
    <xsd:simpleType name="VAR_SOURCE">
        <xsd:restriction base="xsd:string">
            <xsd:pattern value="\.?(\d+\.|\w[\w\d_-]*[\w\d]\.|\w[\w\d]*\.)+(\d+|\w[\w\d_-]*[\w\d]|\w[\w\d]*)(\.\*)?|\w+[\w\d_-]*[\w\d]+|\w+[\w\d]*" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates Data Value Types -->
    <xsd:simpleType name="VAR_TYPE">
        <xsd:restriction base="xsd:string">
            <xsd:enumeration value="rate" />
        </xsd:restriction>
    </xsd:simpleType>

    <!-- Validates 1/0 Booleans -->
    <xsd:simpleType name="INT_BOOL">
        <xsd:restriction base="xsd:integer">
            <xsd:enumeration value="0" />
            <xsd:enumeration value="1" />
        </xsd:restriction>
    </xsd:simpleType>

    <xsd:simpleType name="REGEX_TYPE">
        <xsd:restriction base="xsd:string">
            <xsd:enumeration value="match" />
            <xsd:enumeration value="replace" />
        </xsd:restriction>
    </xsd:simpleType>
</xsd:schema>
{'fraction_non_alphanumeric': 0.17917467046433067, 'fraction_numerical': 0.007551169106444989, 'mean_word_length': 2.4916743755781683, 'pattern_counts': {'":': 0, '<': 285, '<?xml version=': 2, '>': 285, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476145', 'n_tokens_mistral': 5284, 'n_tokens_neox': 4772, 'n_words': 815}