<commit_before><?php
/**
 * Stores weightings and groupings of tokens.
 *
 * @author    <NAME> <<EMAIL>>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Util;

define('T_NONE', 'PHPCS_T_NONE');
define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
define('T_OPEN_SQUARE_BRACKET', 'PHPCS_T_OPEN_SQUARE_BRACKET');
define('T_CLOSE_SQUARE_BRACKET', 'PHPCS_T_CLOSE_SQUARE_BRACKET');
define('T_OPEN_PARENTHESIS', 'PHPCS_T_OPEN_PARENTHESIS');
define('T_CLOSE_PARENTHESIS', 'PHPCS_T_CLOSE_PARENTHESIS');
define('T_COLON', 'PHPCS_T_COLON');
define('T_STRING_CONCAT', 'PHPCS_T_STRING_CONCAT');
define('T_INLINE_THEN', 'PHPCS_T_INLINE_THEN');
define('T_INLINE_ELSE', 'PHPCS_T_INLINE_ELSE');
define('T_NULL', 'PHPCS_T_NULL');
define('T_FALSE', 'PHPCS_T_FALSE');
define('T_TRUE', 'PHPCS_T_TRUE');
define('T_SEMICOLON', 'PHPCS_T_SEMICOLON');
define('T_EQUAL', 'PHPCS_T_EQUAL');
define('T_MULTIPLY', 'PHPCS_T_MULTIPLY');
define('T_DIVIDE', 'PHPCS_T_DIVIDE');
define('T_PLUS', 'PHPCS_T_PLUS');
define('T_MINUS', 'PHPCS_T_MINUS');
define('T_MODULUS', 'PHPCS_T_MODULUS');
define('T_BITWISE_AND', 'PHPCS_T_BITWISE_AND');
define('T_BITWISE_OR', 'PHPCS_T_BITWISE_OR');
define('T_BITWISE_XOR', 'PHPCS_T_BITWISE_XOR');
define('T_ARRAY_HINT', 'PHPCS_T_ARRAY_HINT');
define('T_GREATER_THAN', 'PHPCS_T_GREATER_THAN');
define('T_LESS_THAN', 'PHPCS_T_LESS_THAN');
define('T_BOOLEAN_NOT', 'PHPCS_T_BOOLEAN_NOT');
define('T_SELF', 'PHPCS_T_SELF');
define('T_PARENT', 'PHPCS_T_PARENT');
define('T_DOUBLE_QUOTED_STRING', 'PHPCS_T_DOUBLE_QUOTED_STRING');
define('T_COMMA', 'PHPCS_T_COMMA');
define('T_HEREDOC', 'PHPCS_T_HEREDOC');
define('T_PROTOTYPE', 'PHPCS_T_PROTOTYPE');
define('T_THIS', 'PHPCS_T_THIS');
define('T_REGULAR_EXPRESSION', 'PHPCS_T_REGULAR_EXPRESSION');
define('T_PROPERTY', 'PHPCS_T_PROPERTY');
define('T_LABEL', 'PHPCS_T_LABEL');
define('T_OBJECT', 'PHPCS_T_OBJECT');
define('T_CLOSE_OBJECT', 'PHPCS_T_CLOSE_OBJECT');
define('T_COLOUR', 'PHPCS_T_COLOUR');
define('T_HASH', 'PHPCS_T_HASH');
define('T_URL', 'PHPCS_T_URL');
define('T_STYLE', 'PHPCS_T_STYLE');
define('T_ASPERAND', 'PHPCS_T_ASPERAND');
define('T_DOLLAR', 'PHPCS_T_DOLLAR');
define('T_TYPEOF', 'PHPCS_T_TYPEOF');
define('T_CLOSURE', 'PHPCS_T_CLOSURE');
define('T_ANON_CLASS', 'PHPCS_T_ANON_CLASS');
define('T_BACKTICK', 'PHPCS_T_BACKTICK');
define('T_START_NOWDOC', 'PHPCS_T_START_NOWDOC');
define('T_NOWDOC', 'PHPCS_T_NOWDOC');
define('T_END_NOWDOC', 'PHPCS_T_END_NOWDOC');
define('T_OPEN_SHORT_ARRAY', 'PHPCS_T_OPEN_SHORT_ARRAY');
define('T_CLOSE_SHORT_ARRAY', 'PHPCS_T_CLOSE_SHORT_ARRAY');
define('T_GOTO_LABEL', 'PHPCS_T_GOTO_LABEL');
define('T_BINARY_CAST', 'PHPCS_T_BINARY_CAST');
define('T_EMBEDDED_PHP', 'PHPCS_T_EMBEDDED_PHP');
define('T_RETURN_TYPE', 'PHPCS_T_RETURN_TYPE');
define('T_OPEN_USE_GROUP', 'PHPCS_T_OPEN_USE_GROUP');
define('T_CLOSE_USE_GROUP', 'PHPCS_T_CLOSE_USE_GROUP');
define('T_ZSR', 'PHPCS_T_ZSR');
define('T_ZSR_EQUAL', 'PHPCS_T_ZSR_EQUAL');

// Some PHP 5.3 tokens, replicated for lower versions.
if (defined('T_NAMESPACE') === false) {
    define('T_NAMESPACE', 'PHPCS_T_NAMESPACE');
}

if (defined('T_NS_C') === false) {
    define('T_NS_C', 'PHPCS_T_NS_C');
}

if (defined('T_NS_SEPARATOR') === false) {
    define('T_NS_SEPARATOR', 'PHPCS_T_NS_SEPARATOR');
}

if (defined('T_GOTO') === false) {
    define('T_GOTO', 'PHPCS_T_GOTO');
}

if (defined('T_DIR') === false) {
    define('T_DIR', 'PHPCS_T_DIR');
}

// Some PHP 5.4 tokens, replicated for lower versions.
if (defined('T_TRAIT') === false) {
    define('T_TRAIT', 'PHPCS_T_TRAIT');
}

if (defined('T_TRAIT_C') === false) {
    define('T_TRAIT_C', 'PHPCS_T_TRAIT_C');
}

if (defined('T_INSTEADOF') === false) {
    define('T_INSTEADOF', 'PHPCS_T_INSTEADOF');
}

if (defined('T_CALLABLE') === false) {
    define('T_CALLABLE', 'PHPCS_T_CALLABLE');
}

// Some PHP 5.5 tokens, replicated for lower versions.
if (defined('T_FINALLY') === false) {
    define('T_FINALLY', 'PHPCS_T_FINALLY');
}

if (defined('T_YIELD') === false) {
    define('T_YIELD', 'PHPCS_T_YIELD');
}

// Some PHP 5.6 tokens, replicated for lower versions.
if (defined('T_ELLIPSIS') === false) {
    define('T_ELLIPSIS', 'PHPCS_T_ELLIPSIS');
}

if (defined('T_POW') === false) {
    define('T_POW', 'PHPCS_T_POW');
}

if (defined('T_POW_EQUAL') === false) {
    define('T_POW_EQUAL', 'PHPCS_T_POW_EQUAL');<commit_msg>Removed tokens that were replicated for 5.3 and 5.4 as PHP 5.4 is the new min version<commit_after><?php
/**
 * Stores weightings and groupings of tokens.
 *
 * @author    <NAME> <<EMAIL>>
 * @copyright 2006-2015 Squiz Pty Ltd (ABN 77 084 670 600)
 * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
 */

namespace PHP_CodeSniffer\Util;

define('T_NONE', 'PHPCS_T_NONE');
define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
define('T_OPEN_SQUARE_BRACKET', 'PHPCS_T_OPEN_SQUARE_BRACKET');
define('T_CLOSE_SQUARE_BRACKET', 'PHPCS_T_CLOSE_SQUARE_BRACKET');
define('T_OPEN_PARENTHESIS', 'PHPCS_T_OPEN_PARENTHESIS');
define('T_CLOSE_PARENTHESIS', 'PHPCS_T_CLOSE_PARENTHESIS');
define('T_COLON', 'PHPCS_T_COLON');
define('T_STRING_CONCAT', 'PHPCS_T_STRING_CONCAT');
define('T_INLINE_THEN', 'PHPCS_T_INLINE_THEN');
define('T_INLINE_ELSE', 'PHPCS_T_INLINE_ELSE');
define('T_NULL', 'PHPCS_T_NULL');
define('T_FALSE', 'PHPCS_T_FALSE');
define('T_TRUE', 'PHPCS_T_TRUE');
define('T_SEMICOLON', 'PHPCS_T_SEMICOLON');
define('T_EQUAL', 'PHPCS_T_EQUAL');
define('T_MULTIPLY', 'PHPCS_T_MULTIPLY');
define('T_DIVIDE', 'PHPCS_T_DIVIDE');
define('T_PLUS', 'PHPCS_T_PLUS');
define('T_MINUS', 'PHPCS_T_MINUS');
define('T_MODULUS', 'PHPCS_T_MODULUS');
define('T_BITWISE_AND', 'PHPCS_T_BITWISE_AND');
define('T_BITWISE_OR', 'PHPCS_T_BITWISE_OR');
define('T_BITWISE_XOR', 'PHPCS_T_BITWISE_XOR');
define('T_ARRAY_HINT', 'PHPCS_T_ARRAY_HINT');
define('T_GREATER_THAN', 'PHPCS_T_GREATER_THAN');
define('T_LESS_THAN', 'PHPCS_T_LESS_THAN');
define('T_BOOLEAN_NOT', 'PHPCS_T_BOOLEAN_NOT');
define('T_SELF', 'PHPCS_T_SELF');
define('T_PARENT', 'PHPCS_T_PARENT');
define('T_DOUBLE_QUOTED_STRING', 'PHPCS_T_DOUBLE_QUOTED_STRING');
define('T_COMMA', 'PHPCS_T_COMMA');
define('T_HEREDOC', 'PHPCS_T_HEREDOC');
define('T_PROTOTYPE', 'PHPCS_T_PROTOTYPE');
define('T_THIS', 'PHPCS_T_THIS');
define('T_REGULAR_EXPRESSION', 'PHPCS_T_REGULAR_EXPRESSION');
define('T_PROPERTY', 'PHPCS_T_PROPERTY');
define('T_LABEL', 'PHPCS_T_LABEL');
define('T_OBJECT', 'PHPCS_T_OBJECT');
define('T_CLOSE_OBJECT', 'PHPCS_T_CLOSE_OBJECT');
define('T_COLOUR', 'PHPCS_T_COLOUR');
define('T_HASH', 'PHPCS_T_HASH');
define('T_URL', 'PHPCS_T_URL');
define('T_STYLE', 'PHPCS_T_STYLE');
define('T_ASPERAND', 'PHPCS_T_ASPERAND');
define('T_DOLLAR', 'PHPCS_T_DOLLAR');
define('T_TYPEOF', 'PHPCS_T_TYPEOF');
define('T_CLOSURE', 'PHPCS_T_CLOSURE');
define('T_ANON_CLASS', 'PHPCS_T_ANON_CLASS');
define('T_BACKTICK', 'PHPCS_T_BACKTICK');
define('T_START_NOWDOC', 'PHPCS_T_START_NOWDOC');
define('T_NOWDOC', 'PHPCS_T_NOWDOC');
define('T_END_NOWDOC', 'PHPCS_T_END_NOWDOC');
define('T_OPEN_SHORT_ARRAY', 'PHPCS_T_OPEN_SHORT_ARRAY');
define('T_CLOSE_SHORT_ARRAY', 'PHPCS_T_CLOSE_SHORT_ARRAY');
define('T_GOTO_LABEL', 'PHPCS_T_GOTO_LABEL');
define('T_BINARY_CAST', 'PHPCS_T_BINARY_CAST');
define('T_EMBEDDED_PHP', 'PHPCS_T_EMBEDDED_PHP');
define('T_RETURN_TYPE', 'PHPCS_T_RETURN_TYPE');
define('T_OPEN_USE_GROUP', 'PHPCS_T_OPEN_USE_GROUP');
define('T_CLOSE_USE_GROUP', 'PHPCS_T_CLOSE_USE_GROUP');
define('T_ZSR', 'PHPCS_T_ZSR');
define('T_ZSR_EQUAL', 'PHPCS_T_ZSR_EQUAL');

// Some PHP 5.5 tokens, replicated for lower versions.
if (defined('T_FINALLY') === false) {
    define('T_FINALLY', 'PHPCS_T_FINALLY');
}

if (defined('T_YIELD') === false) {
    define('T_YIELD', 'PHPCS_T_YIELD');
}

// Some PHP 5.6 tokens, replicated for lower versions.
if (defined('T_ELLIPSIS') === false) {
    define('T_ELLIPSIS', 'PHPCS_T_ELLIPSIS');
}

if (defined('T_POW') === false) {
    define('T_POW', 'PHPCS_T_POW');
}

if (defined('T_POW_EQUAL') === false) {
    define('T_POW_EQUAL', 'PHPCS_T_POW_EQUAL');
{'fraction_non_alphanumeric': 0.17898173902527512, 'fraction_numerical': 0.0067722820171725725, 'mean_word_length': 10.598877980364657, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3995042', 'n_tokens_mistral': 3902, 'n_tokens_neox': 3773, 'n_words': 462}