<commit_before><?php

/*
 * This file is part of the Behat.
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Behat\Behat\Definition\Pattern\Policy;

use Behat\Behat\Definition\Pattern\Pattern;
use Behat\Transliterator\Transliterator;

/**
 * Defines a way to handle turnip patterns.
 *
 * @author <NAME> <<EMAIL>>
 */
final class TurnipPatternPolicy implements PatternPolicy
{
    const TOKEN_REGEX = "[\"']?(?P<%s>(?<=\")[^\"]+(?=\")|(?<=')[^']+(?=')|(?<=\W)\w+(?=\W|$))['\"]?";

    const PLACEHOLDER_REGEXP = "/\\\:(\w+)/";
    const OPTIONAL_WORD_REGEXP = '/(\s)?\\\\\(([^\\\]+)\\\\\)(\s)?/';
    const ALTERNATIVE_WORD_REGEXP = '/(\w+)\\\\\/(\w+)/';

    /**
     * @var string[]
     */
    private static $placeholderPatterns = array(
        "/(?<=\W|^)\"[^\"]+\"(?=\W|$)/",
        "/(?<=\W|^)'[^']+'(?=\W|$)/",
        "/(?<=\W|^)\d+(?=\W|$)/"
    );

    /**
     * {@inheritdoc}
     */
    public function supportsPatternType($type)
    {
        return null === $type || 'turnip' === $type;
    }<commit_msg>Fix turnip definitions starting from token<commit_after><?php

/*
 * This file is part of the Behat.
 * (c) <NAME> <<EMAIL>>
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

namespace Behat\Behat\Definition\Pattern\Policy;

use Behat\Behat\Definition\Pattern\Pattern;
use Behat\Transliterator\Transliterator;

/**
 * Defines a way to handle turnip patterns.
 *
 * @author <NAME> <<EMAIL>>
 */
final class TurnipPatternPolicy implements PatternPolicy
{
    const TOKEN_REGEX = "[\"']?(?P<%s>(?<=\")[^\"]+(?=\")|(?<=')[^']+(?=')|(?<=\W|^)\w+(?=\W|$))['\"]?";

    const PLACEHOLDER_REGEXP = "/\\\:(\w+)/";
    const OPTIONAL_WORD_REGEXP = '/(\s)?\\\\\(([^\\\]+)\\\\\)(\s)?/';
    const ALTERNATIVE_WORD_REGEXP = '/(\w+)\\\\\/(\w+)/';

    /**
     * @var string[]
     */
    private static $placeholderPatterns = array(
        "/(?<=\W|^)\"[^\"]+\"(?=\W|$)/",
        "/(?<=\W|^)'[^']+'(?=\W|$)/",
        "/(?<=\W|^)\d+(?=\W|$)/"
    );

    /**
     * {@inheritdoc}
     */
    public function supportsPatternType($type)
    {
        return null === $type || 'turnip' === $type;
    }
{'fraction_non_alphanumeric': 0.27473002159827214, 'fraction_numerical': 0.0, 'mean_word_length': 2.8092105263157894, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3633159', 'n_tokens_mistral': 872, 'n_tokens_neox': 836, 'n_words': 167}