<commit_before><?php
/**
 * Copyright 2006 - 2011 <NAME> (http://ehough.com)
 *
 * This file is part of TubePress (http://tubepress.org)
 *
 * TubePress is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * TubePress is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with TubePress.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

class_exists('org_tubepress_impl_classloader_ClassLoader') || require dirname(__FILE__) . '/../classloader/ClassLoader.class.php';
org_tubepress_impl_classloader_ClassLoader::loadClasses(array(
    'org_tubepress_api_const_options_names_Advanced',
    'org_tubepress_api_ioc_IocService',
    'org_tubepress_api_exec_ExecutionContext',
    'org_tubepress_api_options_OptionValidator',
    'org_tubepress_api_shortcode_ShortcodeParser',
    'org_tubepress_impl_ioc_IocContainer',
    'org_tubepress_impl_log_Log',
));

/**
 * Parses shortcodes.
 */
class org_tubepress_impl_shortcode_SimpleShortcodeParser implements org_tubepress_api_shortcode_ShortcodeParser
{
    const LOG_PREFIX = 'Shortcode parser';

    /**
     * This function is used to parse a shortcode into options that TubePress can use.
     *
     * @param string $content The haystack in which to search
     *
     * @return array The associative array of parsed options.
     */
    public function parse($content)
    {
        try {
            $this->_wrappedParse($content);
        } catch (Exception $e) {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Caught exception when parsing shortcode: ' . $e->getMessage());
        }
    }

    private function _wrappedParse($content)
    {
        $ioc     = org_tubepress_impl_ioc_IocContainer::getInstance();
        $context = $ioc->get('org_tubepress_api_exec_ExecutionContext');

        /* what trigger word are we using? */
        $keyword = $context->get(org_tubepress_api_const_options_names_Advanced::KEYWORD);

        if (!$this->somethingToParse($content, $keyword)) {
            return;
        }

        $toReturn = array();

        /* Match everything in square brackets after the trigger */
        $regexp = "\[$keyword\b(.*)\]";

        org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Regular expression for content is %s', $regexp);

        preg_match("/$regexp/", $content, $matches);

        if (sizeof($matches) === 0) {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'No shortcodes detected in content');
            return;
        }

        org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Found a shortcode: %s', $matches[0]);

        $context->setActualShortcodeUsed($matches[0]);

        /* Anything matched? */
        if (isset($matches[1]) && $matches[1] != '') {

            $text    = preg_replace('/[\x{00a0}\x{200b}]+/u', ' ', $matches[1]);
            $text    = self::_convertQuotes($text);
            $pattern = '/(\w+)\s*=\s*"([^"]*)"(?:\s*,)?(?:\s|$)|(\w+)\s*=\s*\'([^\']*)\'(?:\s*,)?(?:\s|$)|(\w+)\s*=\s*([^\s\'"]+)(?:\s*,)?(?:\s|$)/';

            if ( preg_match_all($pattern, $text, $match, PREG_SET_ORDER) ) {

                org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Custom options detected in shortcode: %s', $matches[0]);

                $toReturn = self::_parseCustomOption($toReturn, $match, $ioc);

                $context->setCustomOptions($toReturn);
            }
        } else {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'No custom options detected in shortcode: %s', $matches[0]);
        }
    }

    /**
     * Determines if the given content contains a shortcode.
     *
     * @param string $content The content to search through
     * @param string $trigger The shortcode trigger word
     *
     * @return boolean True if there's a shortcode in the content, false otherwise.
     */
    public function somethingToParse($content, $trigger = "tubepress")
    {
        return preg_match("/\[$trigger\b(.*)\]/", $content) === 1;
    }

    /**
     * Handles the detection of a custom options
     *
     * @param array                            $customOptions The custom options array
     * @param array                            $match         The array shortcode matches
     * @param org_tubepress_api_ioc_IocService $ioc           The IOC service
     *
     * @return void
     */
    private static function _parseCustomOption($customOptions, $match, org_tubepress_api_ioc_IocService $ioc)
    {
        $inputValidationService = $ioc->get('org_tubepress_api_options_OptionValidator');

        foreach ($match as $m) {

            if (!empty($m[1])) {

                $name  = $m[1];
                $value = self::_normalizeValue($m[2]);
<commit_msg>Fixed regular expression mechanism to support multi-line shortcodes<commit_after><?php
/**
 * Copyright 2006 - 2011 <NAME> (http://ehough.com)
 *
 * This file is part of TubePress (http://tubepress.org)
 *
 * TubePress is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * TubePress is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with TubePress.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

class_exists('org_tubepress_impl_classloader_ClassLoader') || require dirname(__FILE__) . '/../classloader/ClassLoader.class.php';
org_tubepress_impl_classloader_ClassLoader::loadClasses(array(
    'org_tubepress_api_const_options_names_Advanced',
    'org_tubepress_api_ioc_IocService',
    'org_tubepress_api_exec_ExecutionContext',
    'org_tubepress_api_options_OptionValidator',
    'org_tubepress_api_shortcode_ShortcodeParser',
    'org_tubepress_impl_ioc_IocContainer',
    'org_tubepress_impl_log_Log',
));

/**
 * Parses shortcodes.
 */
class org_tubepress_impl_shortcode_SimpleShortcodeParser implements org_tubepress_api_shortcode_ShortcodeParser
{
    const LOG_PREFIX = 'Shortcode parser';

    /**
     * This function is used to parse a shortcode into options that TubePress can use.
     *
     * @param string $content The haystack in which to search
     *
     * @return array The associative array of parsed options.
     */
    public function parse($content)
    {
        try {
            $this->_wrappedParse($content);
        } catch (Exception $e) {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Caught exception when parsing shortcode: ' . $e->getMessage());
        }
    }

    private function _wrappedParse($content)
    {
        $ioc     = org_tubepress_impl_ioc_IocContainer::getInstance();
        $context = $ioc->get('org_tubepress_api_exec_ExecutionContext');
        
	/* what trigger word are we using? */
        $keyword = $context->get(org_tubepress_api_const_options_names_Advanced::KEYWORD);
        
	if (!$this->somethingToParse($content, $keyword)) {
            return;
        }

        $toReturn = array();

        preg_match("/\[$keyword\b(.*)\]/s", $content, $matches);

        if (sizeof($matches) === 0) {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'No shortcodes detected in content');
            return;
        }

        org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Found a shortcode: %s', $matches[0]);

        $context->setActualShortcodeUsed($matches[0]);

        /* Anything matched? */
        if (isset($matches[1]) && $matches[1] != '') {

            $text    = preg_replace('/[\x{00a0}\x{200b}]+/u', ' ', $matches[1]);
            $text    = self::_convertQuotes($text);
            $pattern = '/(\w+)\s*=\s*"([^"]*)"(?:\s*,)?(?:\s|$)|(\w+)\s*=\s*\'([^\']*)\'(?:\s*,)?(?:\s|$)|(\w+)\s*=\s*([^\s\'"]+)(?:\s*,)?(?:\s|$)/';

            if ( preg_match_all($pattern, $text, $match, PREG_SET_ORDER) ) {

                org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'Custom options detected in shortcode: %s', $matches[0]);

                $toReturn = self::_parseCustomOption($toReturn, $match, $ioc);

                $context->setCustomOptions($toReturn);
            }
        } else {
            org_tubepress_impl_log_Log::log(self::LOG_PREFIX, 'No custom options detected in shortcode: %s', $matches[0]);
        }
    }

    /**
     * Determines if the given content contains a shortcode.
     *
     * @param string $content The content to search through
     * @param string $trigger The shortcode trigger word
     *
     * @return boolean True if there's a shortcode in the content, false otherwise.
     */
    public function somethingToParse($content, $trigger = "tubepress")
    {
        return preg_match("/\[$trigger\b(.*)\]/s", $content) === 1;
    }

    /**
     * Handles the detection of a custom options
     *
     * @param array                            $customOptions The custom options array
     * @param array                            $match         The array shortcode matches
     * @param org_tubepress_api_ioc_IocService $ioc           The IOC service
     *
     * @return void
     */
    private static function _parseCustomOption($customOptions, $match, org_tubepress_api_ioc_IocService $ioc)
    {
        $inputValidationService = $ioc->get('org_tubepress_api_options_OptionValidator');

        foreach ($match as $m) {

            if (!empty($m[1])) {

                $name  = $m[1];
                $value = self::_normalizeValue($m[2]);

{'fraction_non_alphanumeric': 0.1314665872306623, 'fraction_numerical': 0.005361930294906166, 'mean_word_length': 2.8888030888030887, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6441828', 'n_tokens_mistral': 3138, 'n_tokens_neox': 3028, 'n_words': 725}