<commit_before><?php

namespace AmpProject\Dom\Document\Filter;

use AmpProject\Dom\Document\BeforeLoadFilter;
use AmpProject\Dom\Document\Option;
use AmpProject\Dom\Options;

/**
 * Handles the html entities present in the html and prevent them from double encoding.
 *
 * @package ampproject/amp-toolbox
 */
final class NormalizeHtmlEntities implements BeforeLoadFilter
{
    /**
     * Options instance to use.
     *
     * @var Options
     */
    private $options;

    /**
     * Whether to use this NormalizeHtmlEntities or not.
     *
     * Accepted values are 'auto', 'always' and 'never'.
     *
     * @var string
     */
    private $normalizeHtmlEntities;

    /**
     * NormalizeHtmlEntities constructor.
     *
     * @param Options $options Options instance to use.
     */
    public function __construct(Options $options)
    {
        $this->options = $options;

        switch ($options[Option::NORMALIZE_HTML_ENTITIES]) {
            case Option::NORMALIZE_HTML_ENTITIES_NEVER:
                $this->normalizeHtmlEntities = Option::NORMALIZE_HTML_ENTITIES_NEVER;
                break;
            case Option::NORMALIZE_HTML_ENTITIES_ALWAYS:
                $this->normalizeHtmlEntities = Option::NORMALIZE_HTML_ENTITIES_ALWAYS;
                break;
            case Option::NORMALIZE_HTML_ENTITIES_AUTO:
            default:
                $this->normalizeHtmlEntities = Option::NORMALIZE_HTML_ENTITIES_AUTO;
                break;
        }
    }

    /**
     * Preprocess the HTML to be loaded into the Dom\Document.
     *
     * @param string $html String of HTML markup to be preprocessed.
     * @return string Preprocessed string of HTML markup.
     */
    public function beforeLoad($html)
    {
        if (
            ($this->normalizeHtmlEntities === Option::NORMALIZE_HTML_ENTITIES_NEVER)
            || (
                ($this->normalizeHtmlEntities === Option::NORMALIZE_HTML_ENTITIES_AUTO)
                && ! $this->hasHtmlEntities($html)
            )
        ) {
            return $html;
        }

        return html_entity_decode(
            $html,
            $this->options[Option::NORMALIZE_HTML_ENTITIES_FLAGS],
            $this->options[Option::ENCODING]
        );
    }

    /**
     * Detect the presence of html entities in the html.
     *
     * @param string $html The html in which this method will to detect the entities.
     * @return bool Whether the html contains entities or not.
     */
    protected function hasHtmlEntities($html)
    {
        // TODO: Discuss other popular entities to look for, especially for languages
        // with different punctuation symbols.
        return preg_match('/&comma;|&period;|&excl;|&quest;/', $html);
    }
}
<commit_msg>Throw InvalidOptionValue exception for invalid option<commit_after><?php

namespace AmpProject\Dom\Document\Filter;

use AmpProject\Dom\Document\BeforeLoadFilter;
use AmpProject\Dom\Document\Option;
use AmpProject\Dom\Options;

/**
 * Handles the html entities present in the html and prevent them from double encoding.
 *
 * @package ampproject/amp-toolbox
 */
final class NormalizeHtmlEntities implements BeforeLoadFilter
{
    /**
     * Options instance to use.
     *
     * @var Options
     */
    private $options;

    /**
     * Whether to use this NormalizeHtmlEntities or not.
     *
     * Accepted values are 'auto', 'always' and 'never'.
     *
     * @var string
     */
    private $normalizeHtmlEntities;

    /**
     * NormalizeHtmlEntities constructor.
     *
     * @param Options $options Options instance to use.
     */
    public function __construct(Options $options)
    {
        $this->options = $options;

		$this->normalizeHtmlEntities = $options[Option::NORMALIZE_HTML_ENTITIES];
		if (! in_array($this->normalizeHtmlEntities, self::VALID_NORMALIZE_OPTION_VALUES, true)) {
			throw InvalidOptionValue::forValue($this->normalizeHtmlEntities, self::VALID_NORMALIZE_OPTION_VALUES);
		}
    }

    /**
     * Preprocess the HTML to be loaded into the Dom\Document.
     *
     * @param string $html String of HTML markup to be preprocessed.
     * @return string Preprocessed string of HTML markup.
     */
    public function beforeLoad($html)
    {
        if (
            ($this->normalizeHtmlEntities === Option::NORMALIZE_HTML_ENTITIES_NEVER)
            || (
                ($this->normalizeHtmlEntities === Option::NORMALIZE_HTML_ENTITIES_AUTO)
                && ! $this->hasHtmlEntities($html)
            )
        ) {
            return $html;
        }

        return html_entity_decode(
            $html,
            $this->options[Option::NORMALIZE_HTML_ENTITIES_FLAGS],
            $this->options[Option::ENCODING]
        );
    }

    /**
     * Detect the presence of html entities in the html.
     *
     * @param string $html The html in which this method will to detect the entities.
     * @return bool Whether the html contains entities or not.
     */
    protected function hasHtmlEntities($html)
    {
        // TODO: Discuss other popular entities to look for, especially for languages
        // with different punctuation symbols.
        return preg_match('/&comma;|&period;|&excl;|&quest;/', $html);
    }
}

{'fraction_non_alphanumeric': 0.09854294478527607, 'fraction_numerical': 0.0, 'mean_word_length': 2.3986970684039086, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4434645', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1465, 'n_words': 364}