<commit_before><?php

require_once KALTURA_ROOT_PATH . '/vendor/htmlpurifier/library/HTMLPurifier.auto.php';

/**
 * @package infra
 * @subpackage utils
 */
class kHtmlPurifier
{
	const HTML_PURIFIER = 'html_purifier';
	const ALLOWED_TAGS = 'allowedTags';
	const ALLOWED_FRAME_TARGETS = 'allowedFrameTargets';
	const ALLOWED_REL = 'allowedRel';
	const HTML_DOCTYPE = "HTML 4.01 Transitional";
	
	private static $purifier = null;
	private static $AllowedProperties = null;
	private static $allowedTokenPatterns;

	public static function purify( $className, $propertyName, $value )
	{
		if ( ! is_string($value)								// Skip objects like KalturaNullField, for example
			|| self::isMarkupAllowed($className, $propertyName)	// Skip fields that are allowed to contain HTML/XML tags
		)
		{
			return $value;
		}

		$tokenMapper = new kRegExTokenMapper();
		$tokenizedValue = $tokenMapper->tokenize($value, self::$allowedTokenPatterns);
		$purifiedValue = self::$purifier->purify( $tokenizedValue );
		$modifiedValue = $tokenMapper->unTokenize($purifiedValue);

		if (kCurrentContext::$HTMLPurifierBehaviour == HTMLPurifierBehaviourType::SANITIZE)
			return $modifiedValue;
		
		$valueTrimmedSpace = preg_replace('/\s+/', '', $value);
		$modifiedValueTrimmedSpace = preg_replace('/\s+/', '', $modifiedValue);
		
		if ($modifiedValueTrimmedSpace != $valueTrimmedSpace)
		{
			$msg = "Potential Unsafe HTML tags found in $className::$propertyName"
					. "\nORIGINAL VALUE: [" . $value . "]"
					. "\nMODIFIED VALUE: [" . $modifiedValue . "]"
				;
			KalturaLog::err( $msg );

			if (kCurrentContext::$HTMLPurifierBehaviour == HTMLPurifierBehaviourType::NOTIFY)
			{
//			$this->notifyAboutHtmlPurification($className, $propertyName, $value);
				KalturaLog::debug("should send notification");
				return $value;
			}
			// If we reach here kCurrentContext::$HTMLPurifierBehaviour must be BLOCK

			$errorMessage = "UNSAFE_HTML_TAGS;Potential Unsafe HTML tags found in [$className]::[$propertyName]";
			throw new Exception($errorMessage);
		}

		return $value;
	}

	public static function isMarkupAllowed( $className, $propertyName )
	{
		// Is it an excluded property?
		if ( array_key_exists($className . ":" . $propertyName, self::$AllowedProperties) )
		{
			return true;
		}

		return false;
	}
	
	public static function init()
	{
		self::initHTMLPurifier();
		self::initAllowedProperties();
		self::initAllowedTokenPatterns();
	}
	
	public static function initHTMLPurifier()
	{
		$cacheKey = null;
		if ( function_exists('apc_fetch') && function_exists('apc_store') )
		{
			$cacheKey = 'kHtmlPurifierPurifier-' . kConf::getCachedVersionId();
			self::$purifier = apc_fetch($cacheKey);
		}
		
		if ( ! self::$purifier )
		{
			$config = HTMLPurifier_Config::createDefault();
			$config->set('Cache.DefinitionImpl', null);
			$config->set('HTML.Doctype', self::HTML_DOCTYPE);
			$htmlPurifierConf = kConf::get(self::HTML_PURIFIER, kConfMapNames::RUNTIME_CONFIG, array());
			if ($htmlPurifierConf)
			{
				if (isset($htmlPurifierConf[self::ALLOWED_TAGS]))
				{
					$config->set('HTML.Allowed', $htmlPurifierConf[self::ALLOWED_TAGS]);
				}
				if (isset($htmlPurifierConf[self::ALLOWED_FRAME_TARGETS]))
				{
					$config->set('Attr.AllowedFrameTargets', $htmlPurifierConf[self::ALLOWED_FRAME_TARGETS]);
				}
				if (isset($htmlPurifierConf[self::ALLOWED_REL]))
				{
					$config->set('Attr.AllowedRel', $htmlPurifierConf[self::ALLOWED_REL]);
				}
			}
			
			self::$purifier = new HTMLPurifier($config);
			
			if ( $cacheKey )
			{
				apc_store( $cacheKey, self::$purifier );
			}<commit_msg>SUP-30667 add html doctype config for html purifier<commit_after><?php

require_once KALTURA_ROOT_PATH . '/vendor/htmlpurifier/library/HTMLPurifier.auto.php';

/**
 * @package infra
 * @subpackage utils
 */
class kHtmlPurifier
{
	const HTML_PURIFIER = 'html_purifier';
	const ALLOWED_TAGS = 'allowedTags';
	const ALLOWED_FRAME_TARGETS = 'allowedFrameTargets';
	const ALLOWED_REL = 'allowedRel';
	const HTML_DOCTYPE = "HTMLDoctype";
	
	private static $purifier = null;
	private static $AllowedProperties = null;
	private static $allowedTokenPatterns;

	public static function purify( $className, $propertyName, $value )
	{
		if ( ! is_string($value)								// Skip objects like KalturaNullField, for example
			|| self::isMarkupAllowed($className, $propertyName)	// Skip fields that are allowed to contain HTML/XML tags
		)
		{
			return $value;
		}

		$tokenMapper = new kRegExTokenMapper();
		$tokenizedValue = $tokenMapper->tokenize($value, self::$allowedTokenPatterns);
		$purifiedValue = self::$purifier->purify( $tokenizedValue );
		$modifiedValue = $tokenMapper->unTokenize($purifiedValue);

		if (kCurrentContext::$HTMLPurifierBehaviour == HTMLPurifierBehaviourType::SANITIZE)
			return $modifiedValue;
		
		$valueTrimmedSpace = preg_replace('/\s+/', '', $value);
		$modifiedValueTrimmedSpace = preg_replace('/\s+/', '', $modifiedValue);
		
		if ($modifiedValueTrimmedSpace != $valueTrimmedSpace)
		{
			$msg = "Potential Unsafe HTML tags found in $className::$propertyName"
					. "\nORIGINAL VALUE: [" . $value . "]"
					. "\nMODIFIED VALUE: [" . $modifiedValue . "]"
				;
			KalturaLog::err( $msg );

			if (kCurrentContext::$HTMLPurifierBehaviour == HTMLPurifierBehaviourType::NOTIFY)
			{
//			$this->notifyAboutHtmlPurification($className, $propertyName, $value);
				KalturaLog::debug("should send notification");
				return $value;
			}
			// If we reach here kCurrentContext::$HTMLPurifierBehaviour must be BLOCK

			$errorMessage = "UNSAFE_HTML_TAGS;Potential Unsafe HTML tags found in [$className]::[$propertyName]";
			throw new Exception($errorMessage);
		}

		return $value;
	}

	public static function isMarkupAllowed( $className, $propertyName )
	{
		// Is it an excluded property?
		if ( array_key_exists($className . ":" . $propertyName, self::$AllowedProperties) )
		{
			return true;
		}

		return false;
	}
	
	public static function init()
	{
		self::initHTMLPurifier();
		self::initAllowedProperties();
		self::initAllowedTokenPatterns();
	}
	
	public static function initHTMLPurifier()
	{
		$cacheKey = null;
		if ( function_exists('apc_fetch') && function_exists('apc_store') )
		{
			$cacheKey = 'kHtmlPurifierPurifier-' . kConf::getCachedVersionId();
			self::$purifier = apc_fetch($cacheKey);
		}
		
		if ( ! self::$purifier )
		{
			$config = HTMLPurifier_Config::createDefault();
			$config->set('Cache.DefinitionImpl', null);
			$htmlPurifierConf = kConf::get(self::HTML_PURIFIER, kConfMapNames::RUNTIME_CONFIG, array());
			if ($htmlPurifierConf)
			{
				if (isset($htmlPurifierConf[self::HTML_DOCTYPE]))
				{
					$config->set('HTML.Doctype', $htmlPurifierConf[self::HTML_DOCTYPE]);
				}
				if (isset($htmlPurifierConf[self::ALLOWED_TAGS]))
				{
					$config->set('HTML.Allowed', $htmlPurifierConf[self::ALLOWED_TAGS]);
				}
				if (isset($htmlPurifierConf[self::ALLOWED_FRAME_TARGETS]))
				{
					$config->set('Attr.AllowedFrameTargets', $htmlPurifierConf[self::ALLOWED_FRAME_TARGETS]);
				}
				if (isset($htmlPurifierConf[self::ALLOWED_REL]))
				{
					$config->set('Attr.AllowedRel', $htmlPurifierConf[self::ALLOWED_REL]);
				}
			}
			
			self::$purifier = new HTMLPurifier($config);
			
			if ( $cacheKey )
			{
				apc_store( $cacheKey, self::$purifier );
			}
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.0010925976509150504, 'mean_word_length': 4.633076923076923, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2999878', 'n_tokens_mistral': 2831, 'n_tokens_neox': 2412, 'n_words': 433}