<commit_before><?php
if (version_compare(PHP_VERSION, FRESHRSS_MIN_PHP_VERSION, '<')) {
	die(sprintf('FreshRSS error: FreshRSS requires PHP %s+!', FRESHRSS_MIN_PHP_VERSION));
}

if (!function_exists('mb_strcut')) {
	function mb_strcut($str, $start, $length = null, $encoding = 'UTF-8') {
		return substr($str, $start, $length);
	}
}

if (!function_exists('str_starts_with')) {
	/** Polyfill for PHP <8.0 */
	function str_starts_with(string $haystack, string $needle): bool {
		return strncmp($haystack, $needle, strlen($needle)) === 0;
	}
}

// @phpstan-ignore-next-line
if (COPY_SYSLOG_TO_STDERR) {
	openlog('FreshRSS', LOG_CONS | LOG_ODELAY | LOG_PID | LOG_PERROR, LOG_USER);
} else {
	openlog('FreshRSS', LOG_CONS | LOG_ODELAY | LOG_PID, LOG_USER);
}

/**
 * Build a directory path by concatenating a list of directory names.
 *
 * @param string ...$path_parts a list of directory names
 * @return string corresponding to the final pathname
 */
function join_path(...$path_parts): string {
	return join(DIRECTORY_SEPARATOR, $path_parts);
}

//<Auto-loading>
function classAutoloader($class) {
	if (strpos($class, 'FreshRSS') === 0) {
		$components = explode('_', $class);
		switch (count($components)) {
			case 1:
				include(APP_PATH . '/' . $components[0] . '.php');
				return;
			case 2:
				include(APP_PATH . '/Models/' . $components[1] . '.php');
				return;
			case 3:	//Controllers, Exceptions
				include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php');
				return;
		}
	} elseif (strpos($class, 'Minz') === 0) {
		include(LIB_PATH . '/' . str_replace('_', '/', $class) . '.php');
	} elseif (strpos($class, 'SimplePie') === 0) {
		include(LIB_PATH . '/SimplePie/' . str_replace('_', '/', $class) . '.php');
	} elseif (str_starts_with($class, 'Gt\\CssXPath\\')) {
		$prefix = 'Gt\\CssXPath\\';
		$base_dir = LIB_PATH . '/phpgt/cssxpath/src/';
		$relative_class_name = substr($class, strlen($prefix));
		require $base_dir . str_replace('\\', '/', $relative_class_name) . '.php';
	} elseif (str_starts_with($class, 'PHPMailer\\PHPMailer\\')) {
		$prefix = 'PHPMailer\\PHPMailer\\';
		$base_dir = LIB_PATH . '/phpmailer/phpmailer/src/';
		$relative_class_name = substr($class, strlen($prefix));
		require $base_dir . str_replace('\\', '/', $relative_class_name) . '.php';
	}
}

spl_autoload_register('classAutoloader');
//</Auto-loading>

/**
 * @param string $url
 * @return string
 */
function idn_to_puny($url) {
	if (function_exists('idn_to_ascii')) {
		$idn = parse_url($url, PHP_URL_HOST);
		if ($idn != '') {
			// https://wiki.php.net/rfc/deprecate-and-remove-intl_idna_variant_2003
			if (defined('INTL_IDNA_VARIANT_UTS46')) {
				$puny = idn_to_ascii($idn, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
			} elseif (defined('INTL_IDNA_VARIANT_2003')) {
				$puny = idn_to_ascii($idn, IDNA_DEFAULT, INTL_IDNA_VARIANT_2003);
			} else {
				$puny = idn_to_ascii($idn);
			}
			$pos = strpos($url, $idn);
			if ($puny != '' && $pos !== false) {
				$url = substr_replace($url, $puny, $pos, strlen($idn));
			}
		}
	}
	return $url;
}

/**
 * @param string $url
 * @param bool $fixScheme
 * @return string|false
 */
function checkUrl($url, $fixScheme = true) {
	$url = trim($url);
	if ($url == '') {
		return '';
	}
	if ($fixScheme && !preg_match('#^https?://#i', $url)) {
		$url = 'https://' . ltrim($url, '/');
	}

	$url = idn_to_puny($url);	//PHP bug #53474 IDN
	$urlRelaxed = str_replace('_', 'z', $url);	//PHP discussion #64948 Underscore

	if (filter_var($urlRelaxed, FILTER_VALIDATE_URL)) {
		return $url;
	} else {
		return false;
	}
}

/**
 * @param string $text
 * @return string
 */
function safe_ascii($text) {
	return filter_var($text, FILTER_DEFAULT, FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH);
}

if (function_exists('mb_convert_encoding')) {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return mb_convert_encoding($text, 'UTF-8', 'UTF-8');
	}
} elseif (function_exists('iconv')) {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return iconv('UTF-8', 'UTF-8//IGNORE', $text);
	}
} else {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return $text;
	}
}

/**
 * @param string $text
 * @param bool $extended
 * @return string
 */
function escapeToUnicodeAlternative($text, $extended = true) {
	$text = htmlspecialchars_decode($text, ENT_QUOTES);

	//Problematic characters
	$problem = array('&', '<', '>');
	//Use their fullwidth Unicode form instead:
	$replace = array('＆', '＜', '＞');

	// https://raw.githubusercontent.com/mihaip/google-reader-api/master/wiki/StreamId.wiki
	if ($extended) {
		$problem += array("'", '"', '^', '?', '\\', '/', ',', ';');
		$replace += array("’", '＂', '＾', '？', '＼', '／', '，', '；');
	}

	return trim(str_replace($problem, $replace, $text));
}

function format_number($n, $precision = 0) {
	// number_format does not seem to be Unicode-compatible
	return str_replace(' ', ' ',	// Thin non-breaking space
		number_format($n, $precision, '.', ' ')
	);
}

function format_bytes($bytes, $precision = 2, $system = 'IEC') {
	if ($system === 'IEC') {
		$base = 1024;
		$units = array('B', 'KiB', 'MiB', 'GiB', 'TiB');
	} elseif ($system === 'SI') {
		$base = 1000;
		$units = array('B', 'KB', 'MB', 'GB', 'TB');
	} else {
		return format_number($bytes, $precision);
	}
	$bytes = max(intval($bytes), 0);
	$pow = $bytes === 0 ? 0 : floor(log($bytes) / log($base));
	$pow = min($pow, count($units) - 1);
	$bytes /= pow($base, $pow);
	return format_number($bytes, $precision) . ' ' . $units[$pow];
}

function timestamptodate ($t, $hour = true) {
	$month = _t('gen.date.' . date('M', $t));
	if ($hour) {
		$date = _t('gen.date.format_date_hour', $month);
	} else {
		$date = _t('gen.date.format_date', $month);
	}

	return @date ($date, $t);
}

/**
 * Decode HTML entities but preserve XML entities.
 * @param string|null $text
 */
function html_only_entity_decode($text): string {
	static $htmlEntitiesOnly = null;
	if ($htmlEntitiesOnly === null) {
		$htmlEntitiesOnly = array_flip(array_diff(
			get_html_translation_table(HTML_ENTITIES, ENT_NOQUOTES, 'UTF-8'),	//Decode HTML entities
			get_html_translation_table(HTML_SPECIALCHARS, ENT_NOQUOTES, 'UTF-8')	//Preserve XML entities
		));
	}
	return $text == '' ? '' : strtr($text, $htmlEntitiesOnly);
}

/**
 * @param array<string,mixed> $attributes
 */
function customSimplePie($attributes = array()): SimplePie {
	$limits = FreshRSS_Context::$system_conf->limits;
	$simplePie = new SimplePie();
	$simplePie->set_useragent(FRESHRSS_USERAGENT);
	$simplePie->set_syslog(FreshRSS_Context::$system_conf->simplepie_syslog_enabled);
	$simplePie->set_cache_name_function('sha1');
	$simplePie->set_cache_location(CACHE_PATH);
	$simplePie->set_cache_duration($limits['cache_duration']);
	$simplePie->enable_order_by_date(false);

	$feed_timeout = empty($attributes['timeout']) ? 0 : intval($attributes['timeout']);
	$simplePie->set_timeout($feed_timeout > 0 ? $feed_timeout : $limits['timeout']);

	$curl_options = FreshRSS_Context::$system_conf->curl_options;
	if (isset($attributes['ssl_verify'])) {
		$curl_options[CURLOPT_SSL_VERIFYHOST] = $attributes['ssl_verify'] ? 2 : 0;
		$curl_options[CURLOPT_SSL_VERIFYPEER] = $attributes['ssl_verify'] ? true : false;
		if (!$attributes['ssl_verify']) {
			$curl_options[CURLOPT_SSL_CIPHER_LIST] = 'DEFAULT@SECLEVEL=1';
		}
	}
	if (!empty($attributes['curl_params']) && is_array($attributes['curl_params'])) {
		foreach ($attributes['curl_params'] as $co => $v) {
			$curl_options[$co] = $v;
		}
	}
	$simplePie->set_curl_options($curl_options);

	$simplePie->strip_comments(true);
	$simplePie->strip_htmltags(array(
		'base', 'blink', 'body', 'doctype', 'embed',
		'font', 'form', 'frame', 'frameset', 'html',
		'link', 'input', 'marquee', 'meta', 'noscript',
		'object', 'param', 'plaintext', 'script', 'style',
		'svg',	//TODO: Support SVG after sanitizing and URL rewriting of xlink:href
	));
	$simplePie->rename_attributes(array('id', 'class'));
	$simplePie->strip_attributes(array_merge($simplePie->strip_attributes, array(
		'autoplay', 'class', 'onload', 'onunload', 'onclick', 'ondblclick', 'onmousedown', 'onmouseup',
		'onmouseover', 'onmousemove', 'onmouseout', 'onfocus', 'onblur',
		'onkeypress', 'onkeydown', 'onkeyup', 'onselect', 'onchange', 'seamless', 'sizes', 'srcset')));
	$simplePie->add_attributes(array(
		'audio' => array('controls' => 'controls', 'preload' => 'none'),
		'iframe' => array('sandbox' => 'allow-scripts allow-same-origin'),
		'video' => array('controls' => 'controls', 'preload' => 'none'),
	));
	$simplePie->set_url_replacements(array(
		'a' => 'href',
		'area' => 'href',
		'audio' => 'src',
		'blockquote' => 'cite',
		'del' => 'cite',
		'form' => 'action',
		'iframe' => 'src',
		'img' => array(
			'longdesc',
			'src'
		),
		'input' => 'src',
		'ins' => 'cite',
		'q' => 'cite',
		'source' => 'src',
		'track' => 'src',
		'video' => array(
			'poster',
			'src',
		),
	));
	$https_domains = array();
	$force = @file(FRESHRSS_PATH . '/force-https.default.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
	if (is_array($force)) {
		$https_domains = array_merge($https_domains, $force);
	}
	$force = @file(DATA_PATH . '/force-https.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
	if (is_array($force)) {
		$https_domains = array_merge($https_domains, $force);
	}
	$simplePie->set_https_domains($https_domains);
	return $simplePie;
}

/**
 * @param int|false $maxLength
 */
function sanitizeHTML($data, string $base = '', $maxLength = false) {
	if (!is_string($data) || ($maxLength !== false && $maxLength <= 0)) {
		return '';
	}
	if ($maxLength !== false) {
		$data = mb_strcut($data, 0, $maxLength, 'UTF-8');
	}
	static $simplePie = null;
	if ($simplePie == null) {
		$simplePie = customSimplePie();
		$simplePie->init();
	}
	$result = html_only_entity_decode($simplePie->sanitize->sanitize($data, SIMPLEPIE_CONSTRUCT_HTML, $base));
	if ($maxLength !== false && strlen($result) > $maxLength) {
		//Sanitizing has made the result too long so try again shorter
		$data = mb_strcut($result, 0, (2 * $maxLength) - strlen($result) - 2, 'UTF-8');
		return sanitizeHTML($data, $base, $maxLength);
	}
	return $result;
}

function cleanCache(int $hours = 720) {
	$files = glob(CACHE_PATH . '/*.{html,spc}', GLOB_BRACE | GLOB_NOSORT);
	foreach ($files as $file) {
		if (substr($file, -10) === 'index.html') {
			continue;
		}
		$cacheMtime = @filemtime($file);
		if ($cacheMtime !== false && $cacheMtime < time() - (3600 * $hours)) {
			unlink($file);
		}
	}
}

/**
 * Set an XML preamble to enforce the HTML content type charset received by HTTP.
 * @param string $html the row downloaded HTML content
 * @param string $contentType an HTTP Content-Type such as 'text/html; charset=utf-8'
 * @return string an HTML string with XML encoding information for DOMDocument::loadHTML()
 */
function enforceHttpEncoding(string $html, string $contentType = ''): string {
	$httpCharset = preg_match('/\bcharset=([0-9a-z_-]{2,12})$/i', $contentType, $matches) === 1 ? $matches[1] : '';
	if ($httpCharset == '') {
		// No charset defined by HTTP, do nothing
		return $html;
	}
	$httpCharsetNormalized = SimplePie_Misc::encoding($httpCharset);
	if ($httpCharsetNormalized === 'windows-1252') {<commit_msg>GLOB_BRACE is not available on all platforms (#4628)<commit_after><?php
if (version_compare(PHP_VERSION, FRESHRSS_MIN_PHP_VERSION, '<')) {
	die(sprintf('FreshRSS error: FreshRSS requires PHP %s+!', FRESHRSS_MIN_PHP_VERSION));
}

if (!function_exists('mb_strcut')) {
	function mb_strcut($str, $start, $length = null, $encoding = 'UTF-8') {
		return substr($str, $start, $length);
	}
}

if (!function_exists('str_starts_with')) {
	/** Polyfill for PHP <8.0 */
	function str_starts_with(string $haystack, string $needle): bool {
		return strncmp($haystack, $needle, strlen($needle)) === 0;
	}
}

// @phpstan-ignore-next-line
if (COPY_SYSLOG_TO_STDERR) {
	openlog('FreshRSS', LOG_CONS | LOG_ODELAY | LOG_PID | LOG_PERROR, LOG_USER);
} else {
	openlog('FreshRSS', LOG_CONS | LOG_ODELAY | LOG_PID, LOG_USER);
}

/**
 * Build a directory path by concatenating a list of directory names.
 *
 * @param string ...$path_parts a list of directory names
 * @return string corresponding to the final pathname
 */
function join_path(...$path_parts): string {
	return join(DIRECTORY_SEPARATOR, $path_parts);
}

//<Auto-loading>
function classAutoloader($class) {
	if (strpos($class, 'FreshRSS') === 0) {
		$components = explode('_', $class);
		switch (count($components)) {
			case 1:
				include(APP_PATH . '/' . $components[0] . '.php');
				return;
			case 2:
				include(APP_PATH . '/Models/' . $components[1] . '.php');
				return;
			case 3:	//Controllers, Exceptions
				include(APP_PATH . '/' . $components[2] . 's/' . $components[1] . $components[2] . '.php');
				return;
		}
	} elseif (strpos($class, 'Minz') === 0) {
		include(LIB_PATH . '/' . str_replace('_', '/', $class) . '.php');
	} elseif (strpos($class, 'SimplePie') === 0) {
		include(LIB_PATH . '/SimplePie/' . str_replace('_', '/', $class) . '.php');
	} elseif (str_starts_with($class, 'Gt\\CssXPath\\')) {
		$prefix = 'Gt\\CssXPath\\';
		$base_dir = LIB_PATH . '/phpgt/cssxpath/src/';
		$relative_class_name = substr($class, strlen($prefix));
		require $base_dir . str_replace('\\', '/', $relative_class_name) . '.php';
	} elseif (str_starts_with($class, 'PHPMailer\\PHPMailer\\')) {
		$prefix = 'PHPMailer\\PHPMailer\\';
		$base_dir = LIB_PATH . '/phpmailer/phpmailer/src/';
		$relative_class_name = substr($class, strlen($prefix));
		require $base_dir . str_replace('\\', '/', $relative_class_name) . '.php';
	}
}

spl_autoload_register('classAutoloader');
//</Auto-loading>

/**
 * @param string $url
 * @return string
 */
function idn_to_puny($url) {
	if (function_exists('idn_to_ascii')) {
		$idn = parse_url($url, PHP_URL_HOST);
		if ($idn != '') {
			// https://wiki.php.net/rfc/deprecate-and-remove-intl_idna_variant_2003
			if (defined('INTL_IDNA_VARIANT_UTS46')) {
				$puny = idn_to_ascii($idn, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
			} elseif (defined('INTL_IDNA_VARIANT_2003')) {
				$puny = idn_to_ascii($idn, IDNA_DEFAULT, INTL_IDNA_VARIANT_2003);
			} else {
				$puny = idn_to_ascii($idn);
			}
			$pos = strpos($url, $idn);
			if ($puny != '' && $pos !== false) {
				$url = substr_replace($url, $puny, $pos, strlen($idn));
			}
		}
	}
	return $url;
}

/**
 * @param string $url
 * @param bool $fixScheme
 * @return string|false
 */
function checkUrl($url, $fixScheme = true) {
	$url = trim($url);
	if ($url == '') {
		return '';
	}
	if ($fixScheme && !preg_match('#^https?://#i', $url)) {
		$url = 'https://' . ltrim($url, '/');
	}

	$url = idn_to_puny($url);	//PHP bug #53474 IDN
	$urlRelaxed = str_replace('_', 'z', $url);	//PHP discussion #64948 Underscore

	if (filter_var($urlRelaxed, FILTER_VALIDATE_URL)) {
		return $url;
	} else {
		return false;
	}
}

/**
 * @param string $text
 * @return string
 */
function safe_ascii($text) {
	return filter_var($text, FILTER_DEFAULT, FILTER_FLAG_STRIP_LOW | FILTER_FLAG_STRIP_HIGH);
}

if (function_exists('mb_convert_encoding')) {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return mb_convert_encoding($text, 'UTF-8', 'UTF-8');
	}
} elseif (function_exists('iconv')) {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return iconv('UTF-8', 'UTF-8//IGNORE', $text);
	}
} else {
	/**
	 * @param string $text
	 * @return string
	 */
	function safe_utf8($text) {
		return $text;
	}
}

/**
 * @param string $text
 * @param bool $extended
 * @return string
 */
function escapeToUnicodeAlternative($text, $extended = true) {
	$text = htmlspecialchars_decode($text, ENT_QUOTES);

	//Problematic characters
	$problem = array('&', '<', '>');
	//Use their fullwidth Unicode form instead:
	$replace = array('＆', '＜', '＞');

	// https://raw.githubusercontent.com/mihaip/google-reader-api/master/wiki/StreamId.wiki
	if ($extended) {
		$problem += array("'", '"', '^', '?', '\\', '/', ',', ';');
		$replace += array("’", '＂', '＾', '？', '＼', '／', '，', '；');
	}

	return trim(str_replace($problem, $replace, $text));
}

function format_number($n, $precision = 0) {
	// number_format does not seem to be Unicode-compatible
	return str_replace(' ', ' ',	// Thin non-breaking space
		number_format($n, $precision, '.', ' ')
	);
}

function format_bytes($bytes, $precision = 2, $system = 'IEC') {
	if ($system === 'IEC') {
		$base = 1024;
		$units = array('B', 'KiB', 'MiB', 'GiB', 'TiB');
	} elseif ($system === 'SI') {
		$base = 1000;
		$units = array('B', 'KB', 'MB', 'GB', 'TB');
	} else {
		return format_number($bytes, $precision);
	}
	$bytes = max(intval($bytes), 0);
	$pow = $bytes === 0 ? 0 : floor(log($bytes) / log($base));
	$pow = min($pow, count($units) - 1);
	$bytes /= pow($base, $pow);
	return format_number($bytes, $precision) . ' ' . $units[$pow];
}

function timestamptodate ($t, $hour = true) {
	$month = _t('gen.date.' . date('M', $t));
	if ($hour) {
		$date = _t('gen.date.format_date_hour', $month);
	} else {
		$date = _t('gen.date.format_date', $month);
	}

	return @date ($date, $t);
}

/**
 * Decode HTML entities but preserve XML entities.
 * @param string|null $text
 */
function html_only_entity_decode($text): string {
	static $htmlEntitiesOnly = null;
	if ($htmlEntitiesOnly === null) {
		$htmlEntitiesOnly = array_flip(array_diff(
			get_html_translation_table(HTML_ENTITIES, ENT_NOQUOTES, 'UTF-8'),	//Decode HTML entities
			get_html_translation_table(HTML_SPECIALCHARS, ENT_NOQUOTES, 'UTF-8')	//Preserve XML entities
		));
	}
	return $text == '' ? '' : strtr($text, $htmlEntitiesOnly);
}

/**
 * @param array<string,mixed> $attributes
 */
function customSimplePie($attributes = array()): SimplePie {
	$limits = FreshRSS_Context::$system_conf->limits;
	$simplePie = new SimplePie();
	$simplePie->set_useragent(FRESHRSS_USERAGENT);
	$simplePie->set_syslog(FreshRSS_Context::$system_conf->simplepie_syslog_enabled);
	$simplePie->set_cache_name_function('sha1');
	$simplePie->set_cache_location(CACHE_PATH);
	$simplePie->set_cache_duration($limits['cache_duration']);
	$simplePie->enable_order_by_date(false);

	$feed_timeout = empty($attributes['timeout']) ? 0 : intval($attributes['timeout']);
	$simplePie->set_timeout($feed_timeout > 0 ? $feed_timeout : $limits['timeout']);

	$curl_options = FreshRSS_Context::$system_conf->curl_options;
	if (isset($attributes['ssl_verify'])) {
		$curl_options[CURLOPT_SSL_VERIFYHOST] = $attributes['ssl_verify'] ? 2 : 0;
		$curl_options[CURLOPT_SSL_VERIFYPEER] = $attributes['ssl_verify'] ? true : false;
		if (!$attributes['ssl_verify']) {
			$curl_options[CURLOPT_SSL_CIPHER_LIST] = 'DEFAULT@SECLEVEL=1';
		}
	}
	if (!empty($attributes['curl_params']) && is_array($attributes['curl_params'])) {
		foreach ($attributes['curl_params'] as $co => $v) {
			$curl_options[$co] = $v;
		}
	}
	$simplePie->set_curl_options($curl_options);

	$simplePie->strip_comments(true);
	$simplePie->strip_htmltags(array(
		'base', 'blink', 'body', 'doctype', 'embed',
		'font', 'form', 'frame', 'frameset', 'html',
		'link', 'input', 'marquee', 'meta', 'noscript',
		'object', 'param', 'plaintext', 'script', 'style',
		'svg',	//TODO: Support SVG after sanitizing and URL rewriting of xlink:href
	));
	$simplePie->rename_attributes(array('id', 'class'));
	$simplePie->strip_attributes(array_merge($simplePie->strip_attributes, array(
		'autoplay', 'class', 'onload', 'onunload', 'onclick', 'ondblclick', 'onmousedown', 'onmouseup',
		'onmouseover', 'onmousemove', 'onmouseout', 'onfocus', 'onblur',
		'onkeypress', 'onkeydown', 'onkeyup', 'onselect', 'onchange', 'seamless', 'sizes', 'srcset')));
	$simplePie->add_attributes(array(
		'audio' => array('controls' => 'controls', 'preload' => 'none'),
		'iframe' => array('sandbox' => 'allow-scripts allow-same-origin'),
		'video' => array('controls' => 'controls', 'preload' => 'none'),
	));
	$simplePie->set_url_replacements(array(
		'a' => 'href',
		'area' => 'href',
		'audio' => 'src',
		'blockquote' => 'cite',
		'del' => 'cite',
		'form' => 'action',
		'iframe' => 'src',
		'img' => array(
			'longdesc',
			'src'
		),
		'input' => 'src',
		'ins' => 'cite',
		'q' => 'cite',
		'source' => 'src',
		'track' => 'src',
		'video' => array(
			'poster',
			'src',
		),
	));
	$https_domains = array();
	$force = @file(FRESHRSS_PATH . '/force-https.default.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
	if (is_array($force)) {
		$https_domains = array_merge($https_domains, $force);
	}
	$force = @file(DATA_PATH . '/force-https.txt', FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
	if (is_array($force)) {
		$https_domains = array_merge($https_domains, $force);
	}
	$simplePie->set_https_domains($https_domains);
	return $simplePie;
}

/**
 * @param int|false $maxLength
 */
function sanitizeHTML($data, string $base = '', $maxLength = false) {
	if (!is_string($data) || ($maxLength !== false && $maxLength <= 0)) {
		return '';
	}
	if ($maxLength !== false) {
		$data = mb_strcut($data, 0, $maxLength, 'UTF-8');
	}
	static $simplePie = null;
	if ($simplePie == null) {
		$simplePie = customSimplePie();
		$simplePie->init();
	}
	$result = html_only_entity_decode($simplePie->sanitize->sanitize($data, SIMPLEPIE_CONSTRUCT_HTML, $base));
	if ($maxLength !== false && strlen($result) > $maxLength) {
		//Sanitizing has made the result too long so try again shorter
		$data = mb_strcut($result, 0, (2 * $maxLength) - strlen($result) - 2, 'UTF-8');
		return sanitizeHTML($data, $base, $maxLength);
	}
	return $result;
}

function cleanCache(int $hours = 720) {
	// N.B.: GLOB_BRACE is not available on all platforms
	$files = array_merge(glob(CACHE_PATH . '/*.html', GLOB_NOSORT), glob(CACHE_PATH . '/*.spc', GLOB_NOSORT));
	foreach ($files as $file) {
		if (substr($file, -10) === 'index.html') {
			continue;
		}
		$cacheMtime = @filemtime($file);
		if ($cacheMtime !== false && $cacheMtime < time() - (3600 * $hours)) {
			unlink($file);
		}
	}
}

/**
 * Set an XML preamble to enforce the HTML content type charset received by HTTP.
 * @param string $html the row downloaded HTML content
 * @param string $contentType an HTTP Content-Type such as 'text/html; charset=utf-8'
 * @return string an HTML string with XML encoding information for DOMDocument::loadHTML()
 */
function enforceHttpEncoding(string $html, string $contentType = ''): string {
	$httpCharset = preg_match('/\bcharset=([0-9a-z_-]{2,12})$/i', $contentType, $matches) === 1 ? $matches[1] : '';
	if ($httpCharset == '') {
		// No charset defined by HTTP, do nothing
		return $html;
	}
	$httpCharsetNormalized = SimplePie_Misc::encoding($httpCharset);
	if ($httpCharsetNormalized === 'windows-1252') {
{'fraction_non_alphanumeric': 0.20156263716969539, 'fraction_numerical': 0.008778860503906593, 'mean_word_length': 4.700025018764073, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 105, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2433311', 'n_tokens_mistral': 9165, 'n_tokens_neox': 8550, 'n_words': 1677}