<commit_before><?php

/**
 * --------------------------------------------------------------------------
 * Helper: squantoCleanupString
 * --------------------------------------------------------------------------
 *
 * Takes an input and cleans up a regular string from unwanted input
 *
 * @param 	string 	$value
 * @return 	string
 */
if(!function_exists('squantoCleanupString'))
{
    function squantoCleanupString( $value )
    {
        $value = strip_tags($value);

        return trim($value);
    }
}

/**
 * --------------------------------------------------------------------------
 * Helper: squantoCleanupHTML
 * --------------------------------------------------------------------------
 *
 * Takes an input and cleans up unwanted / malicious HTML
 *
 * @param 	string 	$value
 * @param 	string 	$whitelist - if false no tagstripping will occur - other than htmLawed
 * @return 	string
 */
if(!function_exists('squantoCleanupHTML'))
{
    function squantoCleanupHTML( $value, $whitelist = null )
    {
        if(!function_exists('htmLawed'))
        {
            require_once __DIR__ . '/vendors/htmlLawed.php';
        }

        if(is_null($whitelist))
        {
            $whitelist = '<code><span><div><label><a><br><p><b><i><del><strike><u><img><video><audio><iframe><object><embed><param><blockquote><mark><cite><small><ul><ol><li><hr><dl><dt><dd><sup><sub><big><pre><code><figure><figcaption><strong><em><table><tr><td><th><tbody><thead><tfoot><h1><h2><h3><h4><h5><h6>';
        }
        // Strip entire blocks of malicious code
        $value = preg_replace(array(
            '@<script[^>]*?>.*?</script>@si',
            '@onclick=[^ ].*? @si'
        ),'',$value);
        // strip unwanted tags via whitelist...
        if(false !== $whitelist) $value = strip_tags($value, $whitelist);
        // cleanup HTML and any unwanted attributes
        $value = htmLawed($value);

        $value  = str_replace('&amp;', '&', $value);

        return $value;
    }
}<commit_msg>Add small comment for & behaviour<commit_after><?php

/**
 * --------------------------------------------------------------------------
 * Helper: squantoCleanupString
 * --------------------------------------------------------------------------
 *
 * Takes an input and cleans up a regular string from unwanted input
 *
 * @param 	string 	$value
 * @return 	string
 */
if(!function_exists('squantoCleanupString'))
{
    function squantoCleanupString( $value )
    {
        $value = strip_tags($value);

        return trim($value);
    }
}

/**
 * --------------------------------------------------------------------------
 * Helper: squantoCleanupHTML
 * --------------------------------------------------------------------------
 *
 * Takes an input and cleans up unwanted / malicious HTML
 *
 * @param 	string 	$value
 * @param 	string 	$whitelist - if false no tagstripping will occur - other than htmLawed
 * @return 	string
 */
if(!function_exists('squantoCleanupHTML'))
{
    function squantoCleanupHTML( $value, $whitelist = null )
    {
        if(!function_exists('htmLawed'))
        {
            require_once __DIR__ . '/vendors/htmlLawed.php';
        }

        if(is_null($whitelist))
        {
            $whitelist = '<code><span><div><label><a><br><p><b><i><del><strike><u><img><video><audio><iframe><object><embed><param><blockquote><mark><cite><small><ul><ol><li><hr><dl><dt><dd><sup><sub><big><pre><code><figure><figcaption><strong><em><table><tr><td><th><tbody><thead><tfoot><h1><h2><h3><h4><h5><h6>';
        }
        // Strip entire blocks of malicious code
        $value = preg_replace(array(
            '@<script[^>]*?>.*?</script>@si',
            '@onclick=[^ ].*? @si'
        ),'',$value);

        // strip unwanted tags via whitelist...
        if(false !== $whitelist) $value = strip_tags($value, $whitelist);

        // cleanup HTML and any unwanted attributes
        $value = htmLawed($value);

        /**
         * htmlLawed converts characters to their encode equivalents. This is something
         * that we need to reverse after the htmlLawed cleanup.
         */
        $value  = str_replace('&amp;', '&', $value);

        return $value;
    }
}

{'fraction_non_alphanumeric': 0.28897881456796, 'fraction_numerical': 0.0028564627469650082, 'mean_word_length': 1.7809397749834546, 'pattern_counts': {'":': 0, '<': 113, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6175409', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1180, 'n_words': 231}