<commit_before><?php


use Boom\TextFilter\Commander as TextFilter;
use Boom\TextFilter\Filter as Filter;

class Model_Chunk_Text extends \ORM
{
    protected $_table_columns = [
        'text'        =>    '',
        'id'        =>    '',
        'slotname'    =>    '',
        'page_vid' => '',
        'is_block'    =>    '',
        'site_text' => '',
    ];

    protected $_table_name = 'chunk_texts';

    public function _cleanText()
    {
        $commander = new TextFilter();

        if ($this->slotname === 'standfirst') {
            $commander->addFilter(new Filter\RemoveAllHTML());
        } elseif ($this->is_block) {
            $commander
                ->addFilter(new Filter\MakeInternalLinksRelative())
                ->addFilter(new Filter\PurifyHTML())
                ->addFilter(new Filter\MungeAssetEmbeds())
                ->addFilter(new Filter\MungeRelativeInternalLinks());
        } else {
            $commander->addFilter(new Filter\RemoveHTMLExceptInlineElements());
        }

        $this->text = $commander->filterText($this->text);
    }

    /**
	 *
	 * @param	Validation $validation
	 * @return 	Boom_Model_Chunk_Text
	 */
    public function create(Validation $validation = null)
    {
        $this->_cleanText();

        // Find which assets are linked to within the text chunk.
        preg_match_all('~hoopdb://((image)|(asset))/(\d+)~', $this->_object['text'], $matches);

        if ($this->_object['is_block']) {
            $commander = new TextFilter();
            $commander
                ->addFilter(new Filter\OEmbed())
                ->addFilter(new Filter\StorifyEmbed())
                ->addFilter(new Filter\UnmungeAssetEmbeds())
                ->addFilter(new Filter\RemoveLinksToInvisiblePages())
                ->addFilter(new Filter\UnmungeInternalLinks());

            $this->site_text = $commander->filterText($this->_object['text']);
        }
        else {
            $this->site_text = $this->_object['text'];
        }

        // Create the text chunk.
        parent::create($validation);

        // Are there any asset links?
        if ( ! empty($matches[4])) {
            $assets = array_unique($matches[4]);

            // Log which assets are being referenced with a multi-value insert.
            $query = DB::insert('chunk_text_assets', ['chunk_id', 'asset_id', 'position']);

            foreach ($assets as $i => $asset_id) {
                $query->values([$this->id, $asset_id, $i]);
            }

            try {
                $query->execute();
            } catch (Database_Exception $e) {
                // Don't let database failures in logging prevent the chunk from being saved.
                Kohana_Exception::log($e);
            }
        }

        return $this;
    }
}<commit_msg>Replace &nbsp; with space in text chunks<commit_after><?php


use Boom\TextFilter\Commander as TextFilter;
use Boom\TextFilter\Filter as Filter;

class Model_Chunk_Text extends \ORM
{
    protected $_table_columns = [
        'text'        =>    '',
        'id'        =>    '',
        'slotname'    =>    '',
        'page_vid' => '',
        'is_block'    =>    '',
        'site_text' => '',
    ];

    protected $_table_name = 'chunk_texts';

    public function _cleanText()
    {
        $commander = new TextFilter();

        if ($this->slotname === 'standfirst') {
            $commander->addFilter(new Filter\RemoveAllHTML());
        } elseif ($this->is_block) {
            $commander
                ->addFilter(new Filter\MakeInternalLinksRelative())
                ->addFilter(new Filter\PurifyHTML())
                ->addFilter(new Filter\MungeAssetEmbeds())
                ->addFilter(new Filter\MungeRelativeInternalLinks());
        } else {
            $commander->addFilter(new Filter\RemoveHTMLExceptInlineElements());
        }

        $this->text = $commander->filterText($this->text);
    }

    /**
	 *
	 * @param	Validation $validation
	 * @return 	Boom_Model_Chunk_Text
	 */
    public function create(Validation $validation = null)
    {
        $this->_cleanText();

        // Find which assets are linked to within the text chunk.
        preg_match_all('~hoopdb://((image)|(asset))/(\d+)~', $this->_object['text'], $matches);

        if ($this->_object['is_block']) {
            $commander = new TextFilter();
            $commander
                ->addFilter(new Filter\OEmbed())
                ->addFilter(new Filter\StorifyEmbed())
                ->addFilter(new Filter\UnmungeAssetEmbeds())
                ->addFilter(new Filter\RemoveLinksToInvisiblePages())
                ->addFilter(new Filter\UnmungeInternalLinks());

            $this->site_text = $commander->filterText($this->_object['text']);
        }
        else {
            $this->site_text = $this->_object['text'];
        }

        // Create the text chunk.
        parent::create($validation);

        // Are there any asset links?
        if ( ! empty($matches[4])) {
            $assets = array_unique($matches[4]);

            // Log which assets are being referenced with a multi-value insert.
            $query = DB::insert('chunk_text_assets', ['chunk_id', 'asset_id', 'position']);

            foreach ($assets as $i => $asset_id) {
                $query->values([$this->id, $asset_id, $i]);
            }

            try {
                $query->execute();
            } catch (Database_Exception $e) {
                // Don't let database failures in logging prevent the chunk from being saved.
                Kohana_Exception::log($e);
            }
        }

        return $this;
    }

    public function filters()
    {
        return array(
            'text' => array(
                array(function($text) {
                    return str_replace('&nbsp;', ' ', $text);
                }),
            ),
        );
    }
}
{'fraction_non_alphanumeric': 0.14293006636038796, 'fraction_numerical': 0.000680619363620895, 'mean_word_length': 1.8533980582524272, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3922842', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1590, 'n_words': 323}