<commit_before><?php
class Kwf_Controller_Action_Cli_Web_ComponentPagesMetaController extends Kwf_Controller_Action
{
    public function checkForInvalidAction()
    {
        set_time_limit(0);
        $model = Kwf_Model_Abstract::getInstance('Kwf_Component_PagesMetaModel');
        $select = new Kwf_Model_Select();
        $it = new Kwf_Model_Iterator_Packages(
            new Kwf_Model_Iterator_Rows($model, $select)
        );
        if ($this->_getParam('debug')) $it = new Kwf_Iterator_ConsoleProgressBar($it);
        $i = 0;
        foreach ($it as $row) {
            $page = Kwf_Component_Data_Root::getInstance()->getComponentById($row->page_id);
            if (!$page) {
                if (!$this->_getParam('slient')) {
                    echo "\n$row->page_id is in pages_meta aber but not in page tree, deleting...\n";
                }
                $row->delete();
            }
            if ($i++ % 100) {
                Kwf_Component_Data_Root::getInstance()->freeMemory();
            }

        }
        exit;
    }<commit_msg>Improve performance of component-pages-meta check-for-invalid<commit_after><?php
class Kwf_Controller_Action_Cli_Web_ComponentPagesMetaController extends Kwf_Controller_Action
{
    public function checkForInvalidAction()
    {
        set_time_limit(0);
        $model = Kwf_Model_Abstract::getInstance('Kwf_Component_PagesMetaModel');
        $select = new Kwf_Model_Select();
        $it = new Kwf_Model_Iterator_Packages(
            new Kwf_Model_Iterator_Rows($model, $select)
        );
        if ($this->_getParam('debug')) $it = new Kwf_Iterator_ConsoleProgressBar($it);
        $i = 0;
        foreach ($it as $row) {
            $page = Kwf_Component_Data_Root::getInstance()->getComponentById($row->page_id);
            if (!$page) {
                if (!$this->_getParam('silent')) {
                    echo "\n$row->page_id is in pages_meta aber but not in page tree, deleting...\n";
                }
                $row->delete();
            }
            if (memory_get_usage() > 128*1024*1024) {
                if ($this->_getParam('debug')) echo "Collect garbage...\n";;
                Kwf_Component_Data_Root::getInstance()->freeMemory();
            }

        }
        exit;
    }
{'fraction_non_alphanumeric': 0.12329370321444298, 'fraction_numerical': 0.007926023778071334, 'mean_word_length': 2.2318634423897583, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2651660', 'n_tokens_mistral': 690, 'n_tokens_neox': 639, 'n_words': 119}