<commit_before><?php
class Vps_Controller_Action_Cli_Web_FulltextController extends Vps_Controller_Action_Cli_Abstract
{
    public static function getHelp()
    {
        return "various fulltext index commands";
    }

    public function optimizeAction()
    {
        Vps_Util_Fulltext::getInstance()->optimize();
        exit;
    }

    public function rebuildAction()
    {
        $queueFile = 'application/temp/fulltextRebuildQueue';

        $componentId = 'root';
        file_put_contents($queueFile, $componentId);
        while(true) {
            $pid = pcntl_fork();
            if ($pid == -1) {
                throw new Vps_Exception("fork failed");
            } else if ($pid) {
                //parent process
                pcntl_wait($status); //Schützt uns vor Zombie Kindern
                if ($status != 0) {
                    throw new Vps_Exception("child process failed");
                }

                //echo "memory_usage (parent): ".(memory_get_usage()/(1024*1024))."MB\n";
                if (!file_get_contents($queueFile)) {
                    echo "fertig.\n";
                    break;
                }
            } else {

                while (true) {
                    //child process

                    //echo "memory_usage (child): ".(memory_get_usage()/(1024*1024))."MB\n";
                    if (memory_get_usage() > 50*1024*1024) {
                        echo "new process...\n";
                        break;
                    }

                    $queue = file_get_contents($queueFile);
                    if (!$queue) break;

                    $queue = explode("\n", $queue);
                    //echo "queued: ".count($queue)."\n";
                    $componentId = array_shift($queue);
                    file_put_contents($queueFile, implode("\n", $queue));

                    //echo "==> ".$componentId.' ';
                    $page = Vps_Component_Data_Root::getInstance()->getComponentById($componentId);
                    //echo "$page->url\n";
                    foreach ($page->getChildPseudoPages(array(), array('pseudoPage'=>false)) as $c) {
                        //echo "queued $c->componentId\n";
                        $queue[] = $c->componentId;
                        file_put_contents($queueFile, implode("\n", $queue));
                    }

                    if (!$page->isPage) continue;

                    //echo "checking for childComponents\n";
                    $fulltextComponents = $page->getRecursiveChildComponents(array('flag'=>'hasFulltext'));
                    if ($fulltextComponents) {
                        echo " *** indexing $page->componentId $page->url...\n";
                        $index = Vps_Util_Fulltext::getInstance();

                        $doc = new Zend_Search_Lucene_Document();

                        //boost, keywords und disable:
                        //können wenns benötigt werden über eigene komponente die als box eingefügt wird implementiert werden

                        $doc->addField(Zend_Search_Lucene_Field::UnIndexed('content', '', 'utf-8'));

                        $t = $page->getTitle();
                        if (substr($t, -3) == ' - ') $t = substr($t, 0, -3);
                        $field = Zend_Search_Lucene_Field::Text('title', $t, 'utf-8');
                        $field->boost = 10;
                        $doc->addField($field);

                        foreach ($fulltextComponents as $c) {
                            $doc = $c->getComponent()->modifyFulltextDocument($doc);
                            //Komponente kann null zurückgeben um zu sagen dass gar nicht indiziert werden soll
                            if (!$doc) break;
                        }

                        if ($doc) {
                            //das wird verwendet um alle dokumente im index zu finden
                            //ned wirklisch a schöne lösung :(
                            $field = Zend_Search_Lucene_Field::UnStored('dummy', 'dummy', 'utf-8');
                            $field->boost = 0.0001;
                            $doc->addField($field);

                            $field = Zend_Search_Lucene_Field::Keyword('componentId', $page->componentId, 'utf-8');
                            $field->boost = 0.0001;
                            $doc->addField($field);

                            $subRoot = $page;
                            while ($subRoot) {
                                if (Vpc_Abstract::getFlag($subRoot->componentClass, 'subroot')) break;
                                $subRoot = $subRoot->parent;
                            }
                            if ($subRoot) {
                                //echo "subroot $subRoot->componentId\n";
                                $field = Zend_Search_Lucene_Field::Keyword('subroot', $subRoot->componentId, 'utf-8');
                                $field->boost = 0.0001;
                                $doc->addField($field);
                            }
                            foreach ($doc->getFieldNames() as $fieldName) {
                                //echo "$fieldName: ".substr($doc->$fieldName, 0, 80)."\n";
                                //echo "$fieldName: ".$doc->$fieldName."\n";
                            }

                            $query = new Zend_Search_Lucene_Search_Query_Term(new Zend_Search_Lucene_Index_Term($page->componentId, 'componentId'));
                            $hits = $index->find($query);
                            foreach ($hits as $hit) {
                                //echo "deleting $hit->componentId\n";
                                $index->delete($hit->id);
                            }

                            $index->addDocument($doc);

                            $m = Vps_Model_Abstract::getInstance('Vpc_FulltextSearch_MetaModel');
                            $row = $m->getRow($page->componentId);
                            if (!$row) {
                                $row = $m->createRow();
                                $row->page_id = $page->componentId;
                            }
                            $row->indexed_date = date('Y-m-d H:i:s');
                            $row->save();
                        }
                    }
                }
                //echo "child finished\n";
                exit(0);
            }
        }
        echo "optimizing...\n";
        Vps_Util_Fulltext::getInstance()->optimize();
        exit;
    }<commit_msg>effizienteres suchen nach einem bereits indiziertem document<commit_after><?php
class Vps_Controller_Action_Cli_Web_FulltextController extends Vps_Controller_Action_Cli_Abstract
{
    public static function getHelp()
    {
        return "various fulltext index commands";
    }

    public function optimizeAction()
    {
        Vps_Util_Fulltext::getInstance()->optimize();
        exit;
    }

    public function rebuildAction()
    {
        $queueFile = 'application/temp/fulltextRebuildQueue';

        $componentId = 'root';
        file_put_contents($queueFile, $componentId);
        while(true) {
            $pid = pcntl_fork();
            if ($pid == -1) {
                throw new Vps_Exception("fork failed");
            } else if ($pid) {
                //parent process
                pcntl_wait($status); //Schützt uns vor Zombie Kindern
                if ($status != 0) {
                    throw new Vps_Exception("child process failed");
                }

                //echo "memory_usage (parent): ".(memory_get_usage()/(1024*1024))."MB\n";
                if (!file_get_contents($queueFile)) {
                    echo "fertig.\n";
                    break;
                }
            } else {

                while (true) {
                    //child process

                    //echo "memory_usage (child): ".(memory_get_usage()/(1024*1024))."MB\n";
                    if (memory_get_usage() > 50*1024*1024) {
                        echo "new process...\n";
                        break;
                    }

                    $queue = file_get_contents($queueFile);
                    if (!$queue) break;

                    $queue = explode("\n", $queue);
                    //echo "queued: ".count($queue)."\n";
                    $componentId = array_shift($queue);
                    file_put_contents($queueFile, implode("\n", $queue));

                    //echo "==> ".$componentId.' ';
                    $page = Vps_Component_Data_Root::getInstance()->getComponentById($componentId);
                    //echo "$page->url\n";
                    foreach ($page->getChildPseudoPages(array(), array('pseudoPage'=>false)) as $c) {
                        //echo "queued $c->componentId\n";
                        $queue[] = $c->componentId;
                        file_put_contents($queueFile, implode("\n", $queue));
                    }

                    if (!$page->isPage) continue;

                    //echo "checking for childComponents\n";
                    $fulltextComponents = $page->getRecursiveChildComponents(array('flag'=>'hasFulltext'));
                    if ($fulltextComponents) {
                        echo " *** indexing $page->componentId $page->url...\n";
                        $index = Vps_Util_Fulltext::getInstance();

                        $doc = new Zend_Search_Lucene_Document();

                        //boost, keywords und disable:
                        //können wenns benötigt werden über eigene komponente die als box eingefügt wird implementiert werden

                        $doc->addField(Zend_Search_Lucene_Field::UnIndexed('content', '', 'utf-8'));

                        $t = $page->getTitle();
                        if (substr($t, -3) == ' - ') $t = substr($t, 0, -3);
                        $field = Zend_Search_Lucene_Field::Text('title', $t, 'utf-8');
                        $field->boost = 10;
                        $doc->addField($field);

                        foreach ($fulltextComponents as $c) {
                            $doc = $c->getComponent()->modifyFulltextDocument($doc);
                            //Komponente kann null zurückgeben um zu sagen dass gar nicht indiziert werden soll
                            if (!$doc) break;
                        }

                        if ($doc) {
                            //das wird verwendet um alle dokumente im index zu finden
                            //ned wirklisch a schöne lösung :(
                            $field = Zend_Search_Lucene_Field::UnStored('dummy', 'dummy', 'utf-8');
                            $field->boost = 0.0001;
                            $doc->addField($field);

                            $field = Zend_Search_Lucene_Field::Keyword('componentId', $page->componentId, 'utf-8');
                            $field->boost = 0.0001;
                            $doc->addField($field);

                            $subRoot = $page;
                            while ($subRoot) {
                                if (Vpc_Abstract::getFlag($subRoot->componentClass, 'subroot')) break;
                                $subRoot = $subRoot->parent;
                            }
                            if ($subRoot) {
                                //echo "subroot $subRoot->componentId\n";
                                $field = Zend_Search_Lucene_Field::Keyword('subroot', $subRoot->componentId, 'utf-8');
                                $field->boost = 0.0001;
                                $doc->addField($field);
                            }
                            foreach ($doc->getFieldNames() as $fieldName) {
                                //echo "$fieldName: ".substr($doc->$fieldName, 0, 80)."\n";
                                //echo "$fieldName: ".$doc->$fieldName."\n";
                            }

                            $term = new Zend_Search_Lucene_Index_Term($page->componentId, 'componentId');
                            $hits = $index->termDocs($term);
                            foreach ($hits as $id) {
                                //echo "deleting $hit->componentId\n";
                                $index->delete($id);
                            }

                            $index->addDocument($doc);

                            $m = Vps_Model_Abstract::getInstance('Vpc_FulltextSearch_MetaModel');
                            $row = $m->getRow($page->componentId);
                            if (!$row) {
                                $row = $m->createRow();
                                $row->page_id = $page->componentId;
                            }
                            $row->indexed_date = date('Y-m-d H:i:s');
                            $row->save();
                        }
                    }
                }
                //echo "child finished\n";
                exit(0);
            }
        }
        echo "optimizing...\n";
        Vps_Util_Fulltext::getInstance()->optimize();
        exit;
    }
{'fraction_non_alphanumeric': 0.1235172572128262, 'fraction_numerical': 0.008724267238080662, 'mean_word_length': 1.1349452703806568, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 98, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3828673', 'n_tokens_mistral': 3417, 'n_tokens_neox': 3155, 'n_words': 601}