<commit_before><?php

class GoogleSitemapCreateCommand extends CConsoleCommand
{
    public function run($args)
    {

        $sitemap_files = array();

        $datumformat = function ($datum) {
            $x = explode(" ", $datum);
            $y = explode("-", $x[0]);
            if ($y[0] < 2000) $y[0] = "2000";
            if ($y[1] < 1 || $y[1] > 12) $y[1] = "01";
            if ($y[2] < 1 || $y[2] > 31) $y[2] = "01";
            return $y[0] . "-" . $y[1] . "-" . $y[2];
        };


        $sitemap_basepath = Yii::app()->getBasePath() . DIRECTORY_SEPARATOR . ".." . DIRECTORY_SEPARATOR . "html" . DIRECTORY_SEPARATOR;


        // Dokumente

        $sql       = Yii::app()->db->createCommand();
        $dokumente = $sql->select("id, datum")->from("dokumente")->order("id")->queryAll();

        $sm_num = Ceil(count($dokumente) / 30000);
        for ($sm_page = 0; $sm_page < $sm_num; $sm_page++) {
            echo "Dokumente - Seite $sm_page\n";

            $sitemap_file = "sitemap-dokumente-" . $sm_page . ".xml";
            $fp           = fopen($sitemap_basepath . $sitemap_file, "w");
            fwrite($fp, '<?xml version="1.0" encoding="UTF-8"?>' . "\n");
            fwrite($fp, '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n");

            for ($i = $sm_page * 30000; $i < ($sm_page + 1) * 30000 && $i < count($dokumente); $i++) {
                $dok = $dokumente[$i];
                fwrite($fp, "<url>\n<loc>" . SITE_BASE_URL . "/dokumente/" . $dok["id"] . "/</loc>\n");
                fwrite($fp, "<changefreq>monthly</changefreq>\n");
                fwrite($fp, "<lastmod>" . $datumformat($dok["datum"]) . "</lastmod>\n");
                fwrite($fp, "</url>\n");
            }

            fwrite($fp, "</urlset>\n");
            fclose($fp);

            $sitemap_files[] = SITE_BASE_URL . "/" . $sitemap_file;
        }
<commit_msg>Gelöschte Dokumente nicht in die Sitemap<commit_after><?php

class GoogleSitemapCreateCommand extends CConsoleCommand
{
    public function run($args)
    {

        $sitemap_files = array();

        $datumformat = function ($datum) {
            $x = explode(" ", $datum);
            $y = explode("-", $x[0]);
            if ($y[0] < 2000) $y[0] = "2000";
            if ($y[1] < 1 || $y[1] > 12) $y[1] = "01";
            if ($y[2] < 1 || $y[2] > 31) $y[2] = "01";
            return $y[0] . "-" . $y[1] . "-" . $y[2];
        };


        $sitemap_basepath = Yii::app()->getBasePath() . DIRECTORY_SEPARATOR . ".." . DIRECTORY_SEPARATOR . "html" . DIRECTORY_SEPARATOR;


        // Dokumente

        $sql       = Yii::app()->db->createCommand();
        $dokumente = $sql->select("id, datum")->from("dokumente")->where("deleted = 0")->order("id")->queryAll();

        $sm_num = Ceil(count($dokumente) / 30000);
        for ($sm_page = 0; $sm_page < $sm_num; $sm_page++) {
            echo "Dokumente - Seite $sm_page\n";

            $sitemap_file = "sitemap-dokumente-" . $sm_page . ".xml";
            $fp           = fopen($sitemap_basepath . $sitemap_file, "w");
            fwrite($fp, '<?xml version="1.0" encoding="UTF-8"?>' . "\n");
            fwrite($fp, '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n");

            for ($i = $sm_page * 30000; $i < ($sm_page + 1) * 30000 && $i < count($dokumente); $i++) {
                $dok = $dokumente[$i];
                fwrite($fp, "<url>\n<loc>" . SITE_BASE_URL . "/dokumente/" . $dok["id"] . "/</loc>\n");
                fwrite($fp, "<changefreq>monthly</changefreq>\n");
                fwrite($fp, "<lastmod>" . $datumformat($dok["datum"]) . "</lastmod>\n");
                fwrite($fp, "</url>\n");
            }

            fwrite($fp, "</urlset>\n");
            fclose($fp);

            $sitemap_files[] = SITE_BASE_URL . "/" . $sitemap_file;
        }

{'fraction_non_alphanumeric': 0.22092115534738485, 'fraction_numerical': 0.0273224043715847, 'mean_word_length': 1.963762528912876, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 2, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6534236', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1408, 'n_words': 240}