<commit_before>#!/usr/bin/env php
<?php
/*
 * StatusNet - a distributed open-source microblogging tool
 * Copyright (C) 2008-2011 StatusNet, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));

$shortoptions = 'x::';
$longoptions = array('extensions=');

$helptext = <<<END_OF_UPGRADE_HELP
php upgrade.php [options]
Upgrade database schema and data to latest software

END_OF_UPGRADE_HELP;

require_once INSTALLDIR.'/scripts/commandline.inc';

function main()
{
    if (Event::handle('StartUpgrade')) {
        fixupConversationURIs();

        updateSchemaCore();
        updateSchemaPlugins();

        // These replace old "fixup_*" scripts

        fixupNoticeConversation();
        initConversation();
        fixupGroupURI();
        fixupFileGeometry();
        deleteLocalFileThumbnailsWithoutFilename();
        deleteMissingLocalFileThumbnails();
        fixupFileThumbnailUrlhash();
        setFilehashOnLocalFiles();

        initGroupProfileId();
        initLocalGroup();
        initNoticeReshare();
    
        initSubscriptionURI();
        initGroupMemberURI();

        initProfileLists();

        migrateProfilePrefs();

        Event::handle('EndUpgrade');
    }
}

function tableDefs()
{
	$schema = array();
	require INSTALLDIR.'/db/core.php';
	return $schema;
}

function updateSchemaCore()
{
    printfnq("Upgrading core schema...");

    $schema = Schema::get();
    $schemaUpdater = new SchemaUpdater($schema);
    foreach (tableDefs() as $table => $def) {
        $schemaUpdater->register($table, $def);
    }
    $schemaUpdater->checkSchema();

    printfnq("DONE.\n");
}

function updateSchemaPlugins()
{
    printfnq("Upgrading plugin schema...");

    Event::handle('BeforePluginCheckSchema');
    Event::handle('CheckSchema');

    printfnq("DONE.\n");
}

function fixupNoticeConversation()
{
    printfnq("Ensuring all notices have a conversation ID...");

    $notice = new Notice();
    $notice->whereAdd('conversation is null');
    $notice->whereAdd('conversation = 0', 'OR');
    $notice->orderBy('id'); // try to get originals before replies
    $notice->find();

    while ($notice->fetch()) {
        try {
            $cid = null;
    
            $orig = clone($notice);
    
            if (!empty($notice->reply_to)) {
                $reply = Notice::getKV('id', $notice->reply_to);

                if ($reply instanceof Notice && !empty($reply->conversation)) {
                    $notice->conversation = $reply->conversation;
                }
                unset($reply);
            }

            // if still empty
            if (empty($notice->conversation)) {
                $child = new Notice();
                $child->reply_to = $notice->getID();
                $child->limit(1);
                if ($child->find(true) && !empty($child->conversation)) {
                    $notice->conversation = $child->conversation;
                }
                unset($child);
            }

            // if _still_ empty we just create our own conversation
            if (empty($notice->conversation)) {
                $notice->conversation = $notice->getID();
            }

            $result = $notice->update($orig);

            unset($orig);
        } catch (Exception $e) {
            print("Error setting conversation: " . $e->getMessage());
        }
    }

    printfnq("DONE.\n");
}

function fixupGroupURI()
{
    printfnq("Ensuring all groups have an URI...");

    $group = new User_group();
    $group->whereAdd('uri IS NULL');

    if ($group->find()) {
        while ($group->fetch()) {
            $orig = User_group::getKV('id', $group->id);
            $group->uri = $group->getUri();
            $group->update($orig);
        }
    }

    printfnq("DONE.\n");
}

function initConversation()
{
    printfnq("Ensuring all conversations have a row in conversation table...");

    $notice = new Notice();
    $notice->query('select distinct notice.conversation from notice '.
                   'where notice.conversation is not null '.
                   'and not exists (select conversation.id from conversation where id = notice.conversation)');

    while ($notice->fetch()) {

        $id = $notice->conversation;

        $uri = common_local_url('conversation', array('id' => $id));

        // @fixme db_dataobject won't save our value for an autoincrement
        // so we're bypassing the insert wrappers
        $conv = new Conversation();
        $sql = "insert into conversation (id,uri,created) values(%d,'%s','%s')";
        $sql = sprintf($sql,
                       $id,
                       $conv->escape($uri),
                       $conv->escape(common_sql_now()));
        $conv->query($sql);
    }

    printfnq("DONE.\n");<commit_msg>Use join instead of exists in SQL<commit_after>#!/usr/bin/env php
<?php
/*
 * StatusNet - a distributed open-source microblogging tool
 * Copyright (C) 2008-2011 StatusNet, Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));

$shortoptions = 'x::';
$longoptions = array('extensions=');

$helptext = <<<END_OF_UPGRADE_HELP
php upgrade.php [options]
Upgrade database schema and data to latest software

END_OF_UPGRADE_HELP;

require_once INSTALLDIR.'/scripts/commandline.inc';

function main()
{
    if (Event::handle('StartUpgrade')) {
        fixupConversationURIs();

        updateSchemaCore();
        updateSchemaPlugins();

        // These replace old "fixup_*" scripts

        fixupNoticeConversation();
        initConversation();
        fixupGroupURI();
        fixupFileGeometry();
        deleteLocalFileThumbnailsWithoutFilename();
        deleteMissingLocalFileThumbnails();
        fixupFileThumbnailUrlhash();
        setFilehashOnLocalFiles();

        initGroupProfileId();
        initLocalGroup();
        initNoticeReshare();
    
        initSubscriptionURI();
        initGroupMemberURI();

        initProfileLists();

        migrateProfilePrefs();

        Event::handle('EndUpgrade');
    }
}

function tableDefs()
{
	$schema = array();
	require INSTALLDIR.'/db/core.php';
	return $schema;
}

function updateSchemaCore()
{
    printfnq("Upgrading core schema...");

    $schema = Schema::get();
    $schemaUpdater = new SchemaUpdater($schema);
    foreach (tableDefs() as $table => $def) {
        $schemaUpdater->register($table, $def);
    }
    $schemaUpdater->checkSchema();

    printfnq("DONE.\n");
}

function updateSchemaPlugins()
{
    printfnq("Upgrading plugin schema...");

    Event::handle('BeforePluginCheckSchema');
    Event::handle('CheckSchema');

    printfnq("DONE.\n");
}

function fixupNoticeConversation()
{
    printfnq("Ensuring all notices have a conversation ID...");

    $notice = new Notice();
    $notice->whereAdd('conversation is null');
    $notice->whereAdd('conversation = 0', 'OR');
    $notice->orderBy('id'); // try to get originals before replies
    $notice->find();

    while ($notice->fetch()) {
        try {
            $cid = null;
    
            $orig = clone($notice);
    
            if (!empty($notice->reply_to)) {
                $reply = Notice::getKV('id', $notice->reply_to);

                if ($reply instanceof Notice && !empty($reply->conversation)) {
                    $notice->conversation = $reply->conversation;
                }
                unset($reply);
            }

            // if still empty
            if (empty($notice->conversation)) {
                $child = new Notice();
                $child->reply_to = $notice->getID();
                $child->limit(1);
                if ($child->find(true) && !empty($child->conversation)) {
                    $notice->conversation = $child->conversation;
                }
                unset($child);
            }

            // if _still_ empty we just create our own conversation
            if (empty($notice->conversation)) {
                $notice->conversation = $notice->getID();
            }

            $result = $notice->update($orig);

            unset($orig);
        } catch (Exception $e) {
            print("Error setting conversation: " . $e->getMessage());
        }
    }

    printfnq("DONE.\n");
}

function fixupGroupURI()
{
    printfnq("Ensuring all groups have an URI...");

    $group = new User_group();
    $group->whereAdd('uri IS NULL');

    if ($group->find()) {
        while ($group->fetch()) {
            $orig = User_group::getKV('id', $group->id);
            $group->uri = $group->getUri();
            $group->update($orig);
        }
    }

    printfnq("DONE.\n");
}

function initConversation()
{
    printfnq("Ensuring all conversations have a row in conversation table...");

    $notice = new Notice();
    $notice->selectAdd();
    $notice->selectAdd('DISTINCT conversation');
    $notice->joinAdd(['conversation', 'conversation:id'], 'LEFT');  // LEFT to get the null values for conversation.id
    $notice->whereAdd('conversation.id IS NULL');

    if ($notice->find()) {
        printfnq(" fixing {$notice->N} missing conversation entries...");
    }

    while ($notice->fetch()) {

        $id = $notice->conversation;

        $uri = common_local_url('conversation', array('id' => $id));

        // @fixme db_dataobject won't save our value for an autoincrement
        // so we're bypassing the insert wrappers
        $conv = new Conversation();
        $sql = "insert into conversation (id,uri,created) values(%d,'%s','%s')";
        $sql = sprintf($sql,
                       $id,
                       $conv->escape($uri),
                       $conv->escape(common_sql_now()));
        $conv->query($sql);
    }

    printfnq("DONE.\n");
{'fraction_non_alphanumeric': 0.14046579330422126, 'fraction_numerical': 0.0020014556040756915, 'mean_word_length': 2.4246105919003114, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 90, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182255', 'n_tokens_mistral': 3159, 'n_tokens_neox': 3041, 'n_words': 782}