<commit_before>#!/usr/bin/php
<?php
/**********************************************************
* File      :   migrate-filestates-to-db.php
* Project   :   Z-Push - tools
* Descr     :   Copies file states to the database
*
* Created   :   12.02.2016
*
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation with the following additional
* term according to sec. 7:
*
* According to sec. 7 of the GNU Affero General Public License, version 3,
* the terms of the AGPL are supplemented with the following terms:
*
* "Zarafa" is a registered trademark of Zarafa B.V.
* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
* The licensing of the Program under the AGPL does not imply a trademark license.
* Therefore any rights, title and interest in our trademarks remain entirely with us.
*
* However, if you propagate an unmodified version of the Program you are
* allowed to use the term "Z-Push" to indicate that you distribute the Program.
* Furthermore you may use our trademarks where it is necessary to indicate
* the intended purpose of a product or service provided you use it in accordance
* with honest practices in industrial or commercial matters.
* If you want to propagate modified versions of the Program under the name "Z-Push",
* you may only do so if you have a written permission by Zarafa Deutschland GmbH
* (to acquire a permission please contact Zarafa at <EMAIL>).
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Consult LICENSE file for details
************************************************/

// Please adjust to match your z-push installation directory, usually /usr/share/z-push
define('ZPUSH_BASE_PATH', "/usr/share/z-push");

/************************************************
 * MAIN
 */
print("Starting the filestate to database migration script." . PHP_EOL);

try {
    if (php_sapi_name() != "cli") {
        die("This script can only be called from the CLI.");
    }

    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
    }

    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);

    include_once('lib/core/zpushdefs.php');
    include_once('lib/core/zpush.php');
    include_once('lib/core/zlog.php');
    include_once('lib/core/statemanager.php');
    include_once('lib/core/stateobject.php');
    include_once('lib/core/asdevice.php');
    include_once('lib/interface/ichanges.php');
    include_once('lib/interface/iexportchanges.php');
    include_once('lib/interface/iimportchanges.php');
    include_once('lib/core/hierarchycache.php');
    include_once('lib/core/interprocessdata.php');
    include_once('lib/core/syncparameters.php');
    include_once('lib/core/bodypreference.php');
    include_once('lib/core/contentparameters.php');
    include_once('lib/core/changesmemorywrapper.php');
    include_once('lib/core/streamer.php');
    include_once('lib/exceptions/exceptions.php');
    include_once('lib/utils/utils.php');
    include_once('lib/request/request.php');
    include_once('lib/request/requestprocessor.php');
    include_once('lib/interface/ibackend.php');
    include_once('lib/interface/ichanges.php');
    include_once('lib/interface/iexportchanges.php');
    include_once('lib/interface/iimportchanges.php');
    include_once('lib/interface/isearchprovider.php');
    include_once('lib/interface/istatemachine.php');
    include_once('lib/syncobjects/syncobject.php');
    include_once('lib/syncobjects/syncfolder.php');
    include_once('config.php');
    include_once('lib/default/filestatemachine.php');
    include_once('backend/sqlstatemachine/sqlstatemachine.php');


    if (STATE_MACHINE != 'FILE') {
        die(sprintf("Only migration from 'FILE' StateMachine is possible. Your STATE_MACHINE setting is '%s'. Please modify your config.php.%s", STATE_MACHINE, PHP_EOL));
    }

    printf("Also check the logfile at %s for more information.%s", LOGFILE, PHP_EOL);

    ZPush::CheckConfig();
    $migrate = new StateMigratorFileToDB();

    if (!$migrate->MigrationNecessary()) {
        print("Migration script was run before and eventually no migration is necessary." . PHP_EOL);
        exit(1);
    }

    $migrate->DoMigration();
}
catch (ZPushException $zpe) {
    die(get_class($zpe) . ": ". $zpe->getMessage() . "\n");
}

sprintf("terminated%s", PHP_EOL);

class StateMigratorFileToDB {
    private $fsm;
    private $dbsm;


    /**
     * Check if the migration is necessary.
     *
     * @access public
     * @throws FatalMisconfigurationException
     * @throws FatalNotImplementedException
     * @return boolean
     */
    public function MigrationNecessary() {
        print("StateMigratorFileToDB->MigrationNecessary(): checking if migration is necessary." . PHP_EOL);
        try {
            $this->dbsm = new SqlStateMachine();
            if ($this->dbsm->checkTablesHaveData()) {
                print ("All state tables have data already. A migration is not necessary." . PHP_EOL);
                return false;
            }
        }
        catch (ZPushException $ex) {
            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
        }
        return true;
    }

    /**
     * Execute the migration.
     *
     * @access public
     * @return true
     */
    public function DoMigration() {
        print("StateMigratorFileToDB->DoMigration(): Starting migration routine." . PHP_EOL);
        try {
            $this->fsm = new FileStateMachine();

            if (!($this->fsm  instanceof FileStateMachine)) {
                throw new FatalNotImplementedException("This conversion script is only able to convert states from the FileStateMachine");
            }

            // get all state information for all devices
            $alldevices = $this->fsm->GetAllDevices(false);
            foreach ($alldevices as $devid) {
                $lowerDevid = strtolower($devid);

                $allStates = $this->fsm->GetAllStatesForDevice($lowerDevid);
                printf("Processing device: %s with %d states\t", $devid, count($allStates));
                foreach ($allStates as $stateInfo) {
                    $state = $this->fsm->GetState($lowerDevid, $stateInfo['type'], $stateInfo['uuid'], (int) $stateInfo['counter'], false);
                    $this->dbsm->SetState($state, $lowerDevid, $stateInfo['type'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
                    printf("%s\t\tsetting state:%s-%s-%d", PHP_EOL, $stateInfo['uuid'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
                }

                // link devices to users
                $devState = $this->fsm->GetState($lowerDevid, IStateMachine::DEVICEDATA);
                foreach ($devState->devices as $user => $dev) {
                    $this->dbsm->LinkUserDevice($user, $dev->deviceid, date("Y-m-d H:i:s", $dev->firstsynctime), date("Y-m-d H:i:s", $dev->lastupdatetime));
                }

                print("\tcompleted" . PHP_EOL);
            }
        }
        catch (ZPushException $ex) {
            print (PHP_EOL . "Something went wrong during the migration. The script will now exit." . PHP_EOL);
            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
        }
        print("StateMigratorFileToDB->DoMigration(): Migration completed successfuly." . PHP_EOL);
    }
}<commit_msg>ZP-230 Fix hierarchy folder data before starting the migration. Fixed typo in output.<commit_after>#!/usr/bin/php
<?php
/**********************************************************
* File      :   migrate-filestates-to-db.php
* Project   :   Z-Push - tools
* Descr     :   Copies file states to the database
*
* Created   :   12.02.2016
*
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation with the following additional
* term according to sec. 7:
*
* According to sec. 7 of the GNU Affero General Public License, version 3,
* the terms of the AGPL are supplemented with the following terms:
*
* "Zarafa" is a registered trademark of Zarafa B.V.
* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
* The licensing of the Program under the AGPL does not imply a trademark license.
* Therefore any rights, title and interest in our trademarks remain entirely with us.
*
* However, if you propagate an unmodified version of the Program you are
* allowed to use the term "Z-Push" to indicate that you distribute the Program.
* Furthermore you may use our trademarks where it is necessary to indicate
* the intended purpose of a product or service provided you use it in accordance
* with honest practices in industrial or commercial matters.
* If you want to propagate modified versions of the Program under the name "Z-Push",
* you may only do so if you have a written permission by Zarafa Deutschland GmbH
* (to acquire a permission please contact Zarafa at <EMAIL>).
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Consult LICENSE file for details
************************************************/

// Please adjust to match your z-push installation directory, usually /usr/share/z-push
define('ZPUSH_BASE_PATH', "/usr/share/z-push");

/************************************************
 * MAIN
 */
print("Starting the filestate to database migration script." . PHP_EOL);

try {
    if (php_sapi_name() != "cli") {
        die("This script can only be called from the CLI.");
    }

    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
    }

    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);

    include_once('lib/core/zpushdefs.php');
    include_once('lib/core/zpush.php');
    include_once('lib/core/zlog.php');
    include_once('lib/core/statemanager.php');
    include_once('lib/core/stateobject.php');
    include_once('lib/core/asdevice.php');
    include_once('lib/interface/ichanges.php');
    include_once('lib/interface/iexportchanges.php');
    include_once('lib/interface/iimportchanges.php');
    include_once('lib/core/hierarchycache.php');
    include_once('lib/core/interprocessdata.php');
    include_once('lib/core/syncparameters.php');
    include_once('lib/core/bodypreference.php');
    include_once('lib/core/contentparameters.php');
    include_once('lib/core/changesmemorywrapper.php');
    include_once('lib/core/streamer.php');
    include_once('lib/exceptions/exceptions.php');
    include_once('lib/utils/utils.php');
    include_once('lib/request/request.php');
    include_once('lib/request/requestprocessor.php');
    include_once('lib/interface/ibackend.php');
    include_once('lib/interface/ichanges.php');
    include_once('lib/interface/iexportchanges.php');
    include_once('lib/interface/iimportchanges.php');
    include_once('lib/interface/isearchprovider.php');
    include_once('lib/interface/istatemachine.php');
    include_once('lib/syncobjects/syncobject.php');
    include_once('lib/syncobjects/syncfolder.php');
    include_once('config.php');
    include_once('lib/default/filestatemachine.php');
    include_once('backend/sqlstatemachine/sqlstatemachine.php');


    if (STATE_MACHINE != 'FILE') {
        die(sprintf("Only migration from 'FILE' StateMachine is possible. Your STATE_MACHINE setting is '%s'. Please modify your config.php.%s", STATE_MACHINE, PHP_EOL));
    }

    printf("Also check the logfile at %s for more information.%s", LOGFILE, PHP_EOL);

    ZPush::CheckConfig();
    $migrate = new StateMigratorFileToDB();

    if (!$migrate->MigrationNecessary()) {
        print("Migration script was run before and eventually no migration is necessary." . PHP_EOL);
        exit(1);
    }

    $migrate->DoMigration();
}
catch (ZPushException $zpe) {
    die(get_class($zpe) . ": ". $zpe->getMessage() . "\n");
}

sprintf("terminated%s", PHP_EOL);

class StateMigratorFileToDB {
    private $fsm;
    private $dbsm;


    /**
     * Check if the migration is necessary.
     *
     * @access public
     * @throws FatalMisconfigurationException
     * @throws FatalNotImplementedException
     * @return boolean
     */
    public function MigrationNecessary() {
        print("StateMigratorFileToDB->MigrationNecessary(): checking if migration is necessary." . PHP_EOL);
        try {
            $this->dbsm = new SqlStateMachine();
            if ($this->dbsm->checkTablesHaveData()) {
                print ("All state tables have data already. A migration is not necessary." . PHP_EOL);
                return false;
            }
        }
        catch (ZPushException $ex) {
            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
        }
        return true;
    }

    /**
     * Execute the migration.
     *
     * @access public
     * @return true
     */
    public function DoMigration() {
        print("StateMigratorFileToDB->DoMigration(): Starting migration routine." . PHP_EOL);
        try {
            // Fix hierarchy folder data before starting the migration
            ZPushAdmin::FixStatesHierarchyFolderData();

            $this->fsm = new FileStateMachine();

            if (!($this->fsm  instanceof FileStateMachine)) {
                throw new FatalNotImplementedException("This conversion script is only able to convert states from the FileStateMachine");
            }

            // get all state information for all devices
            $alldevices = $this->fsm->GetAllDevices(false);
            foreach ($alldevices as $devid) {
                $lowerDevid = strtolower($devid);

                $allStates = $this->fsm->GetAllStatesForDevice($lowerDevid);
                printf("Processing device: %s with %d states\t", $devid, count($allStates));
                foreach ($allStates as $stateInfo) {
                    $state = $this->fsm->GetState($lowerDevid, $stateInfo['type'], $stateInfo['uuid'], (int) $stateInfo['counter'], false);
                    $this->dbsm->SetState($state, $lowerDevid, $stateInfo['type'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
                    printf("%s\t\tsetting state:%s-%s-%d", PHP_EOL, $stateInfo['uuid'], (empty($stateInfo['type']) ? NULL : $stateInfo['type']), (int) $stateInfo['counter']);
                }

                // link devices to users
                $devState = $this->fsm->GetState($lowerDevid, IStateMachine::DEVICEDATA);
                foreach ($devState->devices as $user => $dev) {
                    $this->dbsm->LinkUserDevice($user, $dev->deviceid, date("Y-m-d H:i:s", $dev->firstsynctime), date("Y-m-d H:i:s", $dev->lastupdatetime));
                }

                print("\tcompleted" . PHP_EOL);
            }
        }
        catch (ZPushException $ex) {
            print (PHP_EOL . "Something went wrong during the migration. The script will now exit." . PHP_EOL);
            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
        }
        print("StateMigratorFileToDB->DoMigration(): Migration completed successfuly." . PHP_EOL);
    }
}
{'fraction_non_alphanumeric': 0.14110579870050263, 'fraction_numerical': 0.002758367046708349, 'mean_word_length': 3.2103225806451614, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1538886', 'n_tokens_mistral': 4815, 'n_tokens_neox': 4653, 'n_words': 1283}