<commit_before><?php
/***********************************************
* File      :   loopdetection.php
* Project   :   Z-Push
* Descr     :   detects an outgoing loop by looking
*               if subsequent requests do try to get changes
*               for the same sync key. If more than once a synckey
*               is requested, the amount of items to be sent to the mobile
*               is reduced to one. If then (again) the same synckey is
*               requested, we have most probably found the 'broken' item.
*
* Created   :   20.10.2011
*
* Copyright 2007 - 2011 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation with the following additional
* term according to sec. 7:
*
* According to sec. 7 of the GNU Affero General Public License, version 3,
* the terms of the AGPL are supplemented with the following terms:
*
* "Zarafa" is a registered trademark of Zarafa B.V.
* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
* The licensing of the Program under the AGPL does not imply a trademark license.
* Therefore any rights, title and interest in our trademarks remain entirely with us.
*
* However, if you propagate an unmodified version of the Program you are
* allowed to use the term "Z-Push" to indicate that you distribute the Program.
* Furthermore you may use our trademarks where it is necessary to indicate
* the intended purpose of a product or service provided you use it in accordance
* with honest practices in industrial or commercial matters.
* If you want to propagate modified versions of the Program under the name "Z-Push",
* you may only do so if you have a written permission by Zarafa Deutschland GmbH
* (to acquire a permission please contact Zarafa at <EMAIL>).
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Consult LICENSE file for details
************************************************/


class LoopDetection extends InterProcessData {
    const INTERPROCESSLD = "ipldkey";
    const BROKENMSGS = "bromsgs";
    static private $processident;
    static private $processentry;
    private $ignore_messageid;
    private $broken_message_uuid;
    private $broken_message_counter;


    /**
     * Constructor
     *
     * @access public
     */
    public function LoopDetection() {
        // initialize super parameters
        $this->allocate = 204800; // 200 KB
        $this->type = 1337;
        parent::__construct();

        $this->ignore_messageid = false;
    }

    /**
     * PROCESS LOOP DETECTION
     */

    /**
     * Adds the process entry to the process stack
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionInit() {
        return $this->updateProcessStack();
    }

    /**
     * Marks the process entry as termineted successfully on the process stack
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionTerminate() {
        // just to be sure that the entry is there
        self::GetProcessEntry();

        self::$processentry['end'] = time();
        ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionTerminate()");
        return $this->updateProcessStack();
    }

    /**
     * Returns a unique identifier for the internal process tracking
     *
     * @access public
     * @return string
     */
    public static function GetProcessIdentifier() {
        if (!isset(self::$processident))
            self::$processident = sprintf('%04x%04', mt_rand(0, 0xffff), mt_rand(0, 0xffff));

        return self::$processident;
    }

    /**
     * Returns a unique entry with informations about the current process
     *
     * @access public
     * @return array
     */
    public static function GetProcessEntry() {
        if (!isset(self::$processentry)) {
            self::$processentry = array();
            self::$processentry['id'] = self::GetProcessIdentifier();
            self::$processentry['pid'] = self::$pid;
            self::$processentry['time'] = self::$start;
            self::$processentry['cc'] = Request::GetCommandCode();
        }

        return self::$processentry;
    }

    /**
     * Adds an Exceptions to the process tracking
     *
     * @param Exception     $exception
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionAddException($exception) {
        // generate entry if not already there
        self::GetProcessEntry();

        if (!isset(self::$processentry['stat']))
            self::$processentry['stat'] = array();

        self::$processentry['stat'][get_class($exception)] = $exception->getCode();

        $this->updateProcessStack();
        return true;
    }

    /**
     * Adds a folderid and connected status code to the process tracking
     *
     * @param string    $folderid
     * @param int       $status
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionAddStatus($folderid, $status) {
        // generate entry if not already there
        self::GetProcessEntry();

        if ($folderid === false)
            $folderid = "hierarchy";

        if (!isset(self::$processentry['stat']))
            self::$processentry['stat'] = array();

        self::$processentry['stat'][$folderid] = $status;

        $this->updateProcessStack();

        return true;
    }

    /**
     * Indicates if a full Hierarchy Resync is necessary
     *
     * In some occasions the mobile tries to sync a folder with an invalid/not-existing ID.
     * In these cases a status exception like SYNC_STATUS_FOLDERHIERARCHYCHANGED is returned
     * so the mobile executes a FolderSync expecting that some action is taken on that folder (e.g. remove).
     *
     * If the FolderSync is not doing anything relevant, then the Sync is attempted again
     * resulting in the same error and looping between these two processes.
     *
     * This method checks if in the last process stack a Sync and FolderSync were triggered to
     * catch the loop at the 2nd interaction (Sync->FolderSync->Sync->FolderSync => ReSync)
     * Ticket: https://jira.zarafa.com/browse/ZP-5
     *
     * @access public
     * @return boolean
     *
     */
    public function ProcessLoopDetectionIsHierarchyResyncRequired() {
        $seenFailed = array();
        $seenFolderSync = false;

        $lookback = self::$start - 600; // look at the last 5 min
        foreach ($this->getProcessStack() as $se) {
            if ($se['time'] > $lookback && $se['time'] < (self::$start-1)) {
                // look for sync command
                if (isset($se['stat']) && ($se['cc'] == ZPush::COMMAND_SYNC || $se['cc'] == ZPush::COMMAND_PING)) {
                    foreach($se['stat'] as $key => $value) {
                        if (!isset($seenFailed[$key]))
                            $seenFailed[$key] = 0;
                        $seenFailed[$key]++;
                        ZLog::Write(LOGLEVEL_DEBUG, sprintf("LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): seen command with Exception or folderid '%s' and code '%s'", $key, $value ));
                    }
                }
                // look for FolderSync command with previous failed commands
                if ($se['cc'] == ZPush::COMMAND_FOLDERSYNC && !empty($seenFailed) && $se['id'] != self::GetProcessIdentifier()) {
                    // a full folderresync was already triggered
                    if (isset($se['stat']) && $se['stat']['hierarchy'] == SYNC_FSSTATUS_SYNCKEYERROR) {
                        ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): a full FolderReSync was already requested. Resetting fail counter.");
                        $seenFailed = array();
                    }
                    else {
                        $seenFolderSync = true;
                        if (!empty($seenFailed))
                            ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): seen FolderSync after other failing command");
                    }
                }
            }
        }

        $filtered = array();
        foreach ($seenFailed as $k => $count) {
            if ($count>1)
                $filtered[] = $k;
        }

        if ($seenFolderSync && !empty($filtered)) {
            ZLog::Write(LOGLEVEL_INFO, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): Potential loop detected. Full hierarchysync indicated.");
            return true;
        }

        return false;
    }

    /**
     * Indicates if a previous process could not be terminated
     *
     * Checks if there is an end time for the last entry on the stack
     *
     * @access public
     * @return boolean
     *
     */
    public function ProcessLoopDetectionPreviousConnectionFailed() {
        $stack = $this->getProcessStack();
        if (count($stack) > 1) {
            $se = $stack[0];
            if (!isset($se['end']) && $se['cc'] != ZPush::COMMAND_PING) {
                // there is no end time
                ZLog::Write(LOGLEVEL_ERROR, sprintf("LoopDetection->ProcessLoopDetectionPreviousConnectionFailed() the last request of this user, " .
                                                    "command '%s' at %s with pid '%d' was terminated unexpectedly. Please check your logs for this PID and possible fatal errors, " .
                                                    "like e.g. segmentation faults. Please report your results to the Z-Push dev team.", Utils::GetCommandFromCode($se['cc']), Utils::GetFormattedTime($se['time']), $se['pid']));
            }
        }
    }

    /**
     * Inserts or updates the current process entry on the stack
     *
     * @access private
     * @return boolean
     */
    private function updateProcessStack() {
        // initialize params
        $this->InitializeParams();
        if ($this->blockMutex()) {
            $loopdata = ($this->hasData()) ? $this->getData() : array();

            // check and initialize the array structure
            $this->checkArrayStructure($loopdata, self::INTERPROCESSLD);

            $stack = $loopdata[self::$devid][self::$user][self::INTERPROCESSLD];

            // insert/update current process entry
            $nstack = array();
            $updateentry = self::GetProcessEntry();
            $found = false;

            foreach ($stack as $entry) {
                if ($entry['id'] != $updateentry['id']) {
                    $nstack[] = $entry;
                }
                else {<commit_msg>ZP-96 - changed: log message in case a previous connection did not terminate as expected<commit_after><?php
/***********************************************
* File      :   loopdetection.php
* Project   :   Z-Push
* Descr     :   detects an outgoing loop by looking
*               if subsequent requests do try to get changes
*               for the same sync key. If more than once a synckey
*               is requested, the amount of items to be sent to the mobile
*               is reduced to one. If then (again) the same synckey is
*               requested, we have most probably found the 'broken' item.
*
* Created   :   20.10.2011
*
* Copyright 2007 - 2011 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation with the following additional
* term according to sec. 7:
*
* According to sec. 7 of the GNU Affero General Public License, version 3,
* the terms of the AGPL are supplemented with the following terms:
*
* "Zarafa" is a registered trademark of Zarafa B.V.
* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
* The licensing of the Program under the AGPL does not imply a trademark license.
* Therefore any rights, title and interest in our trademarks remain entirely with us.
*
* However, if you propagate an unmodified version of the Program you are
* allowed to use the term "Z-Push" to indicate that you distribute the Program.
* Furthermore you may use our trademarks where it is necessary to indicate
* the intended purpose of a product or service provided you use it in accordance
* with honest practices in industrial or commercial matters.
* If you want to propagate modified versions of the Program under the name "Z-Push",
* you may only do so if you have a written permission by Zarafa Deutschland GmbH
* (to acquire a permission please contact Zarafa at <EMAIL>).
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Consult LICENSE file for details
************************************************/


class LoopDetection extends InterProcessData {
    const INTERPROCESSLD = "ipldkey";
    const BROKENMSGS = "bromsgs";
    static private $processident;
    static private $processentry;
    private $ignore_messageid;
    private $broken_message_uuid;
    private $broken_message_counter;


    /**
     * Constructor
     *
     * @access public
     */
    public function LoopDetection() {
        // initialize super parameters
        $this->allocate = 204800; // 200 KB
        $this->type = 1337;
        parent::__construct();

        $this->ignore_messageid = false;
    }

    /**
     * PROCESS LOOP DETECTION
     */

    /**
     * Adds the process entry to the process stack
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionInit() {
        return $this->updateProcessStack();
    }

    /**
     * Marks the process entry as termineted successfully on the process stack
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionTerminate() {
        // just to be sure that the entry is there
        self::GetProcessEntry();

        self::$processentry['end'] = time();
        ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionTerminate()");
        return $this->updateProcessStack();
    }

    /**
     * Returns a unique identifier for the internal process tracking
     *
     * @access public
     * @return string
     */
    public static function GetProcessIdentifier() {
        if (!isset(self::$processident))
            self::$processident = sprintf('%04x%04', mt_rand(0, 0xffff), mt_rand(0, 0xffff));

        return self::$processident;
    }

    /**
     * Returns a unique entry with informations about the current process
     *
     * @access public
     * @return array
     */
    public static function GetProcessEntry() {
        if (!isset(self::$processentry)) {
            self::$processentry = array();
            self::$processentry['id'] = self::GetProcessIdentifier();
            self::$processentry['pid'] = self::$pid;
            self::$processentry['time'] = self::$start;
            self::$processentry['cc'] = Request::GetCommandCode();
        }

        return self::$processentry;
    }

    /**
     * Adds an Exceptions to the process tracking
     *
     * @param Exception     $exception
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionAddException($exception) {
        // generate entry if not already there
        self::GetProcessEntry();

        if (!isset(self::$processentry['stat']))
            self::$processentry['stat'] = array();

        self::$processentry['stat'][get_class($exception)] = $exception->getCode();

        $this->updateProcessStack();
        return true;
    }

    /**
     * Adds a folderid and connected status code to the process tracking
     *
     * @param string    $folderid
     * @param int       $status
     *
     * @access public
     * @return boolean
     */
    public function ProcessLoopDetectionAddStatus($folderid, $status) {
        // generate entry if not already there
        self::GetProcessEntry();

        if ($folderid === false)
            $folderid = "hierarchy";

        if (!isset(self::$processentry['stat']))
            self::$processentry['stat'] = array();

        self::$processentry['stat'][$folderid] = $status;

        $this->updateProcessStack();

        return true;
    }

    /**
     * Indicates if a full Hierarchy Resync is necessary
     *
     * In some occasions the mobile tries to sync a folder with an invalid/not-existing ID.
     * In these cases a status exception like SYNC_STATUS_FOLDERHIERARCHYCHANGED is returned
     * so the mobile executes a FolderSync expecting that some action is taken on that folder (e.g. remove).
     *
     * If the FolderSync is not doing anything relevant, then the Sync is attempted again
     * resulting in the same error and looping between these two processes.
     *
     * This method checks if in the last process stack a Sync and FolderSync were triggered to
     * catch the loop at the 2nd interaction (Sync->FolderSync->Sync->FolderSync => ReSync)
     * Ticket: https://jira.zarafa.com/browse/ZP-5
     *
     * @access public
     * @return boolean
     *
     */
    public function ProcessLoopDetectionIsHierarchyResyncRequired() {
        $seenFailed = array();
        $seenFolderSync = false;

        $lookback = self::$start - 600; // look at the last 5 min
        foreach ($this->getProcessStack() as $se) {
            if ($se['time'] > $lookback && $se['time'] < (self::$start-1)) {
                // look for sync command
                if (isset($se['stat']) && ($se['cc'] == ZPush::COMMAND_SYNC || $se['cc'] == ZPush::COMMAND_PING)) {
                    foreach($se['stat'] as $key => $value) {
                        if (!isset($seenFailed[$key]))
                            $seenFailed[$key] = 0;
                        $seenFailed[$key]++;
                        ZLog::Write(LOGLEVEL_DEBUG, sprintf("LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): seen command with Exception or folderid '%s' and code '%s'", $key, $value ));
                    }
                }
                // look for FolderSync command with previous failed commands
                if ($se['cc'] == ZPush::COMMAND_FOLDERSYNC && !empty($seenFailed) && $se['id'] != self::GetProcessIdentifier()) {
                    // a full folderresync was already triggered
                    if (isset($se['stat']) && $se['stat']['hierarchy'] == SYNC_FSSTATUS_SYNCKEYERROR) {
                        ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): a full FolderReSync was already requested. Resetting fail counter.");
                        $seenFailed = array();
                    }
                    else {
                        $seenFolderSync = true;
                        if (!empty($seenFailed))
                            ZLog::Write(LOGLEVEL_DEBUG, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): seen FolderSync after other failing command");
                    }
                }
            }
        }

        $filtered = array();
        foreach ($seenFailed as $k => $count) {
            if ($count>1)
                $filtered[] = $k;
        }

        if ($seenFolderSync && !empty($filtered)) {
            ZLog::Write(LOGLEVEL_INFO, "LoopDetection->ProcessLoopDetectionIsHierarchyResyncRequired(): Potential loop detected. Full hierarchysync indicated.");
            return true;
        }

        return false;
    }

    /**
     * Indicates if a previous process could not be terminated
     *
     * Checks if there is an end time for the last entry on the stack
     *
     * @access public
     * @return boolean
     *
     */
    public function ProcessLoopDetectionPreviousConnectionFailed() {
        $stack = $this->getProcessStack();
        if (count($stack) > 1) {
            $se = $stack[0];
            if (!isset($se['end']) && $se['cc'] != ZPush::COMMAND_PING) {
                // there is no end time
                ZLog::Write(LOGLEVEL_ERROR, sprintf("LoopDetection->ProcessLoopDetectionPreviousConnectionFailed(): Command '%s' at %s with pid '%d' terminated unexpectedly or is still running.", Utils::GetCommandFromCode($se['cc']), Utils::GetFormattedTime($se['time']), $se['pid']));
                ZLog::Write(LOGLEVEL_ERROR, "Please check your logs for this PID and errors like PHP-Fatals or Apache segmentation faults and report your results to the Z-Push dev team.");
            }
        }
    }

    /**
     * Inserts or updates the current process entry on the stack
     *
     * @access private
     * @return boolean
     */
    private function updateProcessStack() {
        // initialize params
        $this->InitializeParams();
        if ($this->blockMutex()) {
            $loopdata = ($this->hasData()) ? $this->getData() : array();

            // check and initialize the array structure
            $this->checkArrayStructure($loopdata, self::INTERPROCESSLD);

            $stack = $loopdata[self::$devid][self::$user][self::INTERPROCESSLD];

            // insert/update current process entry
            $nstack = array();
            $updateentry = self::GetProcessEntry();
            $found = false;

            foreach ($stack as $entry) {
                if ($entry['id'] != $updateentry['id']) {
                    $nstack[] = $entry;
                }
                else {
{'fraction_non_alphanumeric': 0.11339321987992597, 'fraction_numerical': 0.004784904979009615, 'mean_word_length': 2.345009814283557, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 67, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7366946', 'n_tokens_mistral': 5863, 'n_tokens_neox': 5627, 'n_words': 1929}