<commit_before><?php

/**
 * Copyright (c) 2011, Erasmus MC
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of Erasmus MC nor the
 *      names of its contributors may be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * @package    Gems
 * @subpackage Tracker
 * @author     <NAME> <<EMAIL>>
 * @copyright  Copyright (c) 2011 Erasmus MC
 * @license    New BSD License
 * @version    $Id$
 */

/**
 * Utility class for checking and changing tokens.
 *
 * @package    Gems
 * @subpackage Tracker
 * @copyright  Copyright (c) 2011 Erasmus MC
 * @license    New BSD License
 * @since      Class available since version 1.4
 */
class Gems_Tracker_Token extends Gems_Registry_TargetAbstract
{
    const COMPLETION_NOCHANGE = 0;
    const COMPLETION_DATACHANGE = 1;
    const COMPLETION_EVENTCHANGE = 2;

    /**
     *
     * @var array Can hold any data the source likes to store for the token
     */
    private $_cache = array();

    /**
     *
     * @var array The gems token data
     */
    protected $_gemsData = array();

    /**
     *
     * @var Gems_Tracker_Token
     */
    private $_nextToken = null;

    /**
     *
     * @var Gems_Tracker_Token
     */
    private $_previousToken = null;

    /**
     *
     * @var array The answers in raw format
     */
    private $_sourceDataRaw;

    /**
     *
     * @var string The id of the token
     */
    protected $_tokenId;

    /**
     *
     * @var Zend_Db_Adapter_Abstract
     */
    protected $db;

    /**
     * True when the token does exist.
     *
     * @var boolean
     */
    public $exists = true;

    /**
     *
     * @var Gems_Tracker_RespondentTrack
     */
    protected $respTrack;

    /**
     *
     * @var Gems_Loader
     */
    protected $loader;

    /**
     *
     * @var Gems_Tracker_Survey
     */
    protected $survey;

    /**
     *
     * @var Gems_Tracker
     */
    protected $tracker;

    /**
     *
     * @var Zend_Translate_Adapter
     */
    public $translate;

    /**
     *
     * @var Gems_Util
     */
    protected $util;

    /**
     * Creates the token object
     *
     * @param mixed $gemsTokenData Token Id or array containing token record
     */
    public function __construct($gemsTokenData)
    {
        if (is_array($gemsTokenData)) {
            $this->_gemsData = $gemsTokenData;
            $this->_tokenId  = $gemsTokenData['gto_id_token'];
        } else {
            $this->_tokenId  = $gemsTokenData;
        }
    }

    /**
     * Makes sure the receptioncode data is part of the $this->_gemsData
     *
     * @param boolean $reload Optional parameter to force reload or an array with the new values.
     */
    private function _ensureReceptionCode($reload = false)
    {
        if ($reload || (! isset($this->_gemsData['grc_success']))) {
            if (is_array($reload)) {
                $this->_gemsData = $reload + $this->_gemsData;
            } else {
                $sql  = "SELECT * FROM gems__reception_codes WHERE grc_id_reception_code = ?";
                $code = $this->_gemsData['gto_reception_code'];

                if ($row = $this->db->fetchRow($sql, $code)) {
                    $this->_gemsData = $row + $this->_gemsData;
                } else {
                    $token = $this->_tokenId;
                    throw new Gems_Exception("Reception code $code is missing for token $token.");
                }
            }
        }
    }

    /**
     * Makes sure the respondent data is part of the $this->_gemsData
     */
    protected function _ensureRespondentData()
    {
        if (! isset($this->_gemsData['grs_id_user'], $this->_gemsData['gr2o_id_user'], $this->_gemsData['gco_code'])) {
            $sql = "SELECT *
                FROM gems__respondents INNER JOIN
                    gems__respondent2org ON grs_id_user = gr2o_id_user INNER JOIN
                    gems__consents ON gr2o_consent = gco_description
                WHERE gr2o_id_user = ? AND gr2o_id_organization = ? LIMIT 1";

            $respId = $this->_gemsData['gto_id_respondent'];
            $orgId  = $this->_gemsData['gto_id_organization'];

            if ($row = $this->db->fetchRow($sql, array($respId, $orgId))) {
                $this->_gemsData = $this->_gemsData + $row;
            } else {
                $token = $this->_tokenId;
                throw new Gems_Exception("Respondent data missing for token $token.");
            }
        }
    }

    /**
     * Update the token, both in the database and in memory.
     *
     * @param array $values The values that this token should be set to
     * @param int $userId The current user
     * @return int 1 if data changed, 0 otherwise
     */
    protected function _updateToken(array $values, $userId)
    {
        if ($this->tracker->filterChangesOnly($this->_gemsData, $values)) {

            if (Gems_Tracker::$verbose) {
                $echo = '';
                foreach ($values as $key => $val) {
                    $echo .= $key . ': ' . $this->_gemsData[$key] . ' => ' . $val . "\n";
                }
                MUtil_Echo::r($echo, 'Updated values for ' . $this->_tokenId);
            }

            if (! isset($values['gto_changed'])) {
                $values['gto_changed'] = new Zend_Db_Expr('CURRENT_TIMESTAMP');
            }
            if (! isset($values['gto_changed_by'])) {
                $values['gto_changed_by'] = $userId;
            }

            // Update values in this object
            $this->_gemsData = $values + (array) $this->_gemsData;

            // return 1;
            return $this->db->update('gems__tokens', $values, array('gto_id_token = ?' => $this->_tokenId));

        } else {
            return 0;
        }
    }

    /**
     * Set menu parameters from this token
     *
     * @param Gems_Menu_ParameterSource $source
     * @return Gems_Tracker_Token (continuation pattern)
     */
    public function applyToMenuSource(Gems_Menu_ParameterSource $source)
    {
        $source->setTokenId($this->_tokenId);
        if ($this->exists) {
            if (! isset($this->_gemsData['gr2o_patient_nr'])) {
                $this->_ensureRespondentData();
            }
            if (! isset($this->_gemsData['grc_success'])) {
                $this->_ensureReceptionCode();
            }

            $source->setTokenId($this->_tokenId);
            $source->setPatient($this->_gemsData['gr2o_patient_nr'], $this->_gemsData['gto_id_organization']);
            $source->setRespondentTrackId($this->_gemsData['gto_id_respondent_track']);
            $source->setTrackId($this->_gemsData['gto_id_track']);
            $source->setTrackType($this->getTrackEngine()->getTrackType());

            $source->offsetSet('gsu_id_survey', $this->_gemsData['gto_id_survey']);
            $source->offsetSet('grc_success', $this->_gemsData['grc_success']);
            $source->offsetSet('is_completed', (boolean) $this->_gemsData['gto_completion_time']);

            if ($this->_gemsData['grc_success'] &&
                    (! $this->_gemsData['gto_completion_time']) &&
                    ($validFrom = $this->getValidFrom())) {

                $validUntil = $this->getValidUntil();
                $today = new MUtil_Date();

                $can_be_taken = $validFrom->isEarlier($today) && ($validUntil ? $validUntil->isLater($today) : true);
            } else {
                $can_be_taken = false;
            }
            $source->offsetSet('can_be_taken', $can_be_taken);
        }
        return $this;
    }


    /**
     * Retrieve a certain $key from the local cache
     *
     * For speeding up things the token can hold a local cache, living as long as the
     * token object exists in memory. Sources can use this to store reusable information.
     *
     * To reset the cache on an update, the source can use the cacheReset method or the
     * setCache method to update the changed value.
     *
     * @param string $key             The key used in the cache
     * @param mixed  $defaultValue    The optional default value to use when it is not present
     * @return mixed
     */
    public function cacheGet($key, $defaultValue = null) {
        if ($this->cacheHas($key)) {
            return $this->_cache[$key];
        } else {
            return $defaultValue;
        }
    }

    /**
     * find out if a certain key is present in the cache
     *
     * @param string $key
     * @return boolean
     */
    public function cacheHas($key) {
        return isset($this->_cache[$key]);

    }

    /**
     * Reset the local cache for this token
     *
     * You can pass in an optional $key parameter to reset just that key, otherwise all
     * the cache will be reset
     *
     * @param string|null $key The key to reset
     */
    public function cacheReset($key = null) {
        if (is_null($key)) {
            $this->_cache = array();
        } else {
            unset($this->_cache[$key]);
        }
    }

    /**
     * Set a $key in the local cache
     *
     * @param string $key
     * @param mixed  $value
     */
    public function cacheSet($key, $value) {
        $this->_cache[$key] = $value;
    }

    /**
     * Returns the full url Gems should forward to after survey completion.
     *
     * This fix allows multiple sites with multiple url's to share a single
     * installation.
     *
     * @return string
     */
    protected function calculateReturnUrl()
    {
        $currentUri = $this->util->getCurrentURI();

        /*
        // Referrer would be powerful when someone is usng multiple windows, but
        // the loop does not always provide a correct referrer.
        $referrer   = $_SERVER["HTTP_REFERER"];

        // If a referrer was specified and that referral is from the current site, then use it
        // as it is more dependable when the user has multiple windows open on the application.
        if ($referrer && (0 == strncasecmp($referrer, $currentUri, strlen($currentUri)))) {
            return $referrer;
            // MUtil_Echo::track($referrer);
        } // */

        // Use the survey return if available.
        $surveyReturn = $this->loader->getCurrentUser()->getSurveyReturn();
        if ($surveyReturn) {
            // Do not show the base url as it is in $currentUri
            $surveyReturn['NoBase'] = true;

            // Add route reset to prevet the current parameters to be
            // added to the url.
            $surveyReturn['RouteReset'] = true;

            return $currentUri . MUtil_Html::urlString($surveyReturn);
            // MUtil_Echo::track($currentUri . MUtil_Html::urlString($surveyReturn));
        }

        // Ultimate backup solution for reutrn
        return $currentUri . '/ask/forward/' . MUtil_Model::REQUEST_ID . '/' . urlencode($this->getTokenId());
    }

    /**
     * Should be called after answering the request to allow the Target
     * to check if all required registry values have been set correctly.
     *
     * @return boolean False if required are missing.
     */
    public function checkRegistryRequestsAnswers()
    {
        if ($this->db && (! $this->_gemsData)) {
            $this->refresh();
        }

        return $this->exists;
    }

    /**
     * Checks whether the survey for this token was completed and processes the result
     *
     * @param int $userId The id of the gems user
     * @return int self::COMPLETION_NOCHANGE || (self::COMPLETION_DATACHANGE | self::COMPLETION_EVENTCHANGE)
     */
    public function checkTokenCompletion($userId)
    {
        $result = self::COMPLETION_NOCHANGE;

        // Some defaults
        $values['gto_completion_time'] = null;
        $values['gto_duration_in_sec'] = null;
        $values['gto_result']          = null;

        if ($this->inSource()) {
            $survey = $this->getSurvey();

            $values['gto_in_source'] = 1;

            $startTime = $survey->getStartTime($this);
            if ($startTime instanceof MUtil_Date) {
                // Value from source overrules any set date time
                $values['gto_start_time'] = $startTime->toString(Gems_Tracker::DB_DATETIME_FORMAT);

            } else {
                // Otherwise use the time kept by Gems.
                $startTime = $this->getDateTime('gto_start_time');

                //What if we have older tokens... where no gto_start_time was set??
                if (is_null($startTime)) {
                    $startTime = new MUtil_Date();
                }

                // No need to set $values['gto_start_time'], it does not change
            }

            // If there is a start date there can be a completion date
            if ($startTime instanceof MUtil_Date) {
                if ($survey->isCompleted($this)) {
                    $complTime         = $survey->getCompletionTime($this);
                    $setCompletionTime = true;

                    if (! $complTime instanceof MUtil_Date) {
                        // Token is completed but the source cannot tell the time
                        //
                        // Try to see it was stored already
                        $complTime = $this->getDateTime('gto_completion_time');

                        if ($complTime instanceof MUtil_Date) {
                            // Again no need to change a time that did not change
                            unset($values['gto_completion_time']);
                            $setCompletionTime = false;
                        } else {
                            // Well anyhow it was completed now or earlier. Get the current moment.
                            $complTime = new MUtil_Date();
                        }
                    }

                    //Set completion time for completion event
                    if ($setCompletionTime) {
                        $values['gto_completion_time'] = $complTime->toString(Gems_Tracker::DB_DATETIME_FORMAT);
                        //Save the old value
                        $originalCompletionTime = $this->_gemsData['gto_completion_time'];
                        $this->_gemsData['gto_completion_time'] = $values['gto_completion_time'];
                    }

                    // Process any Gems side survey dependent changes
                    if ($changed = $this->handleAfterCompletion()) {

                        // Communicate change
                        $result += self::COMPLETION_EVENTCHANGE;

                        if (Gems_Tracker::$verbose) {
                            MUtil_Echo::r($changed, 'Source values for ' . $this->_tokenId . ' changed by event.');
                        }
                    }

                    if ($setCompletionTime) {
                        //Reset to old value, so changes will be picked up
                        $this->_gemsData['gto_completion_time'] = $originalCompletionTime;
                    }
                    $values['gto_duration_in_sec'] = max($complTime->diffSeconds($startTime), 0);

                    //If the survey has a resultfield, store it
                    if ($resultField = $survey->getResultField()) {
                        $rawAnswers = $this->getRawAnswers();
                        if (isset($rawAnswers[$resultField])) {
                            // Cast to string, because that is the way the result is stored in the db
                            // not casting to strings means e.g. float results always result in
                            // an update, even when they did not change.
                            $values['gto_result'] = (string) $rawAnswers[$resultField];
                        }
                    }
                }
            }
        } else {
            $values['gto_in_source']  = 0;
            $values['gto_start_time'] = null;
        }

        if ($this->_updateToken($values, $userId)) {

            // Communicate change
            $result += self::COMPLETION_DATACHANGE;
        }

        return $result;
    }

    /**
     * Creates an almost exact copy of this token at the same place in the track,
     * only without answers and other source data
     *
     * Returns the new token id
     *
     * @param string $newComment Description of why the token was replaced
     * @param int $userId The current user
     * @return string The new token
     */
    public function createReplacement($newComment, $userId)
    {
        $values['gto_id_respondent_track'] = $this->_gemsData['gto_id_respondent_track'];
        $values['gto_id_round']            = $this->_gemsData['gto_id_round'];
        $values['gto_id_respondent']       = $this->_gemsData['gto_id_respondent'];
        $values['gto_id_organization']     = $this->_gemsData['gto_id_organization'];
        $values['gto_id_track']            = $this->_gemsData['gto_id_track'];
        $values['gto_id_survey']           = $this->_gemsData['gto_id_survey'];
        $values['gto_round_order']         = $this->_gemsData['gto_round_order'];
        $values['gto_round_description']   = $this->_gemsData['gto_round_description'];
        $values['gto_valid_from']          = $this->_gemsData['gto_valid_from'];
        $values['gto_valid_until']         = $this->_gemsData['gto_valid_until'];
        $values['gto_mail_sent_date']      = $this->_gemsData['gto_mail_sent_date'];
        $values['gto_next_mail_date']      = $this->_gemsData['gto_next_mail_date'];
        $values['gto_comment']             = $newComment;

        $tokenId = $this->tracker->createToken($values, $userId);

        return $tokenId;
    }

    /**
     * Returns a field from the raw answers as a date object.
     *
     * @param string $fieldName Name of answer field
     * @return MUtil_Date date time or null
     */
    public function getAnswerDateTime($fieldName)
    {
        $survey = $this->getSurvey();
        return $survey->getAnswerDateTime($fieldName, $this);
    }

    /**
     * Returns a snippet name that can be used to display the answers to this token.
     *
     * @return string
     */
    public function getAnswerSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getAnswerSnippetNames();
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns the staff or respondent id of the person
     * who last changed this token.
     *
     * @return int
     */
    public function getChangedBy()
    {
        return $this->_gemsData['gto_changed_by'];
    }

    /**
     *
     * @return MUtil_Date Completion time as a date or null
     */
    public function getCompletionTime()
    {
        if (isset($this->_gemsData['gto_completion_time']) && $this->_gemsData['gto_completion_time']) {
            return new MUtil_Date($this->_gemsData['gto_completion_time'], Gems_Tracker::DB_DATETIME_FORMAT);
        }
    }

    /**
     *
     * @return string
     */
    public function getConsentCode()
    {
        if ($this->hasSuccesCode()) {
            if (! isset($this->_gemsData['gco_code'])) {
                $this->_ensureRespondentData();
            }

            return $this->_gemsData['gco_code'];
        } else {
            return $this->util->getConsentRejected();
        }
    }

    /**
     *
     * @param string $fieldName
     * @return MUtil_Date
     */
    public function getDateTime($fieldName)
    {
        if (isset($this->_gemsData[$fieldName])) {

            if (Zend_Date::isDate($this->_gemsData[$fieldName], Gems_Tracker::DB_DATETIME_FORMAT)) {
                return new MUtil_Date($this->_gemsData[$fieldName], Gems_Tracker::DB_DATETIME_FORMAT);
            }
            if (Zend_Date::isDate($this->_gemsData[$fieldName], Gems_Tracker::DB_DATE_FORMAT)) {
                return new MUtil_Date($this->_gemsData[$fieldName], Gems_Tracker::DB_DATE_FORMAT);
            }
            if (Gems_Tracker::$verbose)  {
                MUtil_Echo::r($this->_gemsData[$fieldName], 'Missed token date value:');
            }
        }
    }

    /**
     * Returns an array of snippet names that can be used to delete this token.
     *
     * @return array of strings
     */
    public function getDeleteSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenDeleteSnippetNames($this);
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns an array of snippet names that can be used to edit this token.
     *
     * @return array of strings
     */
    public function getEditSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenEditSnippetNames($this);
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns a model that can be used to save, edit, etc. the token
     *
     * @return Gems_Tracker_Model_StandardTokenModel
     */
    public function getModel()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenModel();
        } else {
            return $this->tracker->getTokenModel();
        }
    }

    /**
     * Returns the next token in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getNextToken()
    {
        if (null === $this->_nextToken) {
            $tokenSelect = $this->tracker->getTokenSelect();
            $tokenSelect
                    ->andReceptionCodes()
                    ->forPreviousTokenId($this->_tokenId);

            if ($tokenData = $tokenSelect->fetchRow()) {
                $this->_nextToken = $this->tracker->getToken($tokenData);
                $this->_nextToken->_previousToken = $this;
            } else {
                $this->_nextToken = false;
            }
        }

        return $this->_nextToken;
    }

    /**
     * Returns the next unanswered token for the person answering this token
     *
     * @return Gems_Tracker_Token
     */
    public function getNextUnansweredToken()
    {
        $tokenSelect = $this->tracker->getTokenSelect();
        $tokenSelect
                ->andReceptionCodes()
                // ->andRespondents()
                // ->andRespondentOrganizations()
                // ->andConsents
                ->andSurveys(array())
                ->forRespondent($this->getRespondentId())
                ->forGroupId($this->getSurvey()->getGroupId())
                ->onlySucces()
                ->onlyValid()
                ->order(array('gto_valid_from', 'gto_round_order'));

        if ($tokenData = $tokenSelect->fetchRow()) {
            return $this->tracker->getToken($tokenData);
        }
    }

    /**
     *
     * @return Gems_User_Organization
     */
    public function getOrganization()
    {
        return $this->loader->getOrganization($this->getOrganizationId());
    }

    /**
     *
     * @return int
     */
    public function getOrganizationId()
    {
        return $this->_gemsData['gto_id_organization'];
    }

    /**
     *
     * @return string The respondents patient number
     */
    public function getPatientNumber()
    {
        if (! isset($this->_gemsData['gr2o_patient_nr'])) {
            $this->_ensureRespondentData();
        }

        return $this->_gemsData['gr2o_patient_nr'];
    }

    /**
     * Returns the previous token that has succes in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getPreviousSuccessToken()
    {
        $prev = $this->getPreviousToken();

        while ($prev && (! $prev->hasSuccesCode())) {
            $prev = $prev->getPreviousToken();
        }

        return $prev;
    }

    /**
     * Returns the previous token in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getPreviousToken()
    {
        if (null === $this->_previousToken) {
            $tokenSelect = $this->tracker->getTokenSelect();
            $tokenSelect
                    ->andReceptionCodes()
                    ->forNextTokenId($this->_tokenId);

            if ($tokenData = $tokenSelect->fetchRow()) {
                $this->_previousToken = $this->tracker->getToken($tokenData);
                $this->_previousToken->_nextToken = $this;
            } else {
                $this->_previousToken = false;
            }
        }

        return $this->_previousToken;
    }

    /**
     * Returns the answers in simple raw array format, without value processing etc.
     *
     * Function may return more fields than just the answers.
     *
     * @return array Field => Value array
     */
    public function getRawAnswers()
    {
        if (! is_array($this->_sourceDataRaw)) {
            $this->_sourceDataRaw = $this->getSurvey()->getRawTokenAnswerRow($this->_tokenId);
        }
        return $this->_sourceDataRaw;
    }

    /**
     * Return the Gems_Util_ReceptionCode object
     *
     * @return Gems_Util_ReceptionCode reception code
     */
    public function getReceptionCode()
    {
        return $this->util->getReceptionCode($this->_gemsData['gto_reception_code']);
    }

    /**
     * Returns the gender as a letter code
     *
     * @return string
     */
    public function getRespondentGender()
    {
        if (! isset($this->_gemsData['grs_id_user'])) {
            $this->_ensureRespondentData();
        }

        return $this->_gemsData['grs_gender'];
    }

    /**
     * Returns the gender for use as part of a sentence, e.g. Dear Mr/Mrs
     *
     * @return string
     */
    public function getRespondentGenderHello()
    {
        $greetings = $this->util->getTranslated()->getGenderGreeting();
        $gender    = $this->getRespondentGender();

        if (isset($greetings[$gender])) {
            return $greetings[$gender];
        }
    }

    /**
     *
     * @return int
     */
    public function getRespondentId()
    {
        return $this->_gemsData['gto_id_respondent'];
    }

    /**
     *
     * @return string
     */<commit_msg>Slightly better handling of not completely loaded token<commit_after><?php

/**
 * Copyright (c) 2011, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of Erasmus MC nor the
 *      names of its contributors may be used to endorse or promote products
 *      derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * @package    Gems
 * @subpackage Tracker
 * @author     <NAME> <<EMAIL>>
 * @copyright  Copyright (c) 2011 Erasmus MC
 * @license    New BSD License
 * @version    $Id$
 */

/**
 * Utility class for checking and changing tokens.
 *
 * @package    Gems
 * @subpackage Tracker
 * @copyright  Copyright (c) 2011 Erasmus MC
 * @license    New BSD License
 * @since      Class available since version 1.4
 */
class Gems_Tracker_Token extends Gems_Registry_TargetAbstract
{
    const COMPLETION_NOCHANGE = 0;
    const COMPLETION_DATACHANGE = 1;
    const COMPLETION_EVENTCHANGE = 2;

    /**
     *
     * @var array Can hold any data the source likes to store for the token
     */
    private $_cache = array();

    /**
     *
     * @var array The gems token data
     */
    protected $_gemsData = array();

    /**
     *
     * @var Gems_Tracker_Token
     */
    private $_nextToken = null;

    /**
     *
     * @var Gems_Tracker_Token
     */
    private $_previousToken = null;

    /**
     *
     * @var array The answers in raw format
     */
    private $_sourceDataRaw;

    /**
     *
     * @var string The id of the token
     */
    protected $_tokenId;

    /**
     *
     * @var Zend_Db_Adapter_Abstract
     */
    protected $db;

    /**
     * True when the token does exist.
     *
     * @var boolean
     */
    public $exists = true;

    /**
     *
     * @var Gems_Tracker_RespondentTrack
     */
    protected $respTrack;

    /**
     *
     * @var Gems_Loader
     */
    protected $loader;

    /**
     *
     * @var Gems_Tracker_Survey
     */
    protected $survey;

    /**
     *
     * @var Gems_Tracker
     */
    protected $tracker;

    /**
     *
     * @var Zend_Translate_Adapter
     */
    public $translate;

    /**
     *
     * @var Gems_Util
     */
    protected $util;

    /**
     * Creates the token object
     *
     * @param mixed $gemsTokenData Token Id or array containing token record
     */
    public function __construct($gemsTokenData)
    {
        if (is_array($gemsTokenData)) {
            $this->_gemsData = $gemsTokenData;
            $this->_tokenId  = $gemsTokenData['gto_id_token'];
        } else {
            $this->_tokenId  = $gemsTokenData;
        }
    }

    /**
     * Makes sure the receptioncode data is part of the $this->_gemsData
     *
     * @param boolean $reload Optional parameter to force reload or an array with the new values.
     */
    private function _ensureReceptionCode($reload = false)
    {
        if ($reload || (! isset($this->_gemsData['grc_success']))) {
            if (is_array($reload)) {
                $this->_gemsData = $reload + $this->_gemsData;
            } else {
                $sql  = "SELECT * FROM gems__reception_codes WHERE grc_id_reception_code = ?";
                $code = $this->_gemsData['gto_reception_code'];

                if ($row = $this->db->fetchRow($sql, $code)) {
                    $this->_gemsData = $row + $this->_gemsData;
                } else {
                    $token = $this->_tokenId;
                    throw new Gems_Exception("Reception code $code is missing for token $token.");
                }
            }
        }
    }

    /**
     * Makes sure the respondent data is part of the $this->_gemsData
     */
    protected function _ensureRespondentData()
    {
        if (! isset($this->_gemsData['grs_id_user'], $this->_gemsData['gr2o_id_user'], $this->_gemsData['gco_code'])) {
            $sql = "SELECT *
                FROM gems__respondents INNER JOIN
                    gems__respondent2org ON grs_id_user = gr2o_id_user INNER JOIN
                    gems__consents ON gr2o_consent = gco_description
                WHERE gr2o_id_user = ? AND gr2o_id_organization = ? LIMIT 1";

            $respId = $this->_gemsData['gto_id_respondent'];
            $orgId  = $this->_gemsData['gto_id_organization'];

            if ($row = $this->db->fetchRow($sql, array($respId, $orgId))) {
                $this->_gemsData = $this->_gemsData + $row;
            } else {
                $token = $this->_tokenId;
                throw new Gems_Exception("Respondent data missing for token $token.");
            }
        }
    }

    /**
     * Update the token, both in the database and in memory.
     *
     * @param array $values The values that this token should be set to
     * @param int $userId The current user
     * @return int 1 if data changed, 0 otherwise
     */
    protected function _updateToken(array $values, $userId)
    {
        if ($this->tracker->filterChangesOnly($this->_gemsData, $values)) {

            if (Gems_Tracker::$verbose) {
                $echo = '';
                foreach ($values as $key => $val) {
                    $echo .= $key . ': ' . $this->_gemsData[$key] . ' => ' . $val . "\n";
                }
                MUtil_Echo::r($echo, 'Updated values for ' . $this->_tokenId);
            }

            if (! isset($values['gto_changed'])) {
                $values['gto_changed'] = new Zend_Db_Expr('CURRENT_TIMESTAMP');
            }
            if (! isset($values['gto_changed_by'])) {
                $values['gto_changed_by'] = $userId;
            }

            // Update values in this object
            $this->_gemsData = $values + (array) $this->_gemsData;

            // return 1;
            return $this->db->update('gems__tokens', $values, array('gto_id_token = ?' => $this->_tokenId));

        } else {
            return 0;
        }
    }

    /**
     * Set menu parameters from this token
     *
     * @param Gems_Menu_ParameterSource $source
     * @return Gems_Tracker_Token (continuation pattern)
     */
    public function applyToMenuSource(Gems_Menu_ParameterSource $source)
    {
        $source->setTokenId($this->_tokenId);
        if ($this->exists) {
            if (! isset($this->_gemsData['gr2o_patient_nr'])) {
                $this->_ensureRespondentData();
            }
            if (! isset($this->_gemsData['grc_success'])) {
                $this->_ensureReceptionCode();
            }

            $source->setTokenId($this->_tokenId);
            $source->setPatient($this->_gemsData['gr2o_patient_nr'], $this->_gemsData['gto_id_organization']);
            $source->setRespondentTrackId($this->_gemsData['gto_id_respondent_track']);
            $source->setTrackId($this->_gemsData['gto_id_track']);
            $source->setTrackType($this->getTrackEngine()->getTrackType());

            $source->offsetSet('gsu_id_survey', $this->_gemsData['gto_id_survey']);
            $source->offsetSet('grc_success', $this->_gemsData['grc_success']);
            $source->offsetSet('is_completed', (boolean) $this->_gemsData['gto_completion_time']);

            if ($this->_gemsData['grc_success'] &&
                    (! $this->_gemsData['gto_completion_time']) &&
                    ($validFrom = $this->getValidFrom())) {

                $validUntil = $this->getValidUntil();
                $today = new MUtil_Date();

                $can_be_taken = $validFrom->isEarlier($today) && ($validUntil ? $validUntil->isLater($today) : true);
            } else {
                $can_be_taken = false;
            }
            $source->offsetSet('can_be_taken', $can_be_taken);
        }
        return $this;
    }


    /**
     * Retrieve a certain $key from the local cache
     *
     * For speeding up things the token can hold a local cache, living as long as the
     * token object exists in memory. Sources can use this to store reusable information.
     *
     * To reset the cache on an update, the source can use the cacheReset method or the
     * setCache method to update the changed value.
     *
     * @param string $key             The key used in the cache
     * @param mixed  $defaultValue    The optional default value to use when it is not present
     * @return mixed
     */
    public function cacheGet($key, $defaultValue = null) {
        if ($this->cacheHas($key)) {
            return $this->_cache[$key];
        } else {
            return $defaultValue;
        }
    }

    /**
     * find out if a certain key is present in the cache
     *
     * @param string $key
     * @return boolean
     */
    public function cacheHas($key) {
        return isset($this->_cache[$key]);

    }

    /**
     * Reset the local cache for this token
     *
     * You can pass in an optional $key parameter to reset just that key, otherwise all
     * the cache will be reset
     *
     * @param string|null $key The key to reset
     */
    public function cacheReset($key = null) {
        if (is_null($key)) {
            $this->_cache = array();
        } else {
            unset($this->_cache[$key]);
        }
    }

    /**
     * Set a $key in the local cache
     *
     * @param string $key
     * @param mixed  $value
     */
    public function cacheSet($key, $value) {
        $this->_cache[$key] = $value;
    }

    /**
     * Returns the full url Gems should forward to after survey completion.
     *
     * This fix allows multiple sites with multiple url's to share a single
     * installation.
     *
     * @return string
     */
    protected function calculateReturnUrl()
    {
        $currentUri = $this->util->getCurrentURI();

        /*
        // Referrer would be powerful when someone is usng multiple windows, but
        // the loop does not always provide a correct referrer.
        $referrer   = $_SERVER["HTTP_REFERER"];

        // If a referrer was specified and that referral is from the current site, then use it
        // as it is more dependable when the user has multiple windows open on the application.
        if ($referrer && (0 == strncasecmp($referrer, $currentUri, strlen($currentUri)))) {
            return $referrer;
            // MUtil_Echo::track($referrer);
        } // */

        // Use the survey return if available.
        $surveyReturn = $this->loader->getCurrentUser()->getSurveyReturn();
        if ($surveyReturn) {
            // Do not show the base url as it is in $currentUri
            $surveyReturn['NoBase'] = true;

            // Add route reset to prevet the current parameters to be
            // added to the url.
            $surveyReturn['RouteReset'] = true;

            return $currentUri . MUtil_Html::urlString($surveyReturn);
            // MUtil_Echo::track($currentUri . MUtil_Html::urlString($surveyReturn));
        }

        // Ultimate backup solution for reutrn
        return $currentUri . '/ask/forward/' . MUtil_Model::REQUEST_ID . '/' . urlencode($this->getTokenId());
    }

    /**
     * Should be called after answering the request to allow the Target
     * to check if all required registry values have been set correctly.
     *
     * @return boolean False if required are missing.
     */
    public function checkRegistryRequestsAnswers()
    {
        if ($this->db && (! $this->_gemsData)) {
            $this->refresh();
        }

        return $this->exists;
    }

    /**
     * Checks whether the survey for this token was completed and processes the result
     *
     * @param int $userId The id of the gems user
     * @return int self::COMPLETION_NOCHANGE || (self::COMPLETION_DATACHANGE | self::COMPLETION_EVENTCHANGE)
     */
    public function checkTokenCompletion($userId)
    {
        $result = self::COMPLETION_NOCHANGE;

        // Some defaults
        $values['gto_completion_time'] = null;
        $values['gto_duration_in_sec'] = null;
        $values['gto_result']          = null;

        if ($this->inSource()) {
            $survey = $this->getSurvey();

            $values['gto_in_source'] = 1;

            $startTime = $survey->getStartTime($this);
            if ($startTime instanceof MUtil_Date) {
                // Value from source overrules any set date time
                $values['gto_start_time'] = $startTime->toString(Gems_Tracker::DB_DATETIME_FORMAT);

            } else {
                // Otherwise use the time kept by Gems.
                $startTime = $this->getDateTime('gto_start_time');

                //What if we have older tokens... where no gto_start_time was set??
                if (is_null($startTime)) {
                    $startTime = new MUtil_Date();
                }

                // No need to set $values['gto_start_time'], it does not change
            }

            // If there is a start date there can be a completion date
            if ($startTime instanceof MUtil_Date) {
                if ($survey->isCompleted($this)) {
                    $complTime         = $survey->getCompletionTime($this);
                    $setCompletionTime = true;

                    if (! $complTime instanceof MUtil_Date) {
                        // Token is completed but the source cannot tell the time
                        //
                        // Try to see it was stored already
                        $complTime = $this->getDateTime('gto_completion_time');

                        if ($complTime instanceof MUtil_Date) {
                            // Again no need to change a time that did not change
                            unset($values['gto_completion_time']);
                            $setCompletionTime = false;
                        } else {
                            // Well anyhow it was completed now or earlier. Get the current moment.
                            $complTime = new MUtil_Date();
                        }
                    }

                    //Set completion time for completion event
                    if ($setCompletionTime) {
                        $values['gto_completion_time'] = $complTime->toString(Gems_Tracker::DB_DATETIME_FORMAT);
                        //Save the old value
                        $originalCompletionTime = $this->_gemsData['gto_completion_time'];
                        $this->_gemsData['gto_completion_time'] = $values['gto_completion_time'];
                    }

                    // Process any Gems side survey dependent changes
                    if ($changed = $this->handleAfterCompletion()) {

                        // Communicate change
                        $result += self::COMPLETION_EVENTCHANGE;

                        if (Gems_Tracker::$verbose) {
                            MUtil_Echo::r($changed, 'Source values for ' . $this->_tokenId . ' changed by event.');
                        }
                    }

                    if ($setCompletionTime) {
                        //Reset to old value, so changes will be picked up
                        $this->_gemsData['gto_completion_time'] = $originalCompletionTime;
                    }
                    $values['gto_duration_in_sec'] = max($complTime->diffSeconds($startTime), 0);

                    //If the survey has a resultfield, store it
                    if ($resultField = $survey->getResultField()) {
                        $rawAnswers = $this->getRawAnswers();
                        if (isset($rawAnswers[$resultField])) {
                            // Cast to string, because that is the way the result is stored in the db
                            // not casting to strings means e.g. float results always result in
                            // an update, even when they did not change.
                            $values['gto_result'] = (string) $rawAnswers[$resultField];
                        }
                    }
                }
            }
        } else {
            $values['gto_in_source']  = 0;
            $values['gto_start_time'] = null;
        }

        if ($this->_updateToken($values, $userId)) {

            // Communicate change
            $result += self::COMPLETION_DATACHANGE;
        }

        return $result;
    }

    /**
     * Creates an almost exact copy of this token at the same place in the track,
     * only without answers and other source data
     *
     * Returns the new token id
     *
     * @param string $newComment Description of why the token was replaced
     * @param int $userId The current user
     * @return string The new token
     */
    public function createReplacement($newComment, $userId)
    {
        $values['gto_id_respondent_track'] = $this->_gemsData['gto_id_respondent_track'];
        $values['gto_id_round']            = $this->_gemsData['gto_id_round'];
        $values['gto_id_respondent']       = $this->_gemsData['gto_id_respondent'];
        $values['gto_id_organization']     = $this->_gemsData['gto_id_organization'];
        $values['gto_id_track']            = $this->_gemsData['gto_id_track'];
        $values['gto_id_survey']           = $this->_gemsData['gto_id_survey'];
        $values['gto_round_order']         = $this->_gemsData['gto_round_order'];
        $values['gto_round_description']   = $this->_gemsData['gto_round_description'];
        $values['gto_valid_from']          = $this->_gemsData['gto_valid_from'];
        $values['gto_valid_until']         = $this->_gemsData['gto_valid_until'];
        $values['gto_mail_sent_date']      = $this->_gemsData['gto_mail_sent_date'];
        $values['gto_next_mail_date']      = $this->_gemsData['gto_next_mail_date'];
        $values['gto_comment']             = $newComment;

        $tokenId = $this->tracker->createToken($values, $userId);

        return $tokenId;
    }

    /**
     * Returns a field from the raw answers as a date object.
     *
     * @param string $fieldName Name of answer field
     * @return MUtil_Date date time or null
     */
    public function getAnswerDateTime($fieldName)
    {
        $survey = $this->getSurvey();
        return $survey->getAnswerDateTime($fieldName, $this);
    }

    /**
     * Returns a snippet name that can be used to display the answers to this token.
     *
     * @return string
     */
    public function getAnswerSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getAnswerSnippetNames();
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns the staff or respondent id of the person
     * who last changed this token.
     *
     * @return int
     */
    public function getChangedBy()
    {
        return $this->_gemsData['gto_changed_by'];
    }

    /**
     *
     * @return MUtil_Date Completion time as a date or null
     */
    public function getCompletionTime()
    {
        if (isset($this->_gemsData['gto_completion_time']) && $this->_gemsData['gto_completion_time']) {
            return new MUtil_Date($this->_gemsData['gto_completion_time'], Gems_Tracker::DB_DATETIME_FORMAT);
        }
    }

    /**
     *
     * @return string
     */
    public function getConsentCode()
    {
        if ($this->hasSuccesCode()) {
            if (! isset($this->_gemsData['gco_code'])) {
                $this->_ensureRespondentData();
            }

            return $this->_gemsData['gco_code'];
        } else {
            return $this->util->getConsentRejected();
        }
    }

    /**
     *
     * @param string $fieldName
     * @return MUtil_Date
     */
    public function getDateTime($fieldName)
    {
        if (isset($this->_gemsData[$fieldName])) {

            if (Zend_Date::isDate($this->_gemsData[$fieldName], Gems_Tracker::DB_DATETIME_FORMAT)) {
                return new MUtil_Date($this->_gemsData[$fieldName], Gems_Tracker::DB_DATETIME_FORMAT);
            }
            if (Zend_Date::isDate($this->_gemsData[$fieldName], Gems_Tracker::DB_DATE_FORMAT)) {
                return new MUtil_Date($this->_gemsData[$fieldName], Gems_Tracker::DB_DATE_FORMAT);
            }
            if (Gems_Tracker::$verbose)  {
                MUtil_Echo::r($this->_gemsData[$fieldName], 'Missed token date value:');
            }
        }
    }

    /**
     * Returns an array of snippet names that can be used to delete this token.
     *
     * @return array of strings
     */
    public function getDeleteSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenDeleteSnippetNames($this);
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns an array of snippet names that can be used to edit this token.
     *
     * @return array of strings
     */
    public function getEditSnippetNames()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenEditSnippetNames($this);
        } else {
            return 'TokenNotFoundSnippet';
        }
    }

    /**
     * Returns a model that can be used to save, edit, etc. the token
     *
     * @return Gems_Tracker_Model_StandardTokenModel
     */
    public function getModel()
    {
        if ($this->exists) {
            return $this->getTrackEngine()->getTokenModel();
        } else {
            return $this->tracker->getTokenModel();
        }
    }

    /**
     * Returns the next token in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getNextToken()
    {
        if (null === $this->_nextToken) {
            $tokenSelect = $this->tracker->getTokenSelect();
            $tokenSelect
                    ->andReceptionCodes()
                    ->forPreviousTokenId($this->_tokenId);

            if ($tokenData = $tokenSelect->fetchRow()) {
                $this->_nextToken = $this->tracker->getToken($tokenData);
                $this->_nextToken->_previousToken = $this;
            } else {
                $this->_nextToken = false;
            }
        }

        return $this->_nextToken;
    }

    /**
     * Returns the next unanswered token for the person answering this token
     *
     * @return Gems_Tracker_Token
     */
    public function getNextUnansweredToken()
    {
        $tokenSelect = $this->tracker->getTokenSelect();
        $tokenSelect
                ->andReceptionCodes()
                // ->andRespondents()
                // ->andRespondentOrganizations()
                // ->andConsents
                ->andSurveys(array())
                ->forRespondent($this->getRespondentId())
                ->forGroupId($this->getSurvey()->getGroupId())
                ->onlySucces()
                ->onlyValid()
                ->order(array('gto_valid_from', 'gto_round_order'));

        if ($tokenData = $tokenSelect->fetchRow()) {
            return $this->tracker->getToken($tokenData);
        }
    }

    /**
     *
     * @return Gems_User_Organization
     */
    public function getOrganization()
    {
        return $this->loader->getOrganization($this->getOrganizationId());
    }

    /**
     *
     * @return int
     */
    public function getOrganizationId()
    {
        return $this->_gemsData['gto_id_organization'];
    }

    /**
     *
     * @return string The respondents patient number
     */
    public function getPatientNumber()
    {
        if (! isset($this->_gemsData['gr2o_patient_nr'])) {
            $this->_ensureRespondentData();
        }

        return $this->_gemsData['gr2o_patient_nr'];
    }

    /**
     * Returns the previous token that has succes in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getPreviousSuccessToken()
    {
        $prev = $this->getPreviousToken();

        while ($prev && (! $prev->hasSuccesCode())) {
            $prev = $prev->getPreviousToken();
        }

        return $prev;
    }

    /**
     * Returns the previous token in this track
     *
     * @return Gems_Tracker_Token
     */
    public function getPreviousToken()
    {
        if (null === $this->_previousToken) {
            $tokenSelect = $this->tracker->getTokenSelect();
            $tokenSelect
                    ->andReceptionCodes()
                    ->forNextTokenId($this->_tokenId);

            if ($tokenData = $tokenSelect->fetchRow()) {
                $this->_previousToken = $this->tracker->getToken($tokenData);
                $this->_previousToken->_nextToken = $this;
            } else {
                $this->_previousToken = false;
            }
        }

        return $this->_previousToken;
    }

    /**
     * Returns the answers in simple raw array format, without value processing etc.
     *
     * Function may return more fields than just the answers.
     *
     * @return array Field => Value array
     */
    public function getRawAnswers()
    {
        if (! is_array($this->_sourceDataRaw)) {
            $this->_sourceDataRaw = $this->getSurvey()->getRawTokenAnswerRow($this->_tokenId);
        }
        return $this->_sourceDataRaw;
    }

    /**
     * Return the Gems_Util_ReceptionCode object
     *
     * @return Gems_Util_ReceptionCode reception code
     */
    public function getReceptionCode()
    {
        return $this->util->getReceptionCode($this->_gemsData['gto_reception_code']);
    }

    /**
     * Returns the gender as a letter code
     *
     * @return string
     */
    public function getRespondentGender()
    {
        if (! isset($this->_gemsData['grs_id_user'])) {
            $this->_ensureRespondentData();
        }

        return $this->_gemsData['grs_gender'];
    }

    /**
     * Returns the gender for use as part of a sentence, e.g. Dear Mr/Mrs
     *
     * @return string
     */
    public function getRespondentGenderHello()
    {
        $greetings = $this->util->getTranslated()->getGenderGreeting();
        $gender    = $this->getRespondentGender();

        if (isset($greetings[$gender])) {
            return $greetings[$gender];
        }
    }

    /**
     *
     * @return int
     */
    public function getRespondentId()
    {
        if (array_key_exists('gto_id_respondent', $this->_gemsData)) {
            return $this->_gemsData['gto_id_respondent'];
        } else {
            throw new Gems_Exception('Token not loaded correctly');
        }
    }

    /**
     *
     * @return string
     */
{'fraction_non_alphanumeric': 0.11435288114589788, 'fraction_numerical': 0.001312096803586398, 'mean_word_length': 1.8896787783043707, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 457, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '99430', 'n_tokens_mistral': 15625, 'n_tokens_neox': 14810, 'n_words': 3828}