<commit_before><?php

class SV_UserActivity_Model extends XenForo_Model
{
    const SAMPLE_INTERVAL = 30;

    static $handlers = array();

    public function registerHandler($controllerName, $contentType, $contentIdField)
    {
        self::$handlers[$controllerName] = array($contentType, $contentIdField);
    }

    public function getHandler($controllerName)
    {
        if (empty(self::$handlers[$controllerName]))
        {
            return false;
        }
        return self::$handlers[$controllerName];
    }

    public function GarbageCollectActivity($targetRunTime = null)
    {
        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return;
        }

        $options = XenForo_Application::getOptions();
        $onlineStatusTimeout = $options->onlineStatusTimeout * 60;
        // we need to manually expire records out of the per content hash set if they are kept alive with activity
        $gckey = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activityGC";
        $datakey = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.";

        $end = XenForo_Application::$time - $onlineStatusTimeout;
        $end = $end - ($end  % self::SAMPLE_INTERVAL);

        $keys = array();
        $loopGuard = 10000;
        $s = microtime(true);
        while($loopGuard > 0)
        {
            // randomly grab a key to inspect
            $contentKeyPart = $credis->spop($gckey);
            if (empty($contentKeyPart))
            {
                break;
            }
            // the actual prune operation
            $fullkey = $datakey.$contentKeyPart;
            $credis->zremrangebyscore($fullkey, 0, $end);
            // don't matter about a race condition, as they will have added it back into the set
            if ($credis->zcard($fullkey))
            {
                $keys[] = $contentKeyPart;
            }

            $runTime = microtime(true) - $s;
            if ($targetRunTime && $runTime > $targetRunTime)
            {
                break;
            }
            $loopGuard--;
        }

        // add the keys back in after we have finished inspecting to prevent hitting the same keys we have hit before.
        if ($keys)
        {
            $result = $credis->sadd($gckey, $keys);
            $credis->expire($gckey, $onlineStatusTimeout);
        }
    }

    public function updateSessionActivity($contentType, $contentId, $ip, $robotKey, array $viewingUser = null)
    {
        $this->standardizeViewingUserReference($viewingUser);

        $score = XenForo_Application::$time - ( XenForo_Application::$time  % self::SAMPLE_INTERVAL);
        $data = array
        (
            'user_id' => $viewingUser['user_id'],
            'username' => $viewingUser['username'],
            'visible' => $viewingUser['visible'] && $viewingUser['activity_visible'],
            'robot'  => empty($robotKey) ? 0 : 1,
        );

        $options = XenForo_Application::getOptions();
        if ($viewingUser['user_id'])
        {
            if ($options->RainDD_UA_ThreadViewType == 0)
            {
                $data['display_style_group_id'] = $viewingUser['display_style_group_id'];
            }
            else if ($options->RainDD_UA_ThreadViewType == 1)
            {
                $data['gender'] = $viewingUser['gender'];
                $data['avatar_date'] = $viewingUser['avatar_date'];
                $data['gravatar'] = $viewingUser['gravatar'];
            }
            else
            {
                // unknown display type
                return;
            }
        }
        else
        {
            $data['ip'] = $ip;
        }

        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return;
        }

        // record keeping
        $key = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.{$contentType}.{$contentId}";
        $result = $credis->zadd($key, $score, json_encode($data));
        $credis->expire($key, $options->onlineStatusTimeout * 60);

        // we need to manually expire records out of the per content hash set if they are kept alive with activity
        $key = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activityGC";
        $result = $credis->sadd($key, "{$contentType}.{$contentId}");
        $credis->expire($key, $options->onlineStatusTimeout * 60);
    }

    public function getUsersViewing($contentType, $contentId, array $viewingUser = null)
    {
        $this->standardizeViewingUserReference($viewingUser);

        $memberCount = 1;
        $guestCount = 0;
        $robotCount = 0;
        $records = array($viewingUser);

        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return null;
        }
        else
        {
            $key =  Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.{$contentType}.{$contentId}";

            $options = XenForo_Application::getOptions();
            $start = XenForo_Application::$time  - $options->onlineStatusTimeout * 60;
            $start = $start - ($start  % self::SAMPLE_INTERVAL);
            $end = XenForo_Application::$time + 1;
            $onlineRecords = $credis->zrevrangebyscore($key, $end, $start, array('withscores' => true));
            // check if the activity counter needs pruning
            if (mt_rand() < $options->UA_pruneChance)
            {
                if ($credis->zcard($key) >= count($onlineRecords) * $options->UA_fillFactor)
                {
                    $credis->zremrangebyscore($key, 0, $start - 1);
                }
            }
        }

        if(is_array($onlineRecords))
        {
            $seen = array($viewingUser['user_id'] => true);
            $bypassUserPrivacy = $this->_getUserModel()->canBypassUserPrivacy($null, $viewingUser);

            foreach($onlineRecords as $rec => $score)
            {
                $rec = json_decode($rec, true);
                if ($rec['user_id'])
                {
                    if (empty($seen[$rec['user_id']]))
                    {
                        $seen[$rec['user_id']] = true;
                        $memberCount += 1;
                        if(!empty($rec['visible']) || $bypassUserPrivacy)
                        {
                            $score = $score - ($score % self::SAMPLE_INTERVAL);
                            $rec['effective_last_activity'] = $score;
                            $records[] = $rec;
                        }
                    }
                }
                else if (empty($rec['robot']))
                {
                    $guestCount += 1;
                }
                else
                {
                    $robotCount += 1;
                }
            }
        }

        return array
        (
            'members' => $memberCount,
            'guests'  => $guestCount,<commit_msg>Use a more compact storage format rather than json.<commit_after><?php

class SV_UserActivity_Model extends XenForo_Model
{
    const SAMPLE_INTERVAL = 30;

    static $handlers = array();

    public function registerHandler($controllerName, $contentType, $contentIdField)
    {
        self::$handlers[$controllerName] = array($contentType, $contentIdField);
    }

    public function getHandler($controllerName)
    {
        if (empty(self::$handlers[$controllerName]))
        {
            return false;
        }
        return self::$handlers[$controllerName];
    }

    public function GarbageCollectActivity($targetRunTime = null)
    {
        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return;
        }

        $options = XenForo_Application::getOptions();
        $onlineStatusTimeout = $options->onlineStatusTimeout * 60;
        // we need to manually expire records out of the per content hash set if they are kept alive with activity
        $gckey = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activityGC";
        $datakey = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.";

        $end = XenForo_Application::$time - $onlineStatusTimeout;
        $end = $end - ($end  % self::SAMPLE_INTERVAL);

        $keys = array();
        $loopGuard = 10000;
        $s = microtime(true);
        while($loopGuard > 0)
        {
            // randomly grab a key to inspect
            $contentKeyPart = $credis->spop($gckey);
            if (empty($contentKeyPart))
            {
                break;
            }
            // the actual prune operation
            $fullkey = $datakey.$contentKeyPart;
            $credis->zremrangebyscore($fullkey, 0, $end);
            // don't matter about a race condition, as they will have added it back into the set
            if ($credis->zcard($fullkey))
            {
                $keys[] = $contentKeyPart;
            }

            $runTime = microtime(true) - $s;
            if ($targetRunTime && $runTime > $targetRunTime)
            {
                break;
            }
            $loopGuard--;
        }

        // add the keys back in after we have finished inspecting to prevent hitting the same keys we have hit before.
        if ($keys)
        {
            $result = $credis->sadd($gckey, $keys);
            $credis->expire($gckey, $onlineStatusTimeout);
        }
    }

    public function updateSessionActivity($contentType, $contentId, $ip, $robotKey, array $viewingUser = null)
    {
        $this->standardizeViewingUserReference($viewingUser);

        $score = XenForo_Application::$time - ( XenForo_Application::$time  % self::SAMPLE_INTERVAL);
        $data = array
        (
            'user_id' => $viewingUser['user_id'],
            'username' => $viewingUser['username'],
            'visible' => $viewingUser['visible'] && $viewingUser['activity_visible'] ? 1 : null,
            'robot'  => empty($robotKey) ? null : 1,
            'display_style_group_id' => null,
            'gender' => null,
            'avatar_date' => null,
            'gravatar' => null,
            'ip' => null,
        );

        $options = XenForo_Application::getOptions();
        if ($viewingUser['user_id'])
        {
            if ($options->RainDD_UA_ThreadViewType == 0)
            {
                $data['display_style_group_id'] = $viewingUser['display_style_group_id'];
            }
            else if ($options->RainDD_UA_ThreadViewType == 1)
            {
                $data['gender'] = $viewingUser['gender'];
                $data['avatar_date'] = $viewingUser['avatar_date'];
                $data['gravatar'] = $viewingUser['gravatar'];
            }
            else
            {
                // unknown display type
                return;
            }
        }
        else
        {
            $data['ip'] = $ip;
        }

        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return;
        }

        // encode the data
        $raw = implode("\n", $data);

        // record keeping
        $key = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.{$contentType}.{$contentId}";
        $result = $credis->zadd($key, $score, $raw);
        $credis->expire($key, $options->onlineStatusTimeout * 60);

        // we need to manually expire records out of the per content hash set if they are kept alive with activity
        $key = Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activityGC";
        $result = $credis->sadd($key, "{$contentType}.{$contentId}");
        $credis->expire($key, $options->onlineStatusTimeout * 60);
    }

    public function getUsersViewing($contentType, $contentId, array $viewingUser = null)
    {
        $this->standardizeViewingUserReference($viewingUser);

        $memberCount = 1;
        $guestCount = 0;
        $robotCount = 0;
        $records = array($viewingUser);

        $registry = $this->_getDataRegistryModel();
        $cache = $this->_getCache(true);
        if (!method_exists($registry, 'getCredis') || !($credis = $registry->getCredis($cache)))
        {
            // do not have a fallback
            return null;
        }
        else
        {
            $key =  Cm_Cache_Backend_Redis::PREFIX_KEY. $cache->getOption('cache_id_prefix') . "activity.{$contentType}.{$contentId}";

            $options = XenForo_Application::getOptions();
            $start = XenForo_Application::$time  - $options->onlineStatusTimeout * 60;
            $start = $start - ($start  % self::SAMPLE_INTERVAL);
            $end = XenForo_Application::$time + 1;
            $onlineRecords = $credis->zrevrangebyscore($key, $end, $start, array('withscores' => true));
            // check if the activity counter needs pruning
            if (mt_rand() < $options->UA_pruneChance)
            {
                if ($credis->zcard($key) >= count($onlineRecords) * $options->UA_fillFactor)
                {
                    $credis->zremrangebyscore($key, 0, $start - 1);
                }
            }
        }

        $keys = array
        (
            'user_id',
            'username',
            'visible',
            'robot',
            'display_style_group_id',
            'gender',
            'avatar_date',
            'gravatar',
            'ip',
        );

        if(is_array($onlineRecords))
        {
            $seen = array($viewingUser['user_id'] => true);
            $bypassUserPrivacy = $this->_getUserModel()->canBypassUserPrivacy($null, $viewingUser);

            foreach($onlineRecords as $rec => $score)
            {
                $data = explode("\n", $rec);
                $rec = @array_combine($keys, $data);
                if (empty($rec))
                {
                    continue;
                }
                if ($rec['user_id'])
                {
                    if (empty($seen[$rec['user_id']]))
                    {
                        $seen[$rec['user_id']] = true;
                        $memberCount += 1;
                        if(!empty($rec['visible']) || $bypassUserPrivacy)
                        {
                            $score = $score - ($score % self::SAMPLE_INTERVAL);
                            $rec['effective_last_activity'] = $score;
                            $records[] = $rec;
                        }
                    }
                }
                else if (empty($rec['robot']))
                {
                    $guestCount += 1;
                }
                else
                {
                    $robotCount += 1;
                }
            }
        }

        return array
        (
            'members' => $memberCount,
            'guests'  => $guestCount,
{'fraction_non_alphanumeric': 0.1321103254984394, 'fraction_numerical': 0.0038218994840435697, 'mean_word_length': 1.7390090718771807, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 108, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '919824', 'n_tokens_mistral': 4478, 'n_tokens_neox': 4310, 'n_words': 848}