<commit_before><?php
/**
 * @package plugins.elasticSearch
 * @subpackage lib
 */
class kElasticSearchManager implements kObjectReadyForIndexEventConsumer, kObjectReadyForElasticIndexEventConsumer, kObjectUpdatedEventConsumer, kObjectAddedEventConsumer, kObjectChangedEventConsumer
{

    const CACHE_PREFIX = 'executed_elastic_cluster_';
    const MAX_LENGTH = 32766;
    const MAX_CUE_POINTS = 5000;
    const MAX_SQL_LENGTH = 131072;// 128 * 1024
    const CACHE_PREFIX_STICKY_SESSIONS = 'elastic_large_sql_lock_';
    const REPETITIVE_UPDATES_CONFIG_KEY = 'skip_elastic_repetitive_updates';
    const METADATA_MAX_LENGTH = 256;

    /**
     * @param BaseObject $object
     * @param BatchJob $raisedJob
     * @return bool true if should continue to the next consumer
     */
    public function objectReadyForIndex(BaseObject $object, BatchJob $raisedJob = null)
    {
        if($object instanceof CuePoint && in_array($object->getType(), CuePointPlugin::getElasticIndexOnEntryTypes()) && !in_array($object->getType(), CuePointPlugin::getIndexOnEntryTypes()))
        {
            kCuePointManager::reIndexCuePointEntry($object, false, true);//reindex the entry only on elastic
        }

        if($object instanceof IElasticIndexable)
        {
            if($object->shouldDeleteFromElastic())
                $this->deleteFromElastic($object);
            else
                $this->saveToElastic($object);
        }

        return true;
    }

    /**
     * @param BaseObject $object
     * @return bool true if the consumer should handle the event
     */
    public function shouldConsumeReadyForIndexEvent(BaseObject $object)
    {
        if($object instanceof CuePoint && in_array($object->getType(), CuePointPlugin::getElasticIndexOnEntryTypes()) && !in_array($object->getType(), CuePointPlugin::getIndexOnEntryTypes()))
            return true;

        if($object instanceof IElasticIndexable)
            return true;

        return false;
    }

    public function saveToElastic(IElasticIndexable $object ,$params = null)
    {
        if(kConf::get('disableElastic', 'elastic', true))
            return true;

        $skipSave = $this->checkRepetitiveUpdates($object);
        if($skipSave)
            return true;

        $cmd = $this->getElasticSaveParams($object, $params);

        if(!$cmd)
            return true;

        return $this->execElastic($cmd, $object, $object->getElasticSaveMethod());
    }

    private function checkRepetitiveUpdates($object)
    {
        $className = get_class($object);
        $objectId = $object->getId();

        // track repetitive updates of the same object (e.g. adding many annotations which cause updates of the entry)
        // once hitting a threshold, we will avoid more than one update per minute
        // threshold is defined by kConf parameter skip_elastic_repetitive_updates using lowercase of {className}_{service}_{action}={threshold}
        $saveCounter = 0;
        $cache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
        if ($cache)
        {
            $cacheKey = "ELSSave:$className:$objectId";
            $cache->add($cacheKey, 0, 60);
            $saveCounter = $cache->increment($cacheKey);
        }
	
		list($skipElasticRepetitiveUpdatesValue, $matchKey) = kSearchUtils::getSkipRepetitiveUpdatesValue(self::REPETITIVE_UPDATES_CONFIG_KEY, $className);
		$skipSave = isset($skipElasticRepetitiveUpdatesValue) && $saveCounter > $skipElasticRepetitiveUpdatesValue;
	
		if($skipSave)
		{
			KalturaLog::debug("Skipping save elastic for object [$className] [$objectId] count [$saveCounter] max allowed [$skipElasticRepetitiveUpdatesValue] with match key [$matchKey]");
			return true;
		}
	<commit_msg>No-Plat: Support new type of keys when blocking search engine repetitive updates<commit_after><?php
/**
 * @package plugins.elasticSearch
 * @subpackage lib
 */
class kElasticSearchManager implements kObjectReadyForIndexEventConsumer, kObjectReadyForElasticIndexEventConsumer, kObjectUpdatedEventConsumer, kObjectAddedEventConsumer, kObjectChangedEventConsumer
{

    const CACHE_PREFIX = 'executed_elastic_cluster_';
    const MAX_LENGTH = 32766;
    const MAX_CUE_POINTS = 5000;
    const MAX_SQL_LENGTH = 131072;// 128 * 1024
    const CACHE_PREFIX_STICKY_SESSIONS = 'elastic_large_sql_lock_';
    const REPETITIVE_UPDATES_CONFIG_KEY = 'skip_elastic_repetitive_updates';
    const METADATA_MAX_LENGTH = 256;

    /**
     * @param BaseObject $object
     * @param BatchJob $raisedJob
     * @return bool true if should continue to the next consumer
     */
    public function objectReadyForIndex(BaseObject $object, BatchJob $raisedJob = null)
    {
        if($object instanceof CuePoint && in_array($object->getType(), CuePointPlugin::getElasticIndexOnEntryTypes()) && !in_array($object->getType(), CuePointPlugin::getIndexOnEntryTypes()))
        {
            kCuePointManager::reIndexCuePointEntry($object, false, true);//reindex the entry only on elastic
        }

        if($object instanceof IElasticIndexable)
        {
            if($object->shouldDeleteFromElastic())
                $this->deleteFromElastic($object);
            else
                $this->saveToElastic($object);
        }

        return true;
    }

    /**
     * @param BaseObject $object
     * @return bool true if the consumer should handle the event
     */
    public function shouldConsumeReadyForIndexEvent(BaseObject $object)
    {
        if($object instanceof CuePoint && in_array($object->getType(), CuePointPlugin::getElasticIndexOnEntryTypes()) && !in_array($object->getType(), CuePointPlugin::getIndexOnEntryTypes()))
            return true;

        if($object instanceof IElasticIndexable)
            return true;

        return false;
    }

    public function saveToElastic(IElasticIndexable $object ,$params = null)
    {
        if(kConf::get('disableElastic', 'elastic', true))
            return true;

        $skipSave = $this->checkRepetitiveUpdates($object);
        if($skipSave)
            return true;

        $cmd = $this->getElasticSaveParams($object, $params);

        if(!$cmd)
            return true;

        return $this->execElastic($cmd, $object, $object->getElasticSaveMethod());
    }

    private function checkRepetitiveUpdates($object)
    {
        $className = get_class($object);
        $objectId = $object->getId();

        // track repetitive updates of the same object (e.g. adding many annotations which cause updates of the entry)
        // once hitting a threshold, we will avoid more than one update per minute
        // threshold is defined by kConf parameter skip_elastic_repetitive_updates using lowercase of {className}_{service}_{action}={threshold}
        $saveCounter = 0;
        $cache = kCacheManager::getSingleLayerCache(kCacheManager::CACHE_TYPE_LOCK_KEYS);
        if ($cache)
        {
            $cacheKey = "ELSSave:$className:$objectId";
            $cache->add($cacheKey, 0, 60);
            $saveCounter = $cache->increment($cacheKey);
        }
	
        list($skipElasticRepetitiveUpdatesValue, $matchKey) = kSearchUtils::getSkipRepetitiveUpdatesValue(self::REPETITIVE_UPDATES_CONFIG_KEY, $className);
        $skipSave = isset($skipElasticRepetitiveUpdatesValue) && $saveCounter > $skipElasticRepetitiveUpdatesValue;

        if($skipSave)
        {
            KalturaLog::debug("Skipping save elastic for object [$className] [$objectId] count [$saveCounter] max allowed [$skipElasticRepetitiveUpdatesValue] with match key [$matchKey]");
            return true;
        }
	
{'fraction_non_alphanumeric': 0.10265859436693867, 'fraction_numerical': 0.007633587786259542, 'mean_word_length': 3.337328767123288, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4741336', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2094, 'n_words': 471}