<commit_before><?php
/**
 * Enable transcript assets management for entry objects
 * @package plugins.transcript
 */
class TranscriptPlugin extends KalturaPlugin implements IKalturaEnumerator, IKalturaObjectLoader, IKalturaPending, IKalturaTypeExtender, IKalturaSearchDataContributor
{
	const PLUGIN_NAME = 'transcript';
	const ENTRY_TRANSCRIPT_PREFIX = 'tr_pref';
	const ENTRY_TRANSCRIPT_SUFFIX = 'tr_suf';
	const SEARCH_TEXT_SUFFIX = 'trend';
	const PLUGINS_DATA = 'plugins_data';
	public static $newContent;


    /* (non-PHPdoc)
     * @see IKalturaPlugin::getPluginName()
     */
	public static function getPluginName()
	{
		return self::PLUGIN_NAME;
	}
	
	/* (non-PHPdoc)
	 * @see IKalturaEnumerator::getEnums()
	 */
	public static function getEnums($baseEnumName = null)
	{
		if(is_null($baseEnumName))
			return array('TranscriptAssetType');
	
		if($baseEnumName == 'assetType')
			return array('TranscriptAssetType');
			
		return array();
	}
	
    /* (non-PHPdoc)
     * @see IKalturaTypeExtender::getExtendedTypes()
     */
     public static function getExtendedTypes($baseClass, $enumValue)
     {
        if($baseClass == assetPeer::OM_CLASS && $enumValue == AttachmentPlugin::getAssetTypeCoreValue(AttachmentAssetType::ATTACHMENT))
        {
            return array(
                self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT)
            );
        }

        return null;
    }

	/* (non-PHPdoc)
	 * @see IKalturaObjectLoader::loadObject()
	 */
	public static function loadObject($baseClass, $enumValue, array $constructorArgs = null)
	{
		if($baseClass == 'KalturaAsset' && $enumValue == self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT))
			return new KalturaTranscriptAsset();
	
		return null;
	}

	/* (non-PHPdoc)
	 * @see IKalturaObjectLoader::getObjectClass()
	 */
	public static function getObjectClass($baseClass, $enumValue)
	{
		if($baseClass == 'asset' && $enumValue == self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT))
			return 'TranscriptAsset';
	
		return null;
	}

	/* (non-PHPdoc)
	 * @see IKalturaPending::dependsOn()
	 */
	public static function dependsOn()
	{
		$dependency = new KalturaDependency(AttachmentPlugin::getPluginName());
		return array($dependency);
	}

	/**
	 * @return int id of dynamic enum in the DB.
	 */
	public static function getAssetTypeCoreValue($valueName)
	{
		$value = self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName;
		return kPluginableEnumsManager::apiToCore('assetType', $value);
	}
	
	/**
	 * @return string external API value of dynamic enum.
	 */
	public static function getApiValue($valueName)
	{
		return self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName;
	}

	/* (non-PHPdoc)
 * @see IKalturaSearchDataContributor::getSearchData()
 */
	public static function getSearchData(BaseObject $object)
	{
		if($object instanceof entry)
			return self::getTranscriptSearchData($object);

		return null;
	}

	public static function getValues($item, $key)
	{
		if ($key === 'value')
		{
			self::$newContent .= ($item . ' ');
		}
	}

	public static function getTranscriptSearchData(entry $entry)
	{
		$transcriptAssets = assetPeer::retrieveByEntryId($entry->getId(), array(TranscriptPlugin::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT)));
		if(!$transcriptAssets || !count($transcriptAssets))
			return null;

		$data = array();
		foreach($transcriptAssets as $transcriptAsset)
		{
			/* @var $transcriptAsset TranscriptAsset */

			if($transcriptAsset->getContainerFormat() != AttachmentType::TEXT)
				continue;

			$syncKey = $transcriptAsset->getSyncKey(asset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
			$fileContent = kFileSyncUtils::file_get_contents($syncKey, true, false);
			$decoded = json_decode($fileContent, true);
			if( json_last_error() == JSON_ERROR_NONE ) //handling Json files
			{
				self::$newContent = '';
				array_walk_recursive($decoded, array('TranscriptPlugin', 'getValues'));
				$content = self::$newContent;
			}
			else // handling text files
			{
				$content = $fileContent;
			}
			if(!$content)
				continue;

			$content = trim(preg_replace('/\s+/', ' ', $content));
			if(!$content)
				continue;

			$data[] = $transcriptAsset->getId() . ' ' . self::ENTRY_TRANSCRIPT_PREFIX  . ' ' . $content . ' ' . self::ENTRY_TRANSCRIPT_SUFFIX;
		}

		$searchValues = array(
			self::PLUGINS_DATA => self::PLUGIN_NAME . ' ' . implode(' ', $data) . ' ' . self::SEARCH_TEXT_SUFFIX
		);

		return $searchValues;
	}
	
}<commit_msg>PLAT-9738: Dont use json_decode to avoid memory exhaustion<commit_after><?php
/**
 * Enable transcript assets management for entry objects
 * @package plugins.transcript
 */
class TranscriptPlugin extends KalturaPlugin implements IKalturaEnumerator, IKalturaObjectLoader, IKalturaPending, IKalturaTypeExtender, IKalturaSearchDataContributor
{
	const PLUGIN_NAME = 'transcript';
	const ENTRY_TRANSCRIPT_PREFIX = 'tr_pref';
	const ENTRY_TRANSCRIPT_SUFFIX = 'tr_suf';
	const SEARCH_TEXT_SUFFIX = 'trend';
	const PLUGINS_DATA = 'plugins_data';


    /* (non-PHPdoc)
     * @see IKalturaPlugin::getPluginName()
     */
	public static function getPluginName()
	{
		return self::PLUGIN_NAME;
	}
	
	/* (non-PHPdoc)
	 * @see IKalturaEnumerator::getEnums()
	 */
	public static function getEnums($baseEnumName = null)
	{
		if(is_null($baseEnumName))
			return array('TranscriptAssetType');
	
		if($baseEnumName == 'assetType')
			return array('TranscriptAssetType');
			
		return array();
	}
	
    /* (non-PHPdoc)
     * @see IKalturaTypeExtender::getExtendedTypes()
     */
     public static function getExtendedTypes($baseClass, $enumValue)
     {
        if($baseClass == assetPeer::OM_CLASS && $enumValue == AttachmentPlugin::getAssetTypeCoreValue(AttachmentAssetType::ATTACHMENT))
        {
            return array(
                self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT)
            );
        }

        return null;
    }

	/* (non-PHPdoc)
	 * @see IKalturaObjectLoader::loadObject()
	 */
	public static function loadObject($baseClass, $enumValue, array $constructorArgs = null)
	{
		if($baseClass == 'KalturaAsset' && $enumValue == self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT))
			return new KalturaTranscriptAsset();
	
		return null;
	}

	/* (non-PHPdoc)
	 * @see IKalturaObjectLoader::getObjectClass()
	 */
	public static function getObjectClass($baseClass, $enumValue)
	{
		if($baseClass == 'asset' && $enumValue == self::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT))
			return 'TranscriptAsset';
	
		return null;
	}

	/* (non-PHPdoc)
	 * @see IKalturaPending::dependsOn()
	 */
	public static function dependsOn()
	{
		$dependency = new KalturaDependency(AttachmentPlugin::getPluginName());
		return array($dependency);
	}

	/**
	 * @return int id of dynamic enum in the DB.
	 */
	public static function getAssetTypeCoreValue($valueName)
	{
		$value = self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName;
		return kPluginableEnumsManager::apiToCore('assetType', $value);
	}
	
	/**
	 * @return string external API value of dynamic enum.
	 */
	public static function getApiValue($valueName)
	{
		return self::getPluginName() . IKalturaEnumerator::PLUGIN_VALUE_DELIMITER . $valueName;
	}

	/* (non-PHPdoc)
 * @see IKalturaSearchDataContributor::getSearchData()
 */
	public static function getSearchData(BaseObject $object)
	{
		if($object instanceof entry)
			return self::getTranscriptSearchData($object);

		return null;
	}
	
	public static function
	{
		json_decode($strJson);
		return (json_last_error() === JSON_ERROR_NONE);
	}

	public static function getTranscriptSearchData(entry $entry)
	{
		$transcriptAssets = assetPeer::retrieveByEntryId($entry->getId(), array(TranscriptPlugin::getAssetTypeCoreValue(TranscriptAssetType::TRANSCRIPT)));
		if(!$transcriptAssets || !count($transcriptAssets))
			return null;

		$data = array();
		foreach($transcriptAssets as $transcriptAsset)
		{
			/* @var $transcriptAsset TranscriptAsset */

			if($transcriptAsset->getContainerFormat() != AttachmentType::TEXT)
				continue;

			$syncKey = $transcriptAsset->getSyncKey(asset::FILE_SYNC_FLAVOR_ASSET_SUB_TYPE_ASSET);
			$fileContent = kFileSyncUtils::file_get_contents($syncKey, true, false);
			
			//Get values from string file
			$matches = array();
			preg_match_all('/value": "(.*?)"/', $s, $matches, PREG_PATTERN_ORDER);
			
			$content = $fileContent;
			if(count($matches))
			{
				$matches = $matches[1];
				$content = implode(" ", $matches);
			}
			
			if(!$content)
				continue;

			$content = trim(preg_replace('/\s+/', ' ', $content));
			if(!$content)
				continue;

			$data[] = $transcriptAsset->getId() . ' ' . self::ENTRY_TRANSCRIPT_PREFIX  . ' ' . $content . ' ' . self::ENTRY_TRANSCRIPT_SUFFIX;
		}

		$searchValues = array(
			self::PLUGINS_DATA => self::PLUGIN_NAME . ' ' . implode(' ', $data) . ' ' . self::SEARCH_TEXT_SUFFIX
		);

		return $searchValues;
	}
	
}
{'fraction_non_alphanumeric': 0.13384512683578104, 'fraction_numerical': 0.0005562972852692479, 'mean_word_length': 4.44457904300424, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2882726', 'n_tokens_mistral': 3205, 'n_tokens_neox': 2889, 'n_words': 521}