<commit_before>/* ***** BEGIN LICENSE BLOCK *****
 *
 * This file is part of Weave.
 *
 * The Initial Developer of Weave is the Institute for Visualization
 * and Perception Research at the University of Massachusetts Lowell.
 * Portions created by the Initial Developer are Copyright (C) 2008-2015
 * the Initial Developer. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/.
 * 
 * ***** END LICENSE BLOCK ***** */

package weave.data.DataSources
{
	import avmplus.getQualifiedClassName;
	
	import flash.utils.ByteArray;
	
	import flashx.textLayout.tlf_internal;
	
	import mx.rpc.events.FaultEvent;
	import mx.rpc.events.ResultEvent;
	
	import weave.api.core.ILinkableHashMap;
	import weave.api.core.ILinkableObject;
	import weave.api.data.ColumnMetadata;
	import weave.api.data.IAttributeColumn;
	import weave.api.data.IColumnReference;
	import weave.api.data.IDataSource;
	import weave.api.data.IDataSource_Service;
	import weave.api.data.IKeyFilter;
	import weave.api.data.IWeaveTreeNode;
	import weave.api.disposeObject;
	import weave.api.getCallbackCollection;
	import weave.api.getLinkableDescendants;
	import weave.api.getSessionState;
	import weave.api.linkableObjectIsBusy;
	import weave.api.newLinkableChild;
	import weave.api.registerLinkableChild;
	import weave.api.reportError;
	import weave.api.setSessionState;
	import weave.core.LinkableDynamicObject;
	import weave.core.LinkableFile;
	import weave.core.LinkableHashMap;
	import weave.core.LinkablePromise;
	import weave.core.LinkableString;
	import weave.data.AttributeColumns.ProxyColumn;
	import weave.data.AttributeColumns.ReferencedColumn;
	import weave.data.hierarchy.ColumnTreeNode;
	import weave.services.AMF3Servlet;
	import weave.services.URLRequestUtils;
	import weave.services.addAsyncResponder;
	import weave.utils.ColumnUtils;
	import weave.utils.VectorUtils;
	
	public class WeaveAnalystDataSource extends AbstractDataSource implements IDataSource_Service
	{
		WeaveAPI.ClassRegistry.registerImplementation(IDataSource, WeaveAnalystDataSource, "Weave Analyst");
		
		public static const SCRIPT_OUTPUT_NAME:String = 'scriptOutputName';
		private static const RESULT_DATA:String = "resultData";
		private static const COLUMN_NAMES:String = "columnNames";
		
		public function WeaveAnalystDataSource()
		{
			promise.depend(url, scriptName, inputs, hierarchyRefresh);
			if (!cacheFile)
			{
				// register static LinkableFile as a child of the globalHashMap so it will not be disposed when an instance of this data source is disposed.
				cacheFile = registerLinkableChild(WeaveAPI.globalHashMap, new LinkableFile(URLRequestUtils.LOCAL_FILE_URL_SCHEME + CACHE_FILE_NAME), parseCacheFile);
			}
			registerLinkableChild(this, cacheFile, runScript);
		}
		
		private static const CACHE_STATE:String = 'sessionState';
		private static const CACHE_OUTPUT_STATE:String = 'outputCSV';
		private static const CACHE_FILE_NAME:String = 'WeaveAnalystDataSource.cache';
		private static var cacheFile:LinkableFile = null;
		private static var cache:Object;
		private static function parseCacheFile():void
		{
			try
			{
				cache = cacheFile.result.readObject();
			}
			catch (e:Error)
			{
				// ignore error
			}
		}
		/**
		 * @return true if data was loaded from cache
		 */
		private static function loadCache(source:WeaveAnalystDataSource):Boolean
		{
			var currentState:Object = getSessionState(source);
			var sourceName:String = WeaveAPI.globalHashMap.getName(source);
			if (cache && cache[sourceName])
			{
				if (WeaveAPI.SessionManager.computeDiff(currentState, cache[sourceName][CACHE_STATE]) === undefined)
				{
					setSessionState(source.outputCSV, cache[sourceName][CACHE_OUTPUT_STATE]);
					return true;
				}
			}
			return false;
		}
		private static function saveCache(source:WeaveAnalystDataSource):void
		{
			var name:String = WeaveAPI.globalHashMap.getName(source);
			if (!cache)
				cache = {};
			cache[name] = {};
			cache[name][CACHE_STATE] = getSessionState(source);
			cache[name][CACHE_OUTPUT_STATE] = getSessionState(source.outputCSV);
			var bytes:ByteArray = new ByteArray();
			bytes.writeObject(cache);
			WeaveAPI.URLRequestUtils.saveLocalFile(CACHE_FILE_NAME, bytes);
		}
		
		public const url:LinkableString = registerLinkableChild(this, new LinkableString('/WeaveAnalystServices/ComputationalServlet'), handleURLChange);
		public const scriptName:LinkableString = newLinkableChild(this, LinkableString);
		public const inputs:LinkableHashMap = newLinkableChild(this, LinkableHashMap);
		public const inputKeyFilter:LinkableDynamicObject = registerLinkableChild(this, new LinkableDynamicObject(IKeyFilter));
		
		private const promise:LinkablePromise = registerLinkableChild(this, new LinkablePromise(runScript, describePromise));
		private function describePromise():String { return lang("Running script {0}", scriptName.value); }
		private const outputCSV:CSVDataSource = newLinkableChild(this, CSVDataSource);
		private var _service:AMF3Servlet;
		
		override protected function initialize():void
		{
			super.initialize();
			promise.validate();
		}
		
		private function handleURLChange():void
		{
			if (_service && _service.servletURL == url.value)
				return;
			disposeObject(_service);
			_service = registerLinkableChild(this, new AMF3Servlet(url.value));
			hierarchyRefresh.triggerCallbacks();
		}
		
		private function runScript():void
		{
			// if callbacks are delayed, we assume that hierarchyRefresh will be explicitly triggered later.
			if (getCallbackCollection(this).callbacksAreDelayed)
				return;

			if (linkableObjectIsBusy(cacheFile))
				return;
			
			// read from cache if present
			if (loadCache(this))
				return;
			
			// force retrieval of referenced columns
			for each (var refCol:ReferencedColumn in getLinkableDescendants(inputs, ReferencedColumn))
				refCol.getInternalColumn();
			if (linkableObjectIsBusy(inputs))
				return;<commit_msg>using LinkablePromise validateNow constructor option<commit_after>/* ***** BEGIN LICENSE BLOCK *****
 *
 * This file is part of Weave.
 *
 * The Initial Developer of Weave is the Institute for Visualization
 * and Perception Research at the University of Massachusetts Lowell.
 * Portions created by the Initial Developer are Copyright (C) 2008-2015
 * the Initial Developer. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/.
 * 
 * ***** END LICENSE BLOCK ***** */

package weave.data.DataSources
{
	import avmplus.getQualifiedClassName;
	
	import flash.utils.ByteArray;
	
	import flashx.textLayout.tlf_internal;
	
	import mx.rpc.events.FaultEvent;
	import mx.rpc.events.ResultEvent;
	
	import weave.api.core.ILinkableHashMap;
	import weave.api.core.ILinkableObject;
	import weave.api.data.ColumnMetadata;
	import weave.api.data.IAttributeColumn;
	import weave.api.data.IColumnReference;
	import weave.api.data.IDataSource;
	import weave.api.data.IDataSource_Service;
	import weave.api.data.IKeyFilter;
	import weave.api.data.IWeaveTreeNode;
	import weave.api.disposeObject;
	import weave.api.getCallbackCollection;
	import weave.api.getLinkableDescendants;
	import weave.api.getSessionState;
	import weave.api.linkableObjectIsBusy;
	import weave.api.newLinkableChild;
	import weave.api.registerLinkableChild;
	import weave.api.reportError;
	import weave.api.setSessionState;
	import weave.core.LinkableDynamicObject;
	import weave.core.LinkableFile;
	import weave.core.LinkableHashMap;
	import weave.core.LinkablePromise;
	import weave.core.LinkableString;
	import weave.data.AttributeColumns.ProxyColumn;
	import weave.data.AttributeColumns.ReferencedColumn;
	import weave.data.hierarchy.ColumnTreeNode;
	import weave.services.AMF3Servlet;
	import weave.services.URLRequestUtils;
	import weave.services.addAsyncResponder;
	import weave.utils.ColumnUtils;
	import weave.utils.VectorUtils;
	
	public class WeaveAnalystDataSource extends AbstractDataSource implements IDataSource_Service
	{
		WeaveAPI.ClassRegistry.registerImplementation(IDataSource, WeaveAnalystDataSource, "Weave Analyst");
		
		public static const SCRIPT_OUTPUT_NAME:String = 'scriptOutputName';
		private static const RESULT_DATA:String = "resultData";
		private static const COLUMN_NAMES:String = "columnNames";
		
		public function WeaveAnalystDataSource()
		{
			promise.depend(url, scriptName, inputs, hierarchyRefresh);
			if (!cacheFile)
			{
				// register static LinkableFile as a child of the globalHashMap so it will not be disposed when an instance of this data source is disposed.
				cacheFile = registerLinkableChild(WeaveAPI.globalHashMap, new LinkableFile(URLRequestUtils.LOCAL_FILE_URL_SCHEME + CACHE_FILE_NAME), parseCacheFile);
			}
			registerLinkableChild(this, cacheFile, runScript);
		}
		
		private static const CACHE_STATE:String = 'sessionState';
		private static const CACHE_OUTPUT_STATE:String = 'outputCSV';
		private static const CACHE_FILE_NAME:String = 'WeaveAnalystDataSource.cache';
		private static var cacheFile:LinkableFile = null;
		private static var cache:Object;
		private static function parseCacheFile():void
		{
			try
			{
				cache = cacheFile.result.readObject();
			}
			catch (e:Error)
			{
				// ignore error
			}
		}
		/**
		 * @return true if data was loaded from cache
		 */
		private static function loadCache(source:WeaveAnalystDataSource):Boolean
		{
			var currentState:Object = getSessionState(source);
			var sourceName:String = WeaveAPI.globalHashMap.getName(source);
			if (cache && cache[sourceName])
			{
				if (WeaveAPI.SessionManager.computeDiff(currentState, cache[sourceName][CACHE_STATE]) === undefined)
				{
					setSessionState(source.outputCSV, cache[sourceName][CACHE_OUTPUT_STATE]);
					return true;
				}
			}
			return false;
		}
		private static function saveCache(source:WeaveAnalystDataSource):void
		{
			var name:String = WeaveAPI.globalHashMap.getName(source);
			if (!cache)
				cache = {};
			cache[name] = {};
			cache[name][CACHE_STATE] = getSessionState(source);
			cache[name][CACHE_OUTPUT_STATE] = getSessionState(source.outputCSV);
			var bytes:ByteArray = new ByteArray();
			bytes.writeObject(cache);
			WeaveAPI.URLRequestUtils.saveLocalFile(CACHE_FILE_NAME, bytes);
		}
		
		public const url:LinkableString = registerLinkableChild(this, new LinkableString('/WeaveAnalystServices/ComputationalServlet'), handleURLChange);
		public const scriptName:LinkableString = newLinkableChild(this, LinkableString);
		public const inputs:LinkableHashMap = newLinkableChild(this, LinkableHashMap);
		public const inputKeyFilter:LinkableDynamicObject = registerLinkableChild(this, new LinkableDynamicObject(IKeyFilter));
		
		private const promise:LinkablePromise = registerLinkableChild(this, new LinkablePromise(runScript, describePromise, true));
		private function describePromise():String { return lang("Running script {0}", scriptName.value); }
		private const outputCSV:CSVDataSource = newLinkableChild(this, CSVDataSource);
		private var _service:AMF3Servlet;
		
		override protected function initialize():void
		{
			super.initialize();
		}
		
		private function handleURLChange():void
		{
			if (_service && _service.servletURL == url.value)
				return;
			disposeObject(_service);
			_service = registerLinkableChild(this, new AMF3Servlet(url.value));
			hierarchyRefresh.triggerCallbacks();
		}
		
		private function runScript():void
		{
			// if callbacks are delayed, we assume that hierarchyRefresh will be explicitly triggered later.
			if (getCallbackCollection(this).callbacksAreDelayed)
				return;

			if (linkableObjectIsBusy(cacheFile))
				return;
			
			// read from cache if present
			if (loadCache(this))
				return;
			
			// force retrieval of referenced columns
			for each (var refCol:ReferencedColumn in getLinkableDescendants(inputs, ReferencedColumn))
				refCol.getInternalColumn();
			if (linkableObjectIsBusy(inputs))
				return;
{'fraction_non_alphanumeric': 0.09060292426482668, 'fraction_numerical': 0.0026285526531953343, 'mean_word_length': 4.982800982800983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4739178', 'n_tokens_mistral': 4023, 'n_tokens_neox': 3561, 'n_words': 851}