<commit_before>/*
    Weave (Web-based Analysis and Visualization Environment)
    Copyright (C) 2008-2011 University of Massachusetts Lowell

    This file is a part of Weave.

    Weave is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License, Version 3,
    as published by the Free Software Foundation.

    Weave is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Weave.  If not, see <http://www.gnu.org/licenses/>.
*/

package weave.data.AttributeColumns
{
	import flash.utils.Dictionary;
	import flash.utils.getTimer;
	
	import weave.api.data.ColumnMetadata;
	import weave.api.data.IAttributeColumn;
	import weave.api.data.IBinClassifier;
	import weave.api.data.IPrimitiveColumn;
	import weave.api.data.IQualifiedKey;
	import weave.api.registerLinkableChild;
	import weave.compiler.StandardLib;
	import weave.data.BinClassifiers.NumberClassifier;
	import weave.data.BinningDefinitions.CategoryBinningDefinition;
	import weave.data.BinningDefinitions.DynamicBinningDefinition;
	import weave.data.BinningDefinitions.SimpleBinningDefinition;
	import weave.utils.ColumnUtils;
	
	/**
	 * A binned column maps a record key to a bin key.
	 * 
	 * @author adufilie
	 */
	public class BinnedColumn extends ExtendedDynamicColumn implements IPrimitiveColumn
	{
		public function BinnedColumn()
		{
			binningDefinition.requestLocalObject(SimpleBinningDefinition, false);
			binningDefinition.generateBinClassifiersForColumn(internalDynamicColumn);
			registerLinkableChild(this, binningDefinition.asyncResultCallbacks);
		}
		
		/**
		 * This number overrides the min,max metadata values.
		 * @param propertyName The name of a metadata property.
		 * @return The value of the specified metadata property.
		 */
		override public function getMetadata(propertyName:String):String
		{
			validateBins();
			if (_binClassifiers && _binClassifiers.length)
			{
				switch (propertyName)
				{
					case ColumnMetadata.MIN:
						return numberOfBins > 0 ? "0" : null;
					case ColumnMetadata.MAX:
						var binCount:int = numberOfBins;
						return binCount > 0 ? String(binCount - 1) : null;
				}
			}
			return super.getMetadata(propertyName);
		}
		
		/**
		 * This defines how to generate the bins for this BinnedColumn.
		 * This is used to generate the derivedBins.
		 */
		public const binningDefinition:DynamicBinningDefinition = registerLinkableChild(this, new DynamicBinningDefinition(true));
		
		private var _binNames:Array = []; // maps a bin index to a bin name
		private var _binClassifiers:Array = []; // maps a bin index to an IBinClassifier
		private var _keyToBinIndexMap:Dictionary = new Dictionary(); // maps a record key to a bin index
		private var _binnedKeysArray:Array = []; // maps a bin index to a list of keys in that bin
		private var _binnedKeysMap:Object = {}; // maps a bin name to a list of keys in that bin
		private var _largestBinSize:uint = 0;
		private var _resultTriggerCount:uint = 0;
		
		/**
		 * This function generates bins using the binning definition and the internal column,
		 * and also saves lookups for mapping between bins and keys.
		 */
		private function validateBins():void
		{
			if (WeaveAPI.SessionManager.linkableObjectIsBusy(this))
				return;
			
			if (_resultTriggerCount != binningDefinition.asyncResultCallbacks.triggerCounter)
			{
				_resultTriggerCount = binningDefinition.asyncResultCallbacks.triggerCounter;
				// reset cached values
				_column = internalDynamicColumn.getInternalColumn();
				_keyToBinIndexMap = new Dictionary();
				_binnedKeysArray = [];
				_binnedKeysMap = {};
				_largestBinSize = 0;
				// save bin names for faster lookup
				_binNames = binningDefinition.getBinNames();
				_binClassifiers = binningDefinition.getBinClassifiers();
				// create empty key arrays
				if (_binNames)
					for (var i:int = 0; i < _binNames.length; i++)
						_binnedKeysMap[_binNames[i]] = _binnedKeysArray[i] = []; // same Array pointer
				_keys = internalDynamicColumn.keys;
				_i = 0;
				if (StandardLib.getArrayType(_binClassifiers) == NumberClassifier)
					_dataType = Number;
				else
					_dataType = String;
				// fill all mappings
				if (_column && _binClassifiers)
				{
					// high priority because not much can be done without data
					WeaveAPI.StageUtils.startTask(this, _asyncIterate, WeaveAPI.TASK_PRIORITY_HIGH, triggerCallbacks);
				}
			}
		}
		
		private var _dataType:Class;
		private var _column:IAttributeColumn;
		private var _i:int;
		private var _keys:Array;<commit_msg>improved performance of BinnedColumn by avoiding call to linkableObjectIsBusy() unless necessary<commit_after>/*
    Weave (Web-based Analysis and Visualization Environment)
    Copyright (C) 2008-2011 University of Massachusetts Lowell

    This file is a part of Weave.

    Weave is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License, Version 3,
    as published by the Free Software Foundation.

    Weave is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with Weave.  If not, see <http://www.gnu.org/licenses/>.
*/

package weave.data.AttributeColumns
{
	import flash.utils.Dictionary;
	import flash.utils.getTimer;
	
	import weave.api.data.ColumnMetadata;
	import weave.api.data.IAttributeColumn;
	import weave.api.data.IBinClassifier;
	import weave.api.data.IPrimitiveColumn;
	import weave.api.data.IQualifiedKey;
	import weave.api.registerLinkableChild;
	import weave.compiler.StandardLib;
	import weave.data.BinClassifiers.NumberClassifier;
	import weave.data.BinningDefinitions.DynamicBinningDefinition;
	import weave.data.BinningDefinitions.SimpleBinningDefinition;
	import weave.utils.ColumnUtils;
	
	/**
	 * A binned column maps a record key to a bin key.
	 * 
	 * @author adufilie
	 */
	public class BinnedColumn extends ExtendedDynamicColumn implements IPrimitiveColumn
	{
		public function BinnedColumn()
		{
			binningDefinition.requestLocalObject(SimpleBinningDefinition, false);
			binningDefinition.generateBinClassifiersForColumn(internalDynamicColumn);
			registerLinkableChild(this, binningDefinition.asyncResultCallbacks);
		}
		
		/**
		 * This number overrides the min,max metadata values.
		 * @param propertyName The name of a metadata property.
		 * @return The value of the specified metadata property.
		 */
		override public function getMetadata(propertyName:String):String
		{
			validateBins();
			if (_binClassifiers && _binClassifiers.length)
			{
				switch (propertyName)
				{
					case ColumnMetadata.MIN:
						return numberOfBins > 0 ? "0" : null;
					case ColumnMetadata.MAX:
						var binCount:int = numberOfBins;
						return binCount > 0 ? String(binCount - 1) : null;
				}
			}
			return super.getMetadata(propertyName);
		}
		
		/**
		 * This defines how to generate the bins for this BinnedColumn.
		 * This is used to generate the derivedBins.
		 */
		public const binningDefinition:DynamicBinningDefinition = registerLinkableChild(this, new DynamicBinningDefinition(true));
		
		private var _binNames:Array = []; // maps a bin index to a bin name
		private var _binClassifiers:Array = []; // maps a bin index to an IBinClassifier
		private var _keyToBinIndexMap:Dictionary = new Dictionary(); // maps a record key to a bin index
		private var _binnedKeysArray:Array = []; // maps a bin index to a list of keys in that bin
		private var _binnedKeysMap:Object = {}; // maps a bin name to a list of keys in that bin
		private var _largestBinSize:uint = 0;
		private var _resultTriggerCount:uint = 0;
		
		/**
		 * This function generates bins using the binning definition and the internal column,
		 * and also saves lookups for mapping between bins and keys.
		 */
		private function validateBins():void
		{
			if (_resultTriggerCount != binningDefinition.asyncResultCallbacks.triggerCounter)
			{
				if (WeaveAPI.SessionManager.linkableObjectIsBusy(this))
					return;
				
				_resultTriggerCount = binningDefinition.asyncResultCallbacks.triggerCounter;
				// reset cached values
				_column = internalDynamicColumn.getInternalColumn();
				_keyToBinIndexMap = new Dictionary();
				_binnedKeysArray = [];
				_binnedKeysMap = {};
				_largestBinSize = 0;
				// save bin names for faster lookup
				_binNames = binningDefinition.getBinNames();
				_binClassifiers = binningDefinition.getBinClassifiers();
				// create empty key arrays
				if (_binNames)
					for (var i:int = 0; i < _binNames.length; i++)
						_binnedKeysMap[_binNames[i]] = _binnedKeysArray[i] = []; // same Array pointer
				_keys = internalDynamicColumn.keys;
				_i = 0;
				if (StandardLib.getArrayType(_binClassifiers) == NumberClassifier)
					_dataType = Number;
				else
					_dataType = String;
				// fill all mappings
				if (_column && _binClassifiers)
				{
					// high priority because not much can be done without data
					WeaveAPI.StageUtils.startTask(this, _asyncIterate, WeaveAPI.TASK_PRIORITY_HIGH, triggerCallbacks, lang("Binning {0} records in {1}", _keys.length, debugId(this)));
				}
			}
		}
		
		private var _dataType:Class;
		private var _column:IAttributeColumn;
		private var _i:int;
		private var _keys:Array;
{'fraction_non_alphanumeric': 0.07757375696306994, 'fraction_numerical': 0.003919950484835981, 'mean_word_length': 3.7805719921104535, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2400523', 'n_tokens_mistral': 3182, 'n_tokens_neox': 2693, 'n_words': 872}