<commit_before>/* ***** BEGIN LICENSE BLOCK *****
 *
 * This file is part of Weave.
 *
 * The Initial Developer of Weave is the Institute for Visualization
 * and Perception Research at the University of Massachusetts Lowell.
 * Portions created by the Initial Developer are Copyright (C) 2008-2015
 * the Initial Developer. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/.
 * 
 * ***** END LICENSE BLOCK ***** */

namespace weavejs.data.key
{
	import ILinkableObjectWithNewProperties = weavejs.api.core.ILinkableObjectWithNewProperties;
	import ColumnMetadata = weavejs.api.data.ColumnMetadata;
	import IKeyFilter = weavejs.api.data.IKeyFilter;
	import IQualifiedKey = weavejs.api.data.IQualifiedKey;
	import IObjectWithDescription = weavejs.api.ui.IObjectWithDescription;
	import LinkableBoolean = weavejs.core.LinkableBoolean;
	import LinkableVariable = weavejs.core.LinkableVariable;
	import ColumnUtils = weavejs.data.ColumnUtils;
	import DynamicColumn = weavejs.data.column.DynamicColumn;
	import JS = weavejs.util.JS;

	@Weave.classInfo({id: "weavejs.data.key.ColumnDataFilter", interfaces: [IKeyFilter, ILinkableObjectWithNewProperties, IObjectWithDescription]})
	export class ColumnDataFilter implements IKeyFilter, ILinkableObjectWithNewProperties, IObjectWithDescription
	{
		public getDescription():string
		{
			return Weave.lang("Filter for {0}", ColumnUtils.getTitle(this.column));
		}
		
		public static /* readonly */ REGEXP:string = 'regexp';
		private static /* readonly */ ALTERNATE_REGEX_PROPERTY:string = 'regex';
		
		public /* readonly */ enabled = Weave.linkableChild(this, new LinkableBoolean(true), this._cacheVars);
		public /* readonly */ includeMissingKeyTypes = Weave.linkableChild(this, new LinkableBoolean(true), this._cacheVars);
		public /* readonly */ column = Weave.linkableChild(this, DynamicColumn, this._resetKeyLookup);
		
		/**
		 * An Array of Numbers, Strings and/or Range objects specifying numeric ranges.
		 * A Range object contains two properties: "min" and "max".
		 * Alternatively, you can specify "minInclusive" or "minExclusive" in place of "min"
		 * and "minInclusive" or "maxExclusive" in place of "max".
		 */
		public /* readonly */ values = Weave.linkableChild(this, new LinkableVariable(Array), this._resetKeyLookup);
		
		private _enabled:boolean;
		private _includeMissingKeyTypes:boolean;
		private _stringLookup = new Map<string, boolean>();
		private _numberLookup = new Map<number, boolean>();
		private _ranges:ColumnDataFilterRange[];
		private _regexps:RegExp[];
		private _keyType:string;
		private map_key = new WeakMap<IQualifiedKey, boolean>();
		
		/* ??? */
		private _cacheVars():void
		{
			this._enabled = this.enabled.value;
			this._includeMissingKeyTypes = this.includeMissingKeyTypes.value;
		}
		private _resetKeyLookup():void
		{
			var state = this.values.getSessionState() as (string|number|IRegExpState|IRangeState)[] || [];
			var range:ColumnDataFilterRange;
			var regexp:RegExp;
			
			this._keyType = this.column.getMetadata(ColumnMetadata.KEY_TYPE);
			this.map_key = new WeakMap<IQualifiedKey, boolean>();
			this._numberLookup.clear();
			this._stringLookup.clear();
			this._ranges = null;
			this._regexps = null;
			
			for (var value of state)
			{
				if (typeof value === typeof 0)
				{
					this._numberLookup.set(value as number, true);
				}
				else if (typeof value === typeof "")
				{
					this._stringLookup.set(value as string, true);
				}
				else if (ColumnDataFilterRange.isRange(value))
				{
					try
					{
						range = new ColumnDataFilterRange(value as IRangeState);
						if (!this._ranges)
							this._ranges = [];
						this._ranges.push(range);
					}
					catch (e)
					{
						// ignore this value
					}
				}
				else if (ColumnDataFilter.isRegExp(value))
				{
					if (!this._regexps)
						this._regexps = [];
					regexp = ColumnDataFilter.toRegExp(value);
					this._regexps.push(regexp);
				}
			}
			
			// last step - canonicalize session states containing ranges
			if (this._ranges)
			{
				var newState:(string|number|IRegExpState|IRangeState)[] = [];
				for (value of state)
				{
					if (typeof value === typeof 0 || typeof value === typeof "")
						newState.push(value as number|string);
				}
				for (range of this._ranges)
				{
					newState.push(range.getState());
				}
				for (regexp of this._regexps || [])
				{
					value = {
						regexp: regexp.source
					};

					newState.push(value);
				}
				this.values.setSessionState(newState);
			}
		}
		
		public containsKey(key:IQualifiedKey):boolean<commit_msg>added missing check in if statement<commit_after>/* ***** BEGIN LICENSE BLOCK *****
 *
 * This file is part of Weave.
 *
 * The Initial Developer of Weave is the Institute for Visualization
 * and Perception Research at the University of Massachusetts Lowell.
 * Portions created by the Initial Developer are Copyright (C) 2008-2015
 * the Initial Developer. All Rights Reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/.
 * 
 * ***** END LICENSE BLOCK ***** */

namespace weavejs.data.key
{
	import ILinkableObjectWithNewProperties = weavejs.api.core.ILinkableObjectWithNewProperties;
	import ColumnMetadata = weavejs.api.data.ColumnMetadata;
	import IKeyFilter = weavejs.api.data.IKeyFilter;
	import IQualifiedKey = weavejs.api.data.IQualifiedKey;
	import IObjectWithDescription = weavejs.api.ui.IObjectWithDescription;
	import LinkableBoolean = weavejs.core.LinkableBoolean;
	import LinkableVariable = weavejs.core.LinkableVariable;
	import ColumnUtils = weavejs.data.ColumnUtils;
	import DynamicColumn = weavejs.data.column.DynamicColumn;
	import JS = weavejs.util.JS;

	@Weave.classInfo({id: "weavejs.data.key.ColumnDataFilter", interfaces: [IKeyFilter, ILinkableObjectWithNewProperties, IObjectWithDescription]})
	export class ColumnDataFilter implements IKeyFilter, ILinkableObjectWithNewProperties, IObjectWithDescription
	{
		public getDescription():string
		{
			return Weave.lang("Filter for {0}", ColumnUtils.getTitle(this.column));
		}
		
		public static /* readonly */ REGEXP:string = 'regexp';
		private static /* readonly */ ALTERNATE_REGEX_PROPERTY:string = 'regex';
		
		public /* readonly */ enabled = Weave.linkableChild(this, new LinkableBoolean(true), this._cacheVars);
		public /* readonly */ includeMissingKeyTypes = Weave.linkableChild(this, new LinkableBoolean(true), this._cacheVars);
		public /* readonly */ column = Weave.linkableChild(this, DynamicColumn, this._resetKeyLookup);
		
		/**
		 * An Array of Numbers, Strings and/or Range objects specifying numeric ranges.
		 * A Range object contains two properties: "min" and "max".
		 * Alternatively, you can specify "minInclusive" or "minExclusive" in place of "min"
		 * and "minInclusive" or "maxExclusive" in place of "max".
		 */
		public /* readonly */ values = Weave.linkableChild(this, new LinkableVariable(Array), this._resetKeyLookup);
		
		private _enabled:boolean;
		private _includeMissingKeyTypes:boolean;
		private _stringLookup = new Map<string, boolean>();
		private _numberLookup = new Map<number, boolean>();
		private _ranges:ColumnDataFilterRange[];
		private _regexps:RegExp[];
		private _keyType:string;
		private map_key = new WeakMap<IQualifiedKey, boolean>();
		
		/* ??? */
		private _cacheVars():void
		{
			this._enabled = this.enabled.value;
			this._includeMissingKeyTypes = this.includeMissingKeyTypes.value;
		}
		private _resetKeyLookup():void
		{
			var state = this.values.getSessionState() as (string|number|IRegExpState|IRangeState)[] || [];
			var range:ColumnDataFilterRange;
			var regexp:RegExp;
			
			this._keyType = this.column.getMetadata(ColumnMetadata.KEY_TYPE);
			this.map_key = new WeakMap<IQualifiedKey, boolean>();
			this._numberLookup.clear();
			this._stringLookup.clear();
			this._ranges = null;
			this._regexps = null;
			
			for (var value of state)
			{
				if (typeof value === typeof 0)
				{
					this._numberLookup.set(value as number, true);
				}
				else if (typeof value === typeof "")
				{
					this._stringLookup.set(value as string, true);
				}
				else if (ColumnDataFilterRange.isRange(value))
				{
					try
					{
						range = new ColumnDataFilterRange(value as IRangeState);
						if (!this._ranges)
							this._ranges = [];
						this._ranges.push(range);
					}
					catch (e)
					{
						// ignore this value
					}
				}
				else if (ColumnDataFilter.isRegExp(value))
				{
					if (!this._regexps)
						this._regexps = [];
					regexp = ColumnDataFilter.toRegExp(value);
					this._regexps.push(regexp);
				}
			}
			
			// last step - canonicalize session states containing ranges
			if (this._ranges || this._regexps)
			{
				var newState:(string|number|IRegExpState|IRangeState)[] = [];
				for (value of state)
				{
					if (typeof value === typeof 0 || typeof value === typeof "")
						newState.push(value as number|string);
				}
				for (range of this._ranges || [])
				{
					newState.push(range.getState());
				}
				for (regexp of this._regexps || [])
				{
					value = {
						regexp: regexp.source
					};

					newState.push(value);
				}
				this.values.setSessionState(newState);
			}
		}
		
		public containsKey(key:IQualifiedKey):boolean
{'fraction_non_alphanumeric': 0.11390548045687938, 'fraction_numerical': 0.0031436655139893115, 'mean_word_length': 3.6991629739044805, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6607669', 'n_tokens_mistral': 3465, 'n_tokens_neox': 2856, 'n_words': 717}