<commit_before>"use strict";

(function(ADL){
	
	var XAPIDashboard = function(container){
		this.container = container;
		try {
			this.data = new ADL.Collection();
		}
		catch(e){
			this.data = new ADL.CollectionSync();
		}
	}

	XAPIDashboard.prototype.fetchAllStatements = function(query, wrapper, cb){
		var self = this, statementsArr = [];
		if( !wrapper || typeof(wrapper) === 'function' ){
			if(typeof(wrapper) === 'function' && !cb)
				cb = wrapper;
			wrapper = ADL.XAPIWrapper;
		}

		wrapper.getStatements(query, null, function getMore(r){
			var response = JSON.parse(r.response);
			
			self.data.append(response.statements);

			if(response.more){
				wrapper.getStatements(null, response.more, getMore);
			}
			else {
				cb(self.data);
			}
		});
	};
	
	XAPIDashboard.prototype.clearSavedStatements = function(){
		
		//Update to support async collections!
		this.statements = new ADL.Collection();
	};
	
	XAPIDashboard.prototype.addStatements = function(statementsArr){
	
		if(statementsArr.response){
			try{
				statementsArr = JSON.parse(statementsArr.response).statements;
			}
			catch(e){
				console.error("Error parsing JSON data", statementsArr.response);
				return;
			}
		}
		this.data.append(statementsArr);
	};
	
	// default aggregator requires opts.xField
	// opts.aggregate, opts.pre, opts.post, opts.customize are optional functions
	XAPIDashboard.prototype.createChart = function(type, opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		switch(type){
			case "barChart": opts.chart = new ADL.BarChart(opts); break;
			case "lineChart": opts.chart = new ADL.LineChart(opts); break;
			case "pieChart": opts.chart = new ADL.PieChart(opts); break;
			case "multiBarChart": opts.chart = new ADL.MultiBarChart(opts); break;
			case "linePlusBarChart": opts.chart = new ADL.LinePlusBarChart(opts); break;
			case "table": opts.chart = new ADL.Table(opts); break;
			default: opts.chart = new ADL.Chart(opts);
		}
		
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.BarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createLinePlusBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.LinePlusBarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createMultiBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.MultiBarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createLineChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.LineChart(opts);
		return opts.chart;
	};  
	XAPIDashboard.prototype.createPieChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.PieChart(opts);
		return opts.chart;
	};   
	XAPIDashboard.prototype.createTable = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.Table(opts);
		return opts.chart;
	}; 
	
	/*
	 * Class methods to perform graph "formatting" operations
	 */	 
	ADL.select = function(xpath){		
		
		var innerFn = function(opts, join){
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
			
			opts.xpath = xpath;
			if(join !== true){
				if(opts.range){
					return opts.data = opts.data.groupBy(opts.groupBy, [opts.range.start, opts.range.end, opts.range.increment]).exec(formatData);
				}
				else {
					return opts.data = opts.data.groupBy(opts.groupBy).exec(formatData);
				}
			}
			
			else{
				return opts.data.contents.map(function(element){
					var retrieve = element.data[0];
					element[xpath] = xpathfn(xpath, retrieve);
				});
			}
			
			//Used as an intermediate callback for exec
			function formatData(data){
				for(var i = 0; i < data.length; i++){
					
					data[i].in = data[i].group;
					data[i].out = xpathfn(xpath, data[i].data[0]);
					delete data[i].group;
					delete data[i].data;
				}
				
				opts.cb(data);
			}
		};
		
		innerFn._inner = true;
		return innerFn;
	};

	ADL.count = function(ignoreXpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
				
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).count().select(rangeLabel+' as in, count as out').exec(opts.cb);
			}
			else {
				return ret.count(opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.sum = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).sum(xpath).select(rangeLabel+' as in, sum as out').exec(opts.cb);
			}
			else {
				return ret.sum(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.min = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).min(xpath).select(rangeLabel+' as in, min as out').exec(opts.cb);
			}
			else {
				return ret.min(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.max = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).max(xpath).select(rangeLabel+' as in, max as out').exec(opts.cb);
			}
			else {
				return ret.max(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.average = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).average(opts.xpath).select(rangeLabel+' as in, average as out').exec(opts.cb);
			}
			else {
				
				return ret.average(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.multiAggregate = function(xpath){
		
		var multi;
		if(typeof xpath === "string"){
			multi = Array.prototype.slice.call(arguments, 1);
		}
		else{
			multi = Array.prototype.slice.call(arguments, 0);
			xpath = null;
		}

		return function(opts)
		{
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
			
			opts.groupByLevel = opts.innerGroupBy ? 1 : 0;
			var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
			opts.data = opts.groupByLevel > 0 ? opts.data.groupBy(opts.groupBy).groupBy(opts.innerGroupBy) : opts.data.groupBy(opts.groupBy);
			
			for( var i=0; i<multi.length; i++ ){
				
				//This is a reference directly to the inner function
				if(multi[i]._inner){
					multi[i](opts, true);
				}
				else if(xpath != null){
					multi[i](xpath)(opts, true);
				}
				else{
					console.error("If an xpath is not provided to multiAggregate, then it must be provided to each aggregation function");
				}
			}
			return opts.data = opts.data.exec(tempCb);
			
			function tempCb(data)
			{
				
				var colorRange = d3.scale.category20().range(),
					aggArr = [],
					ignoreKeys = ['data', 'group', 'sample'],
					g = 1,
					tempData = opts.groupByLevel > 0 ? data[0].data[0] : data[0];
				
				
				
				// create series from aggregate fields of data
				if(opts.groupByLevel == 0){
					for(var i in data[0]){
						if( ignoreKeys.indexOf(i) < 0 ){
							aggArr.push({key: i, values: [], color: colorRange[g]});
							g += 2;
						}
					}
				}
				else{
					var firstElem = data[0].data;
					for(var j = 0; j < firstElem.length; j++){
						for(var i in firstElem[j]){
							if( ignoreKeys.indexOf(i) < 0 ){
								var keyName =  firstElem[j].group + " " + i;
								
								//
								for(var z = 0; z < data.length; z++){
									if(data[z].data[j]){
										data[z].data[j][keyName] = data[z].data[j][i] == undefined ? 0 : data[z].data[j][i];
									}
									else{
										//Should anything be done when data is missing? Ex: group by test, second group by verb (pass/fail). Well,
										//it's possible for all students to pass a test, leaving that test without fail statements.
									}
								}
								
								aggArr.push({key: keyName, values: [], color: colorRange[g]});
								g += 2;
							}
						}
					}
				}
				
				// add data to series
				for(var i = 0; i < data.length; i++){
					
					var tempData = data[i];
					var j = 0;
					
					do{
						if(opts.groupByLevel > 0) tempData = data[i].data[j];
						for(g = 0; g < aggArr.length; g++){
							if(tempData[aggArr[g].key] != undefined){
								aggArr[g].values.push({in: data[i].group, out: tempData[aggArr[g].key], series: g});
							}
						}
						
						j++;
					}
					while(opts.groupByLevel > 0 && data[i].data.length > j);
				}
				
				
				opts.cb(aggArr);
			}
		};<commit_msg>Reset multiBar colors<commit_after>"use strict";

(function(ADL){
	
	var XAPIDashboard = function(container){
		this.container = container;
		try {
			this.data = new ADL.Collection();
		}
		catch(e){
			this.data = new ADL.CollectionSync();
		}
	}

	XAPIDashboard.prototype.fetchAllStatements = function(query, wrapper, cb){
		var self = this, statementsArr = [];
		if( !wrapper || typeof(wrapper) === 'function' ){
			if(typeof(wrapper) === 'function' && !cb)
				cb = wrapper;
			wrapper = ADL.XAPIWrapper;
		}

		wrapper.getStatements(query, null, function getMore(r){
			var response = JSON.parse(r.response);
			
			self.data.append(response.statements);

			if(response.more){
				wrapper.getStatements(null, response.more, getMore);
			}
			else {
				cb(self.data);
			}
		});
	};
	
	XAPIDashboard.prototype.clearSavedStatements = function(){
		
		//Update to support async collections!
		this.statements = new ADL.Collection();
	};
	
	XAPIDashboard.prototype.addStatements = function(statementsArr){
	
		if(statementsArr.response){
			try{
				statementsArr = JSON.parse(statementsArr.response).statements;
			}
			catch(e){
				console.error("Error parsing JSON data", statementsArr.response);
				return;
			}
		}
		this.data.append(statementsArr);
	};
	
	// default aggregator requires opts.xField
	// opts.aggregate, opts.pre, opts.post, opts.customize are optional functions
	XAPIDashboard.prototype.createChart = function(type, opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		switch(type){
			case "barChart": opts.chart = new ADL.BarChart(opts); break;
			case "lineChart": opts.chart = new ADL.LineChart(opts); break;
			case "pieChart": opts.chart = new ADL.PieChart(opts); break;
			case "multiBarChart": opts.chart = new ADL.MultiBarChart(opts); break;
			case "linePlusBarChart": opts.chart = new ADL.LinePlusBarChart(opts); break;
			case "table": opts.chart = new ADL.Table(opts); break;
			default: opts.chart = new ADL.Chart(opts);
		}
		
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.BarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createLinePlusBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.LinePlusBarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createMultiBarChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.MultiBarChart(opts);
		return opts.chart;
	}; 
	XAPIDashboard.prototype.createLineChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.LineChart(opts);
		return opts.chart;
	};  
	XAPIDashboard.prototype.createPieChart = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.PieChart(opts);
		return opts.chart;
	};   
	XAPIDashboard.prototype.createTable = function(opts){
		opts.data = this.data;
		opts.container = opts.container ? opts.container : this.container;
		
		opts.chart = new ADL.Table(opts);
		return opts.chart;
	}; 
	
	/*
	 * Class methods to perform graph "formatting" operations
	 */	 
	ADL.select = function(xpath){		
		
		var innerFn = function(opts, join){
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
			
			opts.xpath = xpath;
			if(join !== true){
				if(opts.range){
					return opts.data = opts.data.groupBy(opts.groupBy, [opts.range.start, opts.range.end, opts.range.increment]).exec(formatData);
				}
				else {
					return opts.data = opts.data.groupBy(opts.groupBy).exec(formatData);
				}
			}
			
			else{
				return opts.data.contents.map(function(element){
					var retrieve = element.data[0];
					element[xpath] = xpathfn(xpath, retrieve);
				});
			}
			
			//Used as an intermediate callback for exec
			function formatData(data){
				for(var i = 0; i < data.length; i++){
					
					data[i].in = data[i].group;
					data[i].out = xpathfn(xpath, data[i].data[0]);
					delete data[i].group;
					delete data[i].data;
				}
				
				opts.cb(data);
			}
		};
		
		innerFn._inner = true;
		return innerFn;
	};

	ADL.count = function(ignoreXpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
				
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).count().select(rangeLabel+' as in, count as out').exec(opts.cb);
			}
			else {
				return ret.count(opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.sum = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).sum(xpath).select(rangeLabel+' as in, sum as out').exec(opts.cb);
			}
			else {
				return ret.sum(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.min = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).min(xpath).select(rangeLabel+' as in, min as out').exec(opts.cb);
			}
			else {
				return ret.min(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.max = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).max(xpath).select(rangeLabel+' as in, max as out').exec(opts.cb);
			}
			else {
				return ret.max(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.average = function(xpath){
		var innerFn = function(opts, join){
			if(!opts.groupBy || !xpath){
				console.error("group or xpath has not been specified, aborting aggregation", opts);
				return;
			}
				
			opts.xpath = xpath;
			var ret = opts.data;
			if(join !== true){
				var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
				var rangeLabel;
				if( opts.rangeLabel === 'start' )
					rangeLabel = 'groupStart';
				else if( opts.rangeLabel === 'end' )
					rangeLabel = 'groupEnd';
				else
					rangeLabel = opts.rangeLabel || 'group';
					
				return opts.data = ret.groupBy(opts.groupBy, range).average(opts.xpath).select(rangeLabel+' as in, average as out').exec(opts.cb);
			}
			else {
				
				return ret.average(xpath, opts.groupByLevel);
			}

		};
		
		innerFn._inner = true;
		return innerFn;
	};	
	
	ADL.multiAggregate = function(xpath){
		
		var multi;
		if(typeof xpath === "string"){
			multi = Array.prototype.slice.call(arguments, 1);
		}
		else{
			multi = Array.prototype.slice.call(arguments, 0);
			xpath = null;
		}

		return function(opts)
		{
			if(!opts.groupBy){
				console.error("group has not been specified, aborting aggregation", opts);
				return;
			}
			
			opts.groupByLevel = opts.innerGroupBy ? 1 : 0;
			var range = opts.range ? [opts.range.start, opts.range.end, opts.range.increment] : null;
			opts.data = opts.groupByLevel > 0 ? opts.data.groupBy(opts.groupBy).groupBy(opts.innerGroupBy) : opts.data.groupBy(opts.groupBy);
			
			for( var i=0; i<multi.length; i++ ){
				
				//This is a reference directly to the inner function
				if(multi[i]._inner){
					multi[i](opts, true);
				}
				else if(xpath != null){
					multi[i](xpath)(opts, true);
				}
				else{
					console.error("If an xpath is not provided to multiAggregate, then it must be provided to each aggregation function");
				}
			}
			return opts.data = opts.data.exec(tempCb);
			
			function tempCb(data)
			{
				var colorRange = d3.scale.category10().range(),
					aggArr = [],
					ignoreKeys = ['data', 'group', 'sample'],
					g = 0,
					tempData = opts.groupByLevel > 0 ? data[0].data[0] : data[0];

				// create series from aggregate fields of data
				if(opts.groupByLevel == 0){
					for(var i in data[0]){
						if( ignoreKeys.indexOf(i) < 0 ){
							aggArr.push({key: i, values: [], color: colorRange[g]});
							g++;
						}
					}
				}
				else{
					var firstElem = data[0].data;
					for(var j = 0; j < firstElem.length; j++){
						for(var i in firstElem[j]){
							if( ignoreKeys.indexOf(i) < 0 ){
								var keyName =  firstElem[j].group + " " + i;
								
								//
								for(var z = 0; z < data.length; z++){
									if(data[z].data[j]){
										data[z].data[j][keyName] = data[z].data[j][i] == undefined ? 0 : data[z].data[j][i];
									}
									else{
										//Should anything be done when data is missing? Ex: group by test, second group by verb (pass/fail). Well,
										//it's possible for all students to pass a test, leaving that test without fail statements.
									}
								}
								
								aggArr.push({key: keyName, values: [], color: colorRange[g]});
								g++;
							}
						}
					}
				}
				
				// add data to series
				for(var i = 0; i < data.length; i++){
					var tempData = data[i], j = 0;
					
					do{
						if(opts.groupByLevel > 0) tempData = data[i].data[j];
						for(g = 0; g < aggArr.length; g++){
							if(tempData[aggArr[g].key] != undefined){
								aggArr[g].values.push({in: data[i].group, out: tempData[aggArr[g].key], series: g});
							}
						}
						
						j++;
					}
					while(opts.groupByLevel > 0 && data[i].data.length > j);
				}
				
				opts.cb(aggArr);
			}
		};
{'fraction_non_alphanumeric': 0.147116091904288, 'fraction_numerical': 0.002682705205313487, 'mean_word_length': 2.8385650224215246, 'pattern_counts': {'":': 12, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4845827', 'n_tokens_mistral': 9833, 'n_tokens_neox': 7678, 'n_words': 1719}