<commit_before>var clarifai = new Clarifai({id: '<KEY>', secret: '<KEY>'});

//Clarifai.initAPI(process.env.CLARIFAI_CLIENT_ID, process.env.CLARIFAI_CLIENT_SECRET);

// Setting a throttle handler lets you know when the service is unavailable because of throttling. It will let
// you know when the service is available again. Note that setting the throttle handler causes a timeout handler to
// be set that will prevent your process from existing normally until the timeout expires. If you want to exit fast
// on being throttled, don't set a handler and look for error results instead.

//Clarifai.setThrottleHandler( function( bThrottled, waitSeconds ) { 
//	console.log( bThrottled ? ["throttled. service available again in",waitSeconds,"seconds"].join(' ') : "not throttled");
//});

function filterTags(URL, tags, resultsCallback) {
	// Get a database reference to our posts
	var firebase = new Firebase("https://boiling-inferno-5486.firebaseio.com/foodwords/-K0vUW8ifj42bB3hUETz");

	tags.sort();

	// Attach an asynchronous callback to read the data at our posts reference
	firebase.once("value",
	function (foodWordsWhitelist) {
		var whitelist = foodWordsWhitelist.val();
		var filteredTags = new Array();
		var badTags = new Array();

		while( tags.length > 0 && whitelist.length > 0 )
		{   
		    if (tags[0] < whitelist[0]) {
		    	badTags.add(tags[0]);
		    	tags.shift();
		    }
		    else if (tags[0] > whitelist[0])
		    	whitelist.shift();
		    else {
			    filteredTags.push(tags.shift());<commit_msg>i just aweanna s<commit_after>var clarifai = new Clarifai({id: '34EZ1WNwGt7dvL08d-k2BNfutb-ZqqOh8mmdQXNP', secret: '<KEY>'});

//Clarifai.initAPI(process.env.CLARIFAI_CLIENT_ID, process.env.CLARIFAI_CLIENT_SECRET);

// Setting a throttle handler lets you know when the service is unavailable because of throttling. It will let
// you know when the service is available again. Note that setting the throttle handler causes a timeout handler to
// be set that will prevent your process from existing normally until the timeout expires. If you want to exit fast
// on being throttled, don't set a handler and look for error results instead.

//Clarifai.setThrottleHandler( function( bThrottled, waitSeconds ) { 
//	console.log( bThrottled ? ["throttled. service available again in",waitSeconds,"seconds"].join(' ') : "not throttled");
//});

function filterTags(URL, tags, resultsCallback) {
	// Get a database reference to our posts
	var firebase = new Firebase("https://boiling-inferno-5486.firebaseio.com/foodwords/-K0vUW8ifj42bB3hUETz");

	tags.sort();

	// Attach an asynchronous callback to read the data at our posts reference
	firebase.once("value",
	function (foodWordsWhitelist) {
		var whitelist = foodWordsWhitelist.val();
		var filteredTags = new Array();
		var badTags = new Array();

		while( tags.length > 0 && whitelist.length > 0 )
		{   
		    if (tags[0] < whitelist[0]) {
		    	badTags.push(tags[0]);
		    	tags.shift();
		    }
		    else if (tags[0] > whitelist[0])
		    	whitelist.shift();
		    else {
			    filteredTags.push(tags.shift());
{'fraction_non_alphanumeric': 0.11574523116715163, 'fraction_numerical': 0.012932428063368898, 'mean_word_length': 3.926751592356688, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2643918', 'n_tokens_mistral': 1065, 'n_tokens_neox': 962, 'n_words': 304}