<commit_before>
var http = require("http");
var request = require("request");
var db = require('./config.js');
var FetchedRepo = require('./fetchedRepos.js').FetchedRepo;
var Results = require('./result.js').Results;


String.prototype.contains = function(str, ignoreCase) {
	return (ignoreCase ? this.toUpperCase() : this)
	.indexOf(ignoreCase ? str.toUpperCase() : str) >= 0;
};
// sample url just for testing
// REGEX /\S*.js\w*/gi 
var repoObjs;

var urlRetrieve = FetchedRepo.find(function(err, data){
	if(err){
		throw err;
	} else {
		repoObjs = data;
		repoObjs.forEach(function(item){
			var repo = item;
			parseForJS(repo);
		});
	}
}
);

var parseForJS = function(obj){
	var result;
	var repoData = {
		repoLink : obj.file_url,
		libraryCollection: {
			react : false,
			angular: false,
			ember: false,
			backbone: false,
			mithril: false,
			polymer: false,
			flight: false,
			capuccino: false,
			spine: false,
			aurelia: false
		}
	};
	request(obj.file_url, function (error, response, body) {
		// create an object to track framework occurences
		console.log('URL : ', obj.file_url);
		console.log('Error : ', error);
		if (!error && response.statusCode == 200) {
				// parse raw html for all strings ending in js 
				var test = body.match(/\S*.js\w*/gi);
			// console.log(test);
			if(test !== null){
				for(var i = 0; i < test.length; i++){
				// loop through the array of matches
				test[i] = test[i].match(/[^/]*$/gi);
				var foundlib = test[i][0];

				for(var key in repoData.libraryCollection){
					// compare each framework in our collection 
					// to see if that string is contained in our js strings
					if(foundlib.contains(key, true)){
						repoData.libraryCollection[key] = true;
						// set that framework to true indicating use
					}
				}
			}
		}


		var repoStats = new Results({
			repo_name: obj.repo_name,
			repo_url: obj.repo_url,
			file_url: obj.file_url,
			repo_data: JSON.stringify(repoData)
		});
		repoStats.save(function(err){
			if(err){
				console.log('Error : ', err);
				throw err;
			}
		});
	}
});
};
<commit_msg>added repo draining<commit_after>
var http = require("http");
var request = require("request");
var db = require('./config.js');
var FetchedRepo = require('./fetchedRepos.js').FetchedRepo;
var Results = require('./result.js').Results;


String.prototype.contains = function(str, ignoreCase) {
	return (ignoreCase ? this.toUpperCase() : this)
	.indexOf(ignoreCase ? str.toUpperCase() : str) >= 0;
};
// sample url just for testing
// REGEX /\S*.js\w*/gi 
var repoObjs;

var urlRetrieve = FetchedRepo.find(function(err, data){
	if(err){
		throw err;
	} else {
		repoObjs = data;
		repoObjs.forEach(function(item){
			var repo = item;
			parseForJS(repo);
		});
	}
}
);

var parseForJS = function(obj){
	var result;
	var repoData = {
		repoLink : obj.file_url,
		libraryCollection: {
			react : false,
			angular: false,
			ember: false,
			backbone: false,
			mithril: false,
			polymer: false,
			flight: false,
			capuccino: false,
			spine: false,
			aurelia: false
		}
	};
	request(obj.file_url, function (error, response, body) {
		// create an object to track framework occurences
		console.log('URL : ', obj.file_url);
		console.log('Error : ', error);
		if (!error && response.statusCode == 200) {
				// parse raw html for all strings ending in js 
				var test = body.match(/\S*.js\w*/gi);
			// console.log(test);
			if(test !== null){
				for(var i = 0; i < test.length; i++){
				// loop through the array of matches
				test[i] = test[i].match(/[^/]*$/gi);
				var foundlib = test[i][0];

				for(var key in repoData.libraryCollection){
					// compare each framework in our collection 
					// to see if that string is contained in our js strings
					if(foundlib.contains(key, true)){
						repoData.libraryCollection[key] = true;
						// set that framework to true indicating use
					}
				}
			}
		}


		var repoStats = new Results({
			repo_name: obj.repo_name,
			repo_url: obj.repo_url,
			file_url: obj.file_url,
			repo_data: JSON.stringify(repoData)
		});
		repoStats.save(function(err){
			if(err){
				console.log('Error : ', err);
				throw err;
			} else {
        FetchedRepo.find({repo_name: obj.repo_name}).remove().exec();
      }
		});
	}
});
};

{'fraction_non_alphanumeric': 0.1528005624560581, 'fraction_numerical': 0.002812280290602297, 'mean_word_length': 3.1196911196911197, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7277856', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1457, 'n_words': 371}