<commit_before>/* jshint esversion: 6 */

const http = require('http');
const async = require('async');
const cron = require('node-cron');
var MongoClient = require('mongodb').MongoClient;

const url_db = 'mongodb://localhost:27017/folding';
const url_user = 'http://fah-web.stanford.edu/daily_user_summary.txt';

var lastDailyUpdate = new Date();

var months = [
	'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
	'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
];

MongoClient.connect(url_db, function(err, db) {
	if (err) return console.log(err.message);
		
	db.collection('users').ensureIndex({ score : -1 }, function(err, result) {
		if (err) console.log(err.message);
		
		db.collection('users').find().sort({ score : -1 }).toArray(function(err, users) {
			if (err) return console.log(err.message);
			
			var map = new Map();
			var refs = [];
			for (var i = 0; i < users.length; i++) {
				var key = users[i]._id.name + users[i]._id.teamID;
				userMap.set(key, users[i]);
				refs.push(users[i]);
			}
			
			http.get(url_user, function(res) {
				if (res.statusCode !== 200) {
					console.log('HTTP request for user data failed');
					return;
				}
				
				var newData = '';
				console.log('Downloading new user data');
				res.setEncoding('utf8');
				res.on('data', function(chunk) {
					newData += chunk;
				});
				res.on('end', function() {
					console.log('New user data downloaded successfully');
					
					var timeStamp = new Date(
						parseInt(newData.slice(24, 28)),
						months.indexOf(newData.slice(4, 7)),
						parseInt(newData.slice(8, 10)),
						parseInt(newData.slice(11, 13)),
						parseInt(newData.slice(14, 16)),
						parseInt(newData.slice(17, 19))
					);
					var daily = timeStamp - lastDailyUpdate > 86400000;<commit_msg>fixed an error<commit_after>/* jshint esversion: 6 */

const http = require('http');
const async = require('async');
const cron = require('node-cron');
var MongoClient = require('mongodb').MongoClient;

const url_db = 'mongodb://localhost:27017/folding';
const url_user = 'http://fah-web.stanford.edu/daily_user_summary.txt';

var lastDailyUpdate = new Date();

var months = [
	'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
	'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
];

MongoClient.connect(url_db, function(err, db) {
	if (err) return console.log(err.message);
		
	db.collection('users').ensureIndex({ score : -1 }, function(err, result) {
		if (err) console.log(err.message);
		
		db.collection('users').find().sort({ score : -1 }).toArray(function(err, users) {
			if (err) return console.log(err.message);
			
			var map = new Map();
			var refs = [];
			for (var i = 0; i < users.length; i++) {
				var key = users[i]._id.name + users[i]._id.teamID;
				map.set(key, users[i]);
				refs.push(users[i]);
			}
			
			http.get(url_user, function(res) {
				if (res.statusCode !== 200) {
					console.log('HTTP request for user data failed');
					return;
				}
				
				var newData = '';
				console.log('Downloading new user data');
				res.setEncoding('utf8');
				res.on('data', function(chunk) {
					newData += chunk;
				});
				res.on('end', function() {
					console.log('New user data downloaded successfully');
					
					var timeStamp = new Date(
						parseInt(newData.slice(24, 28)),
						months.indexOf(newData.slice(4, 7)),
						parseInt(newData.slice(8, 10)),
						parseInt(newData.slice(11, 13)),
						parseInt(newData.slice(14, 16)),
						parseInt(newData.slice(17, 19))
					);
					var daily = timeStamp - lastDailyUpdate > 86400000;
{'fraction_non_alphanumeric': 0.18274978650725876, 'fraction_numerical': 0.02391118701964133, 'mean_word_length': 3.397997496871089, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1728583', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1202, 'n_words': 271}