<commit_before>var Datastore = require('nedb')
	, request = require('request')
	, helpers = require('../modules/helpers.js')
	, scrape = require('../modules/scrape.js').scrape;

var db = new Datastore({ filename: './datastore/data.db', autoload: true });

function pad(num) {
	var s = num + ''
		, origLength = s.length;
    while (s.length < ((4 + origLength) - origLength)) s = '0' + s;
    return s;
}

function setupDb(total) {

	var solSlides = []
		, weatherData, sol;

	function parseWeather(callback) {
		helpers.output('Requesting weather data ...');
		request('http://marsweather.ingenology.com/v1/archive/?sol=' + parseInt(sol, 10), callback);
	}

	function parseImageData(images) {
		if (images.length <= 0) {
			parseSlides();
			return;
		}

		var imgData = images.shift();		

		if (weatherData) imgData.weather = weatherData;

		scrape(imgData.url, function($) {
			imgData.description = $('a[href*="msl-raw-images"]').last().closest('td').text().replace('Full Resolution', '').trim();

			helpers.output('Inserting image record into database ... Remaining insertions: ' + images.length);

			db.insert(imgData, function() {
				parseImageData(images);
			});
		});
	}

	function parseImages(callback) {
		scrape('http://mars.jpl.nasa.gov/msl/admin/modules/multimedia/module/inc_ListImages_Raw4.cfm?s=' + parseInt(sol, 10), function($) {
			helpers.output('Parsing images ...');

			var images = [];
			$('.RawImageCaption').each(function() {
				var root = $(this)
					, imgLink = $(this).closest('td').find('a[href*="?rawid="]')
					, webImg = imgLink.find('img');

				var data = {
					sol: sol
					, url: 'http://mars.jpl.nasa.gov/msl/multimedia/raw/' + imgLink.attr('href').replace('./', '')
					, created_on: new Date().toISOString()
					, type: (webImg.attr('width') == 64 || webImg.attr('height') == 64) ? 'thumbnail' : 'full'
					, camera: webImg.attr('alt').replace('Image taken by ', '')
					, captured_time: $('.RawImageUTC', root).text().replace('Full Resolution', '').trim()
					, image: {
						raw: $('a:contains("Full Resolution")', root).attr('href')
						, web: webImg.attr('src')
					}
				};

				images.push(data);
			});

			parseImageData(images);
		});
	}

	function parseSlides() {
		if (solSlides.length <= 0) return;
		
		sol = solSlides.shift();
		helpers.output('Scraping ' + sol + ' ...');

		parseWeather(function(err, resp, data) {
			data = JSON.parse(data);
			if (data.count > 0) weatherData = data.results;
			parseImages();
		});
	
	}

	function startScraping(index) {
		helpers.output('Requesting latest rover data ...');
		scrape('http://mars.jpl.nasa.gov/msl/multimedia/raw/?s=', function($) {
			var items = $('.scroll-content-item').slice(index || 0);
			if (items.length > 0) {
				items.each(function() {
					solSlides.push($(this).text());
				});
				parseSlides();
			} else {
				// Up-to-date stuff
			}
		});
	}

	if (total > 0) {
		db.find({}).sort({ created_on: -1 }).limit(1).exec(function(err, docs) {
			startScraping(docs[0].sol);
		});
	} else {
		startScraping();
	}

}

<commit_msg>setupDb checks for missing links<commit_after>var Datastore = require('nedb')
	, request = require('request')
	, helpers = require('../modules/helpers.js')
	, scrape = require('../modules/scrape.js').scrape;

var db = new Datastore({ filename: './datastore/data.db', autoload: true });

function setupDb(total) {

	var solSlides = []
		, weatherData, sol;

	function parseWeather(callback) {
		helpers.output('Requesting weather data ...');
		request('http://marsweather.ingenology.com/v1/archive/?sol=' + parseInt(sol, 10), callback);
	}

	function parseImageData(images) {
		if (images.length <= 0) {
			parseSlides();
			return;
		}

		var imgData = images.shift();		

		if (weatherData) imgData.weather = weatherData;

		scrape(imgData.url, function($) {
			imgData.description = $('a[href*="msl-raw-images"]').last().closest('td').text().replace('Full Resolution', '').trim();

			helpers.output('Inserting image record into database ... Remaining insertions: ' + images.length);

			db.insert(imgData, function() {
				parseImageData(images);
			});
		});
	}

	function parseImages(callback) {
		scrape('http://mars.jpl.nasa.gov/msl/admin/modules/multimedia/module/inc_ListImages_Raw4.cfm?s=' + parseInt(sol, 10), function($) {
			helpers.output('Parsing images ...');

			var images = []
				, imageEls = []
				, jQuery = $
				, isIncomplete = false;

			$('.RawImageCaption').each(function() {
				imageEls.push($(this));
			});

			db.find({ sol: sol }, function(err, docs) {
				if (docs.length > imageEls.length) {
					helpers.output('Database discrepency found for sol ' + sol);
				} else if (docs.length < imageEls.length || docs.length <= 0) {
					isIncomplete = true;
				}

				function verifyImage() {
					if (imageEls.length <= 0) {
						parseImageData(images);
						return;
					}

					var img = imageEls.shift()
						, root = img
						, imgLink = img.closest('td').find('a[href*="?rawid="]')
						, webImg = imgLink.find('img')
						, imgUrl = 'http://mars.jpl.nasa.gov/msl/multimedia/raw/' + imgLink.attr('href').replace('./', '');

					var data = {
						sol: sol
						, url: imgUrl
						, created_on: new Date().toISOString()
						, type: (webImg.attr('width') == 64 || webImg.attr('height') == 64) ? 'thumbnail' : 'full'
						, camera: webImg.attr('alt').replace('Image taken by ', '')
						, captured_time: img.find('.RawImageUTC').text().replace('Full Resolution', '').trim()
						, image: {
							raw: img.find('a:contains("Full Resolution")').attr('href')
							, web: webImg.attr('src')
						}
					};

					if (isIncomplete) {
						db.find({ url: imgUrl }, function(err, docs) {
							if (docs.length <= 0) images.push(data);
							verifyImage();
						});
					}
				}

				verifyImage();
			});
			
		});
	}

	function parseSlides() {
		if (solSlides.length <= 0) return;
		
		sol = solSlides.shift();
		helpers.output('Scraping ' + sol + ' ...');

		parseWeather(function(err, resp, data) {
			data = JSON.parse(data);
			if (data.count > 0) weatherData = data.results;
			parseImages();
		});
	
	}

	function startScraping(index) {
		helpers.output('Requesting latest rover data ...');
		scrape('http://mars.jpl.nasa.gov/msl/multimedia/raw/?s=', function($) {

			$('.scroll-content-item').each(function(i) {
				if ($(this).text() == index) {
					index = i;
					return;
				}
			});

			var items = $('.scroll-content-item').slice(index || 0);
			if (items.length > 0) {
				items.each(function() {
					solSlides.push($(this).text());
				});
				parseSlides();
			} else {
				// Up-to-date stuff
			}
		});
	}

	if (total > 0) {
		db.find({}).sort({ created_on: -1 }).limit(1).exec(function(err, docs) {
			startScraping(docs[0].sol);
		});
	} else {
		startScraping();
	}

}


{'fraction_non_alphanumeric': 0.20067527891955372, 'fraction_numerical': 0.00631238990017616, 'mean_word_length': 3.156802928615009, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5679841', 'n_tokens_mistral': 2874, 'n_tokens_neox': 2349, 'n_words': 413}