<commit_before>var q = require('q');
var https = require('https');
var cheerio = require('cheerio');
var fs = require('fs');
var url = require('url');

const MAX_PAGES = 10;
var hostname = 'unsplash.com';
var image_num = 1;
var page = 1;

function start() {
	getPageNum(page++)
	.then(function() {
		if (page <= MAX_PAGES) {
			start();
		}
	})
	.catch(function(err) {
		console.log(err);
		console.trace(err);
	});
}

function getPageNum(page) {
	return getPage(hostname, '/?page=' + page.toString())
	.then(function(data) {
		var $ = cheerio.load(data);
		var images = $('img[alt^="Photo By"]');
		if (images.length === 0) {
			console.log('Done!');
			return exit(1);
		}
	
		return q.all(images.map(function() {
			var image_url = url.parse($(this).attr('src'));
			return getPage(image_url.host, image_url.path);
		}));
	})
	.then(function(images) {
		return q.all(Object.keys(images).map(function(i) {
			var img = images[i];
			return q.nfcall(fs.writeFile, 'images/' + image_num++ + '.jpg', img); 
		}));
	})
	.then(function() {
		console.log('Page %d downloaded', page);
	});
}

function getPage(hostname, url) {
	console.log('getPage', hostname, url);
	return q.promise(function(resolve, reject) {
		https.request({
			hostname: hostname,
			port: 443,
			path: url,
			agent: false,
			method: 'GET',
			rejectUnauthorized: false,
		}, function(res) {
			var buf = new Buffer(parseInt(res.headers['content-length'], 10) + 1);
			var pos = 0;

			res.on('data', function(d) {
				d.copy(buf, pos, 0);
				pos += d.length;
			});<commit_msg>Use high quality version of images<commit_after>var q = require('q');
var https = require('https');
var cheerio = require('cheerio');
var fs = require('fs');
var url = require('url');

const MAX_PAGES = 10;
var hostname = 'unsplash.com';
var image_num = 1;
var page = 1;

function start() {
	getPageNum(page++)
	.then(function() {
		if (page <= MAX_PAGES) {
			start();
		}
	})
	.catch(function(err) {
		console.log(err);
		console.trace(err);
	});
}

function getPageNum(page) {
	return getPage(hostname, '/?page=' + page.toString())
	.then(function(data) {
		var $ = cheerio.load(data);
		var images = $('img[alt^="Photo By"]');
		if (images.length === 0) {
			console.log('Done!');
			return exit(1);
		}
	
		return q.all(images.map(function() {
			var image_url = url.parse($(this).attr('src'), true);
			return getPage(image_url.host, image_url.path.replace('q=80&fm=jpg&w=1080&fit=max&', ''));
		}));
	})
	.then(function(images) {
		return q.all(Object.keys(images).map(function(i) {
			var img = images[i];
			return q.nfcall(fs.writeFile, 'images/' + image_num++ + '.jpg', img); 
		}));
	})
	.then(function() {
		console.log('Page %d downloaded', page);
	});
}

function getPage(hostname, url) {
	console.log('getPage', hostname, url);
	return q.promise(function(resolve, reject) {
		https.request({
			hostname: hostname,
			port: 443,
			path: url,
			agent: false,
			method: 'GET',
			rejectUnauthorized: false,
		}, function(res) {
			var buf = new Buffer(parseInt(res.headers['content-length'], 10) + 1);
			var pos = 0;

			res.on('data', function(d) {
				d.copy(buf, pos, 0);
				pos += d.length;
			});
{'fraction_non_alphanumeric': 0.20031545741324921, 'fraction_numerical': 0.010725552050473186, 'mean_word_length': 3.931570762052877, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '990051', 'n_tokens_mistral': 1327, 'n_tokens_neox': 1180, 'n_words': 248}