<commit_before>// builtin
var https = require('https');

// vendor
var _ = require('underscore');

// local
var converter = require('./converter');

var options = {
			title: /^title: (.*)/m,
			theme: /^theme: (.*)/m,
			transition: /^transition: (.*)/m,
			horizontalOnly: /^horizOnly:(.*)/m
		},
		defaults = {
			title: "RevealMe Presentations",
			theme: "default",
			transition: "default",
			horizontalOnly: false
		},
		isgist = /\/gist\/([a-zA-Z0-9]+)\/([a-f0-9]+)\/?([a-f0-9]+)?\/?(.+\.(html|md))?/;

module.exports = retrieve;

function retrieve(path, callback) {
	var m;
	if (m = path.match(isgist)) {
		var url = 'https://gist.github.com/'
			+ m[1]
			+ '/' + m[2]
			+ '/raw'
			+ (m[3] ? '/'+m[3] : '')
			+ '/' + (m[4] ? m[4] : 'README.md');

		getFile(url, callback)
	} else {
		getFile('https://raw.github.com' + path, callback);
	}
}

function getFile(url, callback) {
	console.log('Retrieving ' + url);
	https.get(url, function (res) {
		if (res.statusCode != 200) {
			callback(new Error("non-200 status code: " + res.statusCode));
			return;
		}

		var buf = "";

		res.on('data', function (data) {
			buf += data;
		});

		res.on('error', callback);

		res.on('end', function () {
			callback(null, buf, extractOptions(buf));
		});

	}).on('error', callback).end();
}

function extractOptions(data) {
	var opts = _.object(_.keys(defaults), _.values(defaults));
	_.each(_.pairs(options), function (pair) {
		var key = pair[0],
				rgx = pair[1],
				match = data.match(rgx);

		if (match) {
			opts[key] = match[1];
		}
	});
	return opts;
}
<commit_msg>change github raw url<commit_after>// builtin
var https = require('https');

// vendor
var _ = require('underscore');

// local
var converter = require('./converter');

var options = {
			title: /^title: (.*)/m,
			theme: /^theme: (.*)/m,
			transition: /^transition: (.*)/m,
			horizontalOnly: /^horizOnly:(.*)/m
		},
		defaults = {
			title: "RevealMe Presentations",
			theme: "default",
			transition: "default",
			horizontalOnly: false
		},
		isgist = /\/gist\/([a-zA-Z0-9]+)\/([a-f0-9]+)\/?([a-f0-9]+)?\/?(.+\.(html|md))?/;

module.exports = retrieve;

function retrieve(path, callback) {
	var m;
	if (m = path.match(isgist)) {
		var url = 'https://gist.github.com/'
			+ m[1]
			+ '/' + m[2]
			+ '/raw'
			+ (m[3] ? '/'+m[3] : '')
			+ '/' + (m[4] ? m[4] : 'README.md');

		getFile(url, callback)
	} else {
		getFile('https://raw.githubusercontent.com' + path, callback);
	}
}

function getFile(url, callback) {
	console.log('Retrieving ' + url);
	https.get(url, function (res) {
		if (res.statusCode != 200) {
			callback(new Error("non-200 status code: " + res.statusCode));
			return;
		}

		var buf = "";

		res.on('data', function (data) {
			buf += data;
		});

		res.on('error', callback);

		res.on('end', function () {
			callback(null, buf, extractOptions(buf));
		});

	}).on('error', callback).end();
}

function extractOptions(data) {
	var opts = _.object(_.keys(defaults), _.values(defaults));
	_.each(_.pairs(options), function (pair) {
		var key = pair[0],
				rgx = pair[1],
				match = data.match(rgx);

		if (match) {
			opts[key] = match[1];
		}
	});
	return opts;
}

{'fraction_non_alphanumeric': 0.23384809328710998, 'fraction_numerical': 0.013236684525685471, 'mean_word_length': 3.1328125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1942995', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1229, 'n_words': 241}