<commit_before>var fs = require("fs");
var path = require("path");
var ical2json = require("ical2json");
var _ = require("lodash");
var groups = require("./groups.json");

var icsDir = "ics";
var eventJson = "data/events.json";

function wholePath(file) {
	return path.join(icsDir, file);
}
function readUtf8(path) {
	return fs.readFileSync(path, { encoding: 'UTF8' });
}
function cal2vcal(cal) {
	return cal.VCALENDAR;
}
function cal2tz(cal) {
	return cal.VTIMEZONE || [];
}
function tz2event(tz) {
	if (tz.VTIMEZONE) {
		tz = tz.VTIMEZONE[0];
	}
	return tz.VEVENT;
}

function hiddenFile(file) {
	return file[0] === ".";
}

function isOld(event) {
	var start = toDate(startTime(event));

	var now = new Date(Date.now());
	now.setDate(now.getDate() - 1);

	return start < now;
}

function loadCal(filename) {
	var cal = cal2vcal(ical2json.convert(readUtf8(wholePath(filename))));
	var group = filename.substr(0, filename.indexOf("."));
	var events = _.chain(cal)
		.flatten()
		.map(cal2tz)
		.flatten()
		.map(tz2event)
		.flatten()
		.map(fixEvent.bind(undefined, group))
		.value();
	return events;
}

var events = _.chain(fs.readdirSync(icsDir))
	.reject(hiddenFile)
	.map(loadCal)
	.flatten()
	.sortBy(function(event) {
		return toDate(startTime(event));
	})
	.reject(isOld)
	.value();

function startTime(event) {
	for (var prop in event) {
		if (event.hasOwnProperty(prop)) {
			if (prop.indexOf("DTSTART") === 0) {
				return event[prop];
			}
		}
	}
}

function toDate(timeStr) {
	var parts = timeStr.split(":");

	var tz = "US-Eastern";
	var datetime = timeStr;
	if (parts.length > 1) {
		if (parts.indexOf("TZID=") === 0) {
			tz = parts.substr(5);
		}
		datetime = parts[1];
	}
	parts = datetime.split("T");
	var date = parts[0];
	var time = parts[1] || "";

	var year = parseInt(date.substr(0, 4));
	var month = parseInt(date.substr(4, 2)) - 1;
	var day = parseInt(date.substr(6, 2));
	var hour = parseInt(time.substr(0, 2)) || 0;
	var min = parseInt(time.substr(2, 2)) || 0;
	var sec = parseInt(time.substr(4, 2)) || 0;
	if (time[6] === "Z") {
		return new Date(Date.UTC(year, month, day, hour, min, sec));
	} else {
		return new Date(year, month, day, hour, min, sec);
	}
}

function fixText(text) {
	return text.replace(/\\n/g, "\n").replace(/\\,/g, ",");
}

function fixEvent(group, event) {
	event.SUMMARY = fixText(event.SUMMARY);
	event.DESCRIPTION = fixText(event.DESCRIPTION);
	event.startDate = toDate(startTime(event)).toString();
	event.jsonDate = toDate(startTime(event)).toJSON();
	event.group = group;
	event.groupName = groups[group].name;
	event.groupUrl = groups[group].web;
	return event;
}

fs.writeFileSync(eventJson, JSON.stringify(events, null, 4), { encoding: 'UTF8' });<commit_msg>Reorder file for readability top-to-bottom<commit_after>var fs = require("fs");
var path = require("path");
var ical2json = require("ical2json");
var _ = require("lodash");
var groups = require("./groups.json");

var icsDir = "ics";
var eventJson = "data/events.json";

var events = _.chain(fs.readdirSync(icsDir))
	.reject(hiddenFile)
	.map(loadCal)
	.flatten()
	.sortBy(function(event) {
		return toDate(startTime(event));
	})
	.reject(isOld)
	.value();

fs.writeFileSync(eventJson, JSON.stringify(events, null, 4), { encoding: 'UTF8' });

function hiddenFile(file) {
	return file[0] === ".";
}

function loadCal(filename) {
	var cal = cal2vcal(ical2json.convert(readUtf8(wholePath(filename))));
	var group = filename.substr(0, filename.indexOf("."));
	var events = _.chain(cal)
		.flatten()
		.map(cal2tz)
		.flatten()
		.map(tz2event)
		.flatten()
		.map(fixEvent.bind(undefined, group))
		.value();
	return events;
}

function wholePath(file) {
	return path.join(icsDir, file);
}
function readUtf8(path) {
	return fs.readFileSync(path, { encoding: 'UTF8' });
}
function cal2vcal(cal) {
	return cal.VCALENDAR;
}
function cal2tz(cal) {
	return cal.VTIMEZONE || [];
}
function tz2event(tz) {
	if (tz.VTIMEZONE) {
		tz = tz.VTIMEZONE[0];
	}
	return tz.VEVENT;
}

function isOld(event) {
	var start = toDate(startTime(event));

	var now = new Date(Date.now());
	now.setDate(now.getDate() - 1);

	return start < now;
}

function startTime(event) {
	for (var prop in event) {
		if (event.hasOwnProperty(prop)) {
			if (prop.indexOf("DTSTART") === 0) {
				return event[prop];
			}
		}
	}
}

function toDate(timeStr) {
	var parts = timeStr.split(":");

	var tz = "US-Eastern";
	var datetime = timeStr;
	if (parts.length > 1) {
		if (parts.indexOf("TZID=") === 0) {
			tz = parts.substr(5);
		}
		datetime = parts[1];
	}
	parts = datetime.split("T");
	var date = parts[0];
	var time = parts[1] || "";

	var year = parseInt(date.substr(0, 4));
	var month = parseInt(date.substr(4, 2)) - 1;
	var day = parseInt(date.substr(6, 2));
	var hour = parseInt(time.substr(0, 2)) || 0;
	var min = parseInt(time.substr(2, 2)) || 0;
	var sec = parseInt(time.substr(4, 2)) || 0;
	if (time[6] === "Z") {
		return new Date(Date.UTC(year, month, day, hour, min, sec));
	} else {
		return new Date(year, month, day, hour, min, sec);
	}
}

function fixText(text) {
	return text.replace(/\\n/g, "\n").replace(/\\,/g, ",");
}

function fixEvent(group, event) {
	event.SUMMARY = fixText(event.SUMMARY);
	event.DESCRIPTION = fixText(event.DESCRIPTION);
	event.startDate = toDate(startTime(event)).toString();
	event.jsonDate = toDate(startTime(event)).toJSON();
	event.group = group;
	event.groupName = groups[group].name;
	event.groupUrl = groups[group].web;
	return event;
}
{'fraction_non_alphanumeric': 0.18833424757725362, 'fraction_numerical': 0.015359297860669226, 'mean_word_length': 4.347018572825024, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2413903', 'n_tokens_mistral': 2195, 'n_tokens_neox': 2096, 'n_words': 442}