<commit_before>var http = require('http');
var express = require('express');
var fs = require('fs');
var csv = require('csv');
var StateTableFormatter = require('StateTableFormatter');

// Express
var app = express();
app.set('views', __dirname);
app.set('view engine', 'ejs');
app.use(express.errorHandler({ dumpExceptions:false, showStack:false }));

// Static resource passthrough
app.use("/js", express.static(__dirname + '/js'));
app.use("/stylesheets", express.static(__dirname + '/stylesheets'));
app.use("/images", express.static(__dirname + '/images'));

app.get('/', function(request, response) {
	response.render('index');
});

app.get('/file', function(request, response, next) {

	var country = (request.query.country || '').toLowerCase();
	var time = (request.query.time || '').toLowerCase();
	var occupied = (request.query.occupied || '').toLowerCase();
	var format = (request.query.format || '').toLowerCase();
	var division = (request.query.division || '').toLowerCase();
	var dc = ((request.query.dc || '').toLowerCase() === 'true');

	var sourceFile = csv().from.stream(fs.createReadStream(__dirname + '/state.csv'));
	var headers, formatter;
	
	if(format === '') {
		return next(new Error('invalid format'));
	}

	formatter = StateTableFormatter.factory(format);
	if(formatter === undefined) {
		return next(new Error('invalid format'));
	}

	// Download headers
	response.setHeader("Expires", "0");
	response.setHeader("Cache-Control", "must-revalidate, post-check=0, pre-check=0");
	response.setHeader("Content-Type", "application/force-download");
	response.setHeader("Content-Type: application/octet-stream");
	response.setHeader("Content-Type: application/download");
	response.setHeader("Content-Transfer-Encoding", "binary");

	sourceFile.on('record', function(row, index) {

		var header_index;

		if(headers === undefined) {

			// Header row
			headers = row;
			formatter.header(headers);

		}
		else {

			// Normal row
			var assocRow = [];
			for(header_index in headers) {
				assocRow[headers[header_index]] = row[header_index];
			}

			if(country !== 'all' && country !== assocRow['country'].toLowerCase()) {
				return;
			}

			if(time !== 'all' && time !== assocRow['status'].toLowerCase()) {
				return;
			}

			if(occupied !== 'all' && assocRow['type'].toLowerCase() == 'minor') {
				return;
			}

			if(division !== 'all' && assocRow['type'].toLowerCase() !== 'state' && assocRow['type'].toLowerCase() !== 'province' && assocRow['type'].toLowerCase() !== 'capitol' ) {
				return;
			}

			if(!dc && assocRow['type'].toLowerCase() === 'capitol') {
				return;
			}

			formatter.row(assocRow);
		}
	});

	sourceFile.on('end', function(count) {
		formatter.footer();
		response.setHeader("Content-Disposition", "attachment; filename=" + formatter.filename + ";");
		response.setHeader("Content-Length", formatter.output_buffer.output.length());
		response.send(formatter.output_buffer.render());<commit_msg>let's just leave off the conten length header for now<commit_after>var http = require('http');
var express = require('express');
var fs = require('fs');
var csv = require('csv');
var StateTableFormatter = require('StateTableFormatter');

// Express
var app = express();
app.set('views', __dirname);
app.set('view engine', 'ejs');
app.use(express.errorHandler({ dumpExceptions:false, showStack:false }));

// Static resource passthrough
app.use("/js", express.static(__dirname + '/js'));
app.use("/stylesheets", express.static(__dirname + '/stylesheets'));
app.use("/images", express.static(__dirname + '/images'));

app.get('/', function(request, response) {
	response.render('index');
});

app.get('/file', function(request, response, next) {

	var country = (request.query.country || '').toLowerCase();
	var time = (request.query.time || '').toLowerCase();
	var occupied = (request.query.occupied || '').toLowerCase();
	var format = (request.query.format || '').toLowerCase();
	var division = (request.query.division || '').toLowerCase();
	var dc = ((request.query.dc || '').toLowerCase() === 'true');

	var sourceFile = csv().from.stream(fs.createReadStream(__dirname + '/state.csv'));
	var headers, formatter;
	
	if(format === '') {
		return next(new Error('invalid format'));
	}

	formatter = StateTableFormatter.factory(format);
	if(formatter === undefined) {
		return next(new Error('invalid format'));
	}

	// Download headers
	response.setHeader("Expires", "0");
	response.setHeader("Cache-Control", "must-revalidate, post-check=0, pre-check=0");
	response.setHeader("Content-Type", "application/force-download");
	response.setHeader("Content-Type: application/octet-stream");
	response.setHeader("Content-Type: application/download");
	response.setHeader("Content-Transfer-Encoding", "binary");

	sourceFile.on('record', function(row, index) {

		var header_index;

		if(headers === undefined) {

			// Header row
			headers = row;
			formatter.header(headers);

		}
		else {

			// Normal row
			var assocRow = [];
			for(header_index in headers) {
				assocRow[headers[header_index]] = row[header_index];
			}

			if(country !== 'all' && country !== assocRow['country'].toLowerCase()) {
				return;
			}

			if(time !== 'all' && time !== assocRow['status'].toLowerCase()) {
				return;
			}

			if(occupied !== 'all' && assocRow['type'].toLowerCase() == 'minor') {
				return;
			}

			if(division !== 'all' && assocRow['type'].toLowerCase() !== 'state' && assocRow['type'].toLowerCase() !== 'province' && assocRow['type'].toLowerCase() !== 'capitol' ) {
				return;
			}

			if(!dc && assocRow['type'].toLowerCase() === 'capitol') {
				return;
			}

			formatter.row(assocRow);
		}
	});

	sourceFile.on('end', function(count) {
		formatter.footer();
		response.setHeader("Content-Disposition", "attachment; filename=" + formatter.filename + ";");
		response.send(formatter.output_buffer.render());
{'fraction_non_alphanumeric': 0.20057990789698107, 'fraction_numerical': 0.0010233668770254136, 'mean_word_length': 4.858141858141858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7091720', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1805, 'n_words': 344}