<commit_before>const cors = require('cors'); 
const fs = require('fs');
const express = require('express');
const {VM} = require('vm2');

const app = express();
app.use(cors());

//configure ports, automatically choosing between the Heroku port and development port
var httpPort = Number(process.env.PORT || 3000);
var bookDir = process.env.BOOKDIR || '1';

var dir = "public/" + bookDir + "/";
var files = fs.readdirSync(dir);

var data = []
for (var i=0; i<files.length; i++) {
	if (files[i].startsWith(".")) 
		continue;

	var content = fs.readFileSync(dir + files[i], 'utf8');
	data.push(content);
}

app.get('/', function (req, res) {
	res.send(JSON.stringify(data, null, 2));
})

app.get('/map', function (req, res) {
	var vm = new VM({
    	timeout: 30000,
    	sandbox: {
    		map: function(input) { return input; },
    		data: data,
    		__codeBlockCounter__: 0
    	}
	});
	if (!req.query.mapfcn) req.query.mapfcn = '';
	
	try {
		var rawresponse = vm.run(req.query.mapfcn + 'data.map(map);');
		res.send(JSON.stringify(rawresponse));
	} catch (e) {
		res.send(e.name);
	}
})

app.get('/map_combine', function (req, res) {
	var vm = new VM({
    	timeout: 30000,
    	sandbox: {
    		map: function(input) { return input; },
    		combine: function(input) {return input},
    		data: data,
    		cmRunStepCount: 0
    	}
	});
	
	if (!req.query.mapfcn) req.query.mapfcn = '';
	if (!req.query.combinefcn) req.query.combinefcn = '';

	try {
		var rawresponse = vm.run(req.query.mapfcn + " " + req.query.combinefcn + 'var result = data.map(map); combine(result);');
		res.send(JSON.stringify(rawresponse));
	} catch (e) {
		res.send(e.name);<commit_msg>update var names<commit_after>const cors = require('cors'); 
const fs = require('fs');
const express = require('express');
const {VM} = require('vm2');

const app = express();
app.use(cors());

//configure ports, automatically choosing between the Heroku port and development port
var httpPort = Number(process.env.PORT || 3000);
var bookDir = process.env.BOOKDIR || '1';

var dir = "public/" + bookDir + "/";
var files = fs.readdirSync(dir);

var data = []
for (var i=0; i<files.length; i++) {
	if (files[i].startsWith(".")) 
		continue;

	var content = fs.readFileSync(dir + files[i], 'utf8');
	data.push(content);
}

app.get('/', function (req, res) {
	res.send(JSON.stringify(data, null, 2));
})

app.get('/map', function (req, res) {
	var vm = new VM({
    	timeout: 30000,
    	sandbox: {
    		map: function(input) { return input; },
    		data: data,
    		__codeBlockCounter__: 0
    	}
	});
	if (!req.query.mapfcn) req.query.mapfcn = '';
	
	try {
		var rawresponse = vm.run(req.query.mapfcn + 'data.map(map);');
		res.send(JSON.stringify(rawresponse));
	} catch (e) {
		res.send(e.name);
	}
})

app.get('/map_combine', function (req, res) {
	var vm = new VM({
    	timeout: 30000,
    	sandbox: {
    		map: function(input) { return input; },
    		combine: function(input) {return input},
    		data: data,
    		__codeBlockCounter__: 0
    	}
	});
	
	if (!req.query.mapfcn) req.query.mapfcn = '';
	if (!req.query.combinefcn) req.query.combinefcn = '';

	try {
		var rawresponse = vm.run(req.query.mapfcn + " " + req.query.combinefcn + 'var result = data.map(map); combine(result);');
		res.send(JSON.stringify(rawresponse));
	} catch (e) {
		res.send(e.name);
{'fraction_non_alphanumeric': 0.186018601860186, 'fraction_numerical': 0.012601260126012601, 'mean_word_length': 3.5360544217687075, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3149619', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1249, 'n_words': 277}