<commit_before>if (typeof exports != "undefined") {
    
    var tio = {};
    var tio_args = [];

    var coopy = exports;
    var fs = require('fs');
    var exec = require('child_process').exec;
    var readline = null;
    var Fiber = null;
    var sqlite3 = null;
    var tty = null;
    
    tio.getContent = function(name) {
	if (name=="-") {
	    // only works on Linux, all other solutions seem broken
	    return fs.readFileSync('/dev/stdin',"utf8");
	}
	return fs.readFileSync(name,"utf8");
    }
    
    tio.saveContent = function(name,txt) {
	return fs.writeFileSync(name,txt,"utf8");
    }
    
    tio.args = function() {
	return tio_args;
    }
    
    tio.writeStdout = function(txt) {
	process.stdout.write(txt);
    }
    
    tio.writeStderr = function(txt) {
	process.stderr.write(txt);
    }
    
    tio.async = function() {
	return true;
    }

    tio.exists = function(path) {
	return fs.existsSync(path);
    }

    tio.isTtyKnown = function() {
        return true;
    }

    tio.isTty = function() {
        if (typeof process.stdout.isTTY !== 'undefined') {
            if (process.stdout.isTTY) return true;
        } else {
            // fall back on tty api
            if (tty==null) tty = require('tty');
            if (tty.isatty(process.stdout.fd)) return true;
        }
        // There's a wrinkle when called from git.  Git may have started a pager that
        // respects color but which will not be detected as a terminal.  In this case,
        // it appears that git defines GIT_PAGER_IN_USE, so we watch out for that.
        if (process.env.GIT_PAGER_IN_USE == 'true') return true;
        return false;
    }

    tio.openSqliteDatabase = function(path) {
	if (Fiber) {
	    return new SqliteDatabase(new sqlite3.Database(path),Fiber);
	}
	throw("run inside Fiber plz");
	return null;
    }

    tio.sendToBrowser = function(html) {
        var http = require("http");
        var server = http.createServer(function(request, response) {
            response.writeHead(200, 
                               {
                                   "Content-Type": "text/html; charset=UTF-8",
                                   "Connection": "close"
                               });
            response.write(html);
            response.end();
            server.close();
        });
        server.listen(0);
        var target = "http://" + server.address().address + ":" + server.address().port;
        var exec = require('child_process').exec;
        var cmd = "xdg-open";
        switch (process.platform) {
        case 'darwin':
            cmd = 'open';
            break;
        case 'win32':
            cmd = 'start ""';
            break;
        }
        exec(cmd + ' "' + target + '"', function(error) { 
            if(error) {
                console.error(error);
                server.close();
            }
        });
    }

    var cmd_result = 1;
    var cmd_pending = null;

    tio.command = function(cmd,args) {
	// we promise not to use any arguments with quotes in them
	for (var i=0; i<args.length; i++) {
	    var argi = args[i];
	    if (argi.indexOf(" ")>=0) {
		argi = "\"" + argi + "\"";<commit_msg>tweak node web server to remove possible races, see #21<commit_after>if (typeof exports != "undefined") {
    
    var tio = {};
    var tio_args = [];

    var coopy = exports;
    var fs = require('fs');
    var exec = require('child_process').exec;
    var readline = null;
    var Fiber = null;
    var sqlite3 = null;
    var tty = null;
    
    tio.getContent = function(name) {
	if (name=="-") {
	    // only works on Linux, all other solutions seem broken
	    return fs.readFileSync('/dev/stdin',"utf8");
	}
	return fs.readFileSync(name,"utf8");
    }
    
    tio.saveContent = function(name,txt) {
	return fs.writeFileSync(name,txt,"utf8");
    }
    
    tio.args = function() {
	return tio_args;
    }
    
    tio.writeStdout = function(txt) {
	process.stdout.write(txt);
    }
    
    tio.writeStderr = function(txt) {
	process.stderr.write(txt);
    }
    
    tio.async = function() {
	return true;
    }

    tio.exists = function(path) {
	return fs.existsSync(path);
    }

    tio.isTtyKnown = function() {
        return true;
    }

    tio.isTty = function() {
        if (typeof process.stdout.isTTY !== 'undefined') {
            if (process.stdout.isTTY) return true;
        } else {
            // fall back on tty api
            if (tty==null) tty = require('tty');
            if (tty.isatty(process.stdout.fd)) return true;
        }
        // There's a wrinkle when called from git.  Git may have started a pager that
        // respects color but which will not be detected as a terminal.  In this case,
        // it appears that git defines GIT_PAGER_IN_USE, so we watch out for that.
        if (process.env.GIT_PAGER_IN_USE == 'true') return true;
        return false;
    }

    tio.openSqliteDatabase = function(path) {
	if (Fiber) {
	    return new SqliteDatabase(new sqlite3.Database(path),Fiber);
	}
	throw("run inside Fiber plz");
	return null;
    }

    tio.sendToBrowser = function(html) {
        var http = require("http");
	var shutdown = null;
        var server = http.createServer(function(request, response) {
            response.writeHead(200, 
                               {
                                   "Content-Type": "text/html; charset=UTF-8",
                                   "Connection": "close"
                               });
            response.write(html);
            response.end();
	    shutdown();
        });
	var sockets = {}, nextSocketId = 0;
	server.on('connection', function (socket) {
	    var socketId = nextSocketId++;
	    sockets[socketId] = socket;
	    socket.on('close', function () {
		delete sockets[socketId];
	    });
	});
	shutdown = function() {
	    server.close();
	    for (var socketId in sockets) {
		sockets[socketId].destroy();
	    }
	};
        server.listen(0,null,null,function() {
            var target = "http://localhost:" + server.address().port;
            var exec = require('child_process').exec;
            var cmd = "xdg-open";
            switch (process.platform) {
            case 'darwin':
		cmd = 'open';
		break;
            case 'win32':
		cmd = 'start ""';
		break;
            }
            exec(cmd + ' "' + target + '"', function(error) { 
		if (error) {
                    console.error(error);
                    server.close();
		}
            });
	});
    }

    var cmd_result = 1;
    var cmd_pending = null;

    tio.command = function(cmd,args) {
	// we promise not to use any arguments with quotes in them
	for (var i=0; i<args.length; i++) {
	    var argi = args[i];
	    if (argi.indexOf(" ")>=0) {
		argi = "\"" + argi + "\"";
{'fraction_non_alphanumeric': 0.1353905211870663, 'fraction_numerical': 0.004872287021999114, 'mean_word_length': 1.861850443599493, 'pattern_counts': {'":': 5, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4959979', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2088, 'n_words': 511}