<commit_before>var express = require('express');
var http = require('http');
var path = require('path');
var textStat = require('text-statistics');
var Boilerpipe = require('boilerpipe');

var app = express();

// all environments
app.set('port', process.env.PORT || 3020);
app.set('views', path.join(__dirname, ''));
app.engine('html', require('ejs').renderFile);
app.use(express.logger('dev'));
app.use(express.json());
app.use(express.urlencoded());
app.use(express.methodOverride());
app.use(express.bodyParser());


app.use(app.router);
app.configure(function() {
  app.use(express.static(path.join(__dirname, 'public')));
});

// development only
if ('development' == app.get('env')) {
  app.use(express.errorHandler());
}

app.get('/', function(req, res){
    res.send(200, {});
});

app.post('/score', function(req, res){
    var params = req.body;
    var readlevel = null;

    console.log("[POST/score] Params: ");
    console.dir(params);

    var boilerpipe = new Boilerpipe({
        extractor: Boilerpipe.Extractor.Article,
        url: params.link
    });

    boilerpipe.getText(function(err, text) {
        if (err) {
            console.log("[POST/score] ERROR: (boilerpipe)" + err);
            res.send(500);
            return;
        }
        console.log("[POST/score] boilerpipe.err: " + err);
        console.log("[POST/score] boilerpipe.text: " + text);
        var ts = textStat(text);
        readlevel = ts.fleschKincaidGradeLevel();
        res.send(200, {score: readlevel});
    });
});<commit_msg>added average grade level<commit_after>var express = require('express');
var http = require('http');
var path = require('path');
var textStat = require('text-statistics');
var Boilerpipe = require('boilerpipe');

var app = express();

// all environments
app.set('port', process.env.PORT || 3020);
app.set('views', path.join(__dirname, ''));
app.engine('html', require('ejs').renderFile);
app.use(express.logger('dev'));
app.use(express.json());
app.use(express.urlencoded());
app.use(express.methodOverride());
app.use(express.bodyParser());


app.use(app.router);
app.configure(function() {
  app.use(express.static(path.join(__dirname, 'public')));
});

// development only
if ('development' == app.get('env')) {
  app.use(express.errorHandler());
}

app.get('/', function(req, res){
    res.send(200, {});
});

app.post('/score', function(req, res){
    var params = req.body;
    var readlevel = null;

    console.log("[POST/score] Params: ");
    console.dir(params);

    var boilerpipe = new Boilerpipe({
        extractor: Boilerpipe.Extractor.Article,
        url: params.link
    });

    boilerpipe.getText(function(err, text) {
        if (err) {
            console.log("[POST/score] ERROR: (boilerpipe)" + err);
            res.send(500);
            return;
        }
        console.log("[POST/score] boilerpipe.err: " + err);
        console.log("[POST/score] boilerpipe.text: " + text);
        var ts = textStat(text);
	
        fkgl = ts.fleschKincaidGradeLevel();
	gfs = ts.gunningFogScore();
	cli = ts.colemanLiauIndex();
	si = ts.smogIndex();	
	ari = automatedReadabilityIndex();

	avgGradeLvl = (fkgl + gfs + cli + si + ari)/5;
        res.send(200, {score: avgGradeLvl});
    });
});
{'fraction_non_alphanumeric': 0.1918568784700802, 'fraction_numerical': 0.008328192473781616, 'mean_word_length': 3.5547752808988764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4752349', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1061, 'n_words': 197}