<commit_before>var express = require('express'),
    config = require('./config'),
    panlex = require('panlex'),
    http = require('http'),
    path = require('path'),
    fs = require('fs'),
    qs = require('qs'),
    sprintf = require('sprintf').sprintf,
    XRegExp = require('xregexp').XRegExp,
    unorm = require('unorm');

var app = express();

app.configure(function(){
  app.set('port', config.port || 3000);
  app.set('views', __dirname + '/views');
  app.set('view engine', 'jade');
  
  app.use(express.favicon());
  app.use(express.logger('dev'));
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(setHeaders);
  app.use(prepareRequest);
  app.use(app.router);
  app.use(express.static(path.join(__dirname, 'public')));
  app.use(errorResponse);
});

app.configure('development', function(){
  app.use(express.errorHandler());
});

app.locals({
  base: config.base,
  urlroot: '/demo',
  lcvcUid: function(obj) {
    return sprintf('%s-%03d', obj.lc, obj.vc);
  }
});

loadLanguages();

app.get('/', index);
app.post('/1', op1);
app.post('/2', op2);

http.createServer(app).listen(app.get('port'), function(){
  console.log("Express server listening on port " + app.get('port'));
});

function loadLanguages() {
  var lgs = {};
  
  fs.readdirSync(path.join(__dirname, 'i18n')).forEach(function (file) {
    if (!file.match(/\.json$/)) return;
    var lg = file.replace(/\.json/, '');
    lgs[lg] = require(path.join(__dirname, 'i18n', file));
    lgs[lg].lc = lg;
  });
  
  app.set('lgs', lgs);
}

function setHeaders(req, res, next) {
  res.set('Expires', 0);
  next();
}

function prepareRequest(req, res, next) {
  var lgs = app.get('lgs'),
      lg = req.query.lg || 'eng';
    
  if (lgs[lg] === undefined) next('TeraDict error: unknown language "' + lg + '"');
  
  res.locals.lg = res.lg = lgs[lg];
  res.locals.subheading = res.lg.teradict_description;
  res.locals.qs = makeQuery(req, res);
  
  for (var i in req.body) {
    if (req.body[i] !== undefined) {
      req.body[i] = req.body[i].trim();
      res.locals[i] = req.body[i];
    }
  }
  
  next();
}

function makeQuery(req) {
    return function(p) {
        var txt;
        if (p) {
          var query = {};
          for (var i in req.query) query[i] = req.query[i];
          for (var i in p) {
              if (p[i] === undefined) delete query[i];
              else query[i] = p[i];
          }
          txt = qs.stringify(query);        
        }
        else txt = qs.stringify(req.query);
        
        if (txt !== '') txt = '?' + txt;
        return txt;
    }
}

function errorResponse(err, req, res, next) {
  res.render('error', { error: err });
}

function index(req, res, next) {
  res.render('index', { et0: '' });
}

function op1(req, res, next) {
  if (blank(req.body, ['et0'])) return res.render('index');
  var et0 = normalize(req.body.et0);
  
  panlex.query('/ex', { lv: [res.lg.lv], tt: [et0] }, function (err, data) {
    if (err) return next(err);
    if (data.resultNum === 0) return res.render('index', { subheading: res.lg.ex_not_found});
    
    var ex0 = data.result[0].ex;
    panlex.queryAll('/lv', { tr: [ex0], sort: ["lc","vc"] }, function (err, data) {
      if (err) return next(err);
      if (data.resultNum === 0) res.render('index', { subheading: res.lg.tr_not_found });
      else res.render('op1', { result: data.result, ex0: ex0 });
    });
  });
}

function op2(req, res, next) {
  if (blank(req.body, ['et0', 'ex0', 'lv1']))
    return next('TeraDict error: missing required parameter');

  var ex0 = Number(req.body.ex0),
      lv1 = res.locals.lv1 = JSON.parse(req.body.lv1);
  
  panlex.queryAll('/tr', { ex: [ex0], lv: [lv1.lv], sort: ["tt"] }, function (err, data) {
    if (err) return next(err);
    res.render('op2', { result: data.result });<commit_msg>update teradict for new api<commit_after>var express = require('express'),
    config = require('./config'),
    panlex = require('panlex'),
    http = require('http'),
    path = require('path'),
    fs = require('fs'),
    qs = require('qs'),
    sprintf = require('sprintf').sprintf,
    XRegExp = require('xregexp').XRegExp,
    unorm = require('unorm');

var app = express();

app.configure(function(){
  app.set('port', config.port || 3000);
  app.set('views', __dirname + '/views');
  app.set('view engine', 'jade');
  
  app.use(express.favicon());
  app.use(express.logger('dev'));
  app.use(express.bodyParser());
  app.use(express.methodOverride());
  app.use(setHeaders);
  app.use(prepareRequest);
  app.use(app.router);
  app.use(express.static(path.join(__dirname, 'public')));
  app.use(errorResponse);
});

app.configure('development', function(){
  app.use(express.errorHandler());
});

app.locals({
  base: config.base,
  urlroot: '/demo',
  lcvcUid: function(obj) {
    return sprintf('%s-%03d', obj.lc, obj.vc);
  }
});

loadLanguages();

app.get('/', index);
app.post('/1', op1);
app.post('/2', op2);

http.createServer(app).listen(app.get('port'), function(){
  console.log("Express server listening on port " + app.get('port'));
});

function loadLanguages() {
  var lgs = {};
  
  fs.readdirSync(path.join(__dirname, 'i18n')).forEach(function (file) {
    if (!file.match(/\.json$/)) return;
    var lg = file.replace(/\.json/, '');
    lgs[lg] = require(path.join(__dirname, 'i18n', file));
    lgs[lg].lc = lg;
  });
  
  app.set('lgs', lgs);
}

function setHeaders(req, res, next) {
  res.set('Expires', 0);
  next();
}

function prepareRequest(req, res, next) {
  var lgs = app.get('lgs'),
      lg = req.query.lg || 'eng';
    
  if (lgs[lg] === undefined) next('TeraDict error: unknown language "' + lg + '"');
  
  res.locals.lg = res.lg = lgs[lg];
  res.locals.subheading = res.lg.teradict_description;
  res.locals.qs = makeQuery(req, res);
  
  for (var i in req.body) {
    if (req.body[i] !== undefined) {
      req.body[i] = req.body[i].trim();
      res.locals[i] = req.body[i];
    }
  }
  
  next();
}

function makeQuery(req) {
    return function(p) {
        var txt;
        if (p) {
          var query = {};
          for (var i in req.query) query[i] = req.query[i];
          for (var i in p) {
              if (p[i] === undefined) delete query[i];
              else query[i] = p[i];
          }
          txt = qs.stringify(query);        
        }
        else txt = qs.stringify(req.query);
        
        if (txt !== '') txt = '?' + txt;
        return txt;
    }
}

function errorResponse(err, req, res, next) {
  res.render('error', { error: err });
}

function index(req, res, next) {
  res.render('index', { et0: '' });
}

function op1(req, res, next) {
  if (blank(req.body, ['et0'])) return res.render('index');
  var et0 = normalize(req.body.et0);
  
  panlex.query('/ex', { lv: [res.lg.lv], tt: [et0] }, function (err, data) {
    if (err) return next(err);
    if (data.resultNum === 0) return res.render('index', { subheading: res.lg.ex_not_found});
    
    var ex0 = data.result[0].ex;
    panlex.queryAll('/lv', { tr: [ex0], sort: ["lc","vc"] }, function (err, data) {
      if (err) return next(err);
      if (data.resultNum === 0) res.render('index', { subheading: res.lg.tr_not_found });
      else res.render('op1', { result: data.result, ex0: ex0 });
    });
  });
}

function op2(req, res, next) {
  if (blank(req.body, ['et0', 'ex0', 'lv1']))
    return next('TeraDict error: missing required parameter');

  var ex0 = Number(req.body.ex0),
      lv1 = res.locals.lv1 = JSON.parse(req.body.lv1);
  
  panlex.queryAll('/ex', { tr: [ex0], lv: [lv1.lv], sort: ["tt"] }, function (err, data) {
    if (err) return next(err);
    res.render('op2', { result: data.result });
{'fraction_non_alphanumeric': 0.20859375, 'fraction_numerical': 0.010677083333333334, 'mean_word_length': 3.0490247759620455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2899825', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2804, 'n_words': 586}