<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
const http = require('http');
const express = require('express');
const consola = require('consola');
const {Nuxt, Builder} = require('nuxt');
const MongoClient = require('mongodb').MongoClient;
const universalAnalytics = require('universal-analytics');
const rp = require(`request-promise`);
const asyncHandler = fn => (req, res, next) =>
    Promise
        .resolve(fn(req, res, next))
        .catch(next);
const logger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'index'
});

const apiLogger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'api'
});

const perfLogger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'perf'
});

const logReqPerf = function (req, res, next) {
  // Credit for inspiration: http://www.sheshbabu.com/posts/measuring-response-times-of-express-route-handlers/
  perfLogger.info(` log request starts for ${req.method} ${req.originalUrl}:`, {
    method: req.method,
    original_url: req.originalUrl,
    ga_id: req.cookies._ga,
  });
  const startNs = process.hrtime.bigint();
  res.on(`finish`, () => {
    const endNs = process.hrtime.bigint();
    perfLogger.info(` log response ends for ${req.method} ${req.originalUrl}:`, {
      method: req.method,
      original_url: req.originalUrl,
      ga_id: req.cookies._ga,
      time_lapse_ns: endNs - startNs,
      start_ns: startNs,
      end_ns: endNs
    });
  });
  next();
};

let docCounter = 0;
let allDocCounter = 0;
function isEmpty(value) {
  return typeof value == 'string' && !value.trim() || typeof value == 'undefined' || value === null;
}

const useStiki = isEmpty(process.env.STIKI_MYSQL);
const useOauth = isEmpty(process.env.MEDIAWIKI_CONSUMER_SECRET) && isEmpty(process.env.MEDIAWIKI_CONSUMER_KEY);
// Import and Set Nuxt.js options
const config = require('../nuxt.config.js');
config.dev = !(process.env.NODE_ENV === 'production');

function computeOresField(oresJson, wiki, revId) {<commit_msg>Fix flag retrieval<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
require(`dotenv`).config();
const http = require('http');
const express = require('express');
const consola = require('consola');
const {Nuxt, Builder} = require('nuxt');
const MongoClient = require('mongodb').MongoClient;
const universalAnalytics = require('universal-analytics');
const rp = require(`request-promise`);
const asyncHandler = fn => (req, res, next) =>
    Promise
        .resolve(fn(req, res, next))
        .catch(next);
const logger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'index'
});

const apiLogger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'api'
});

const perfLogger = new (require('heroku-logger').Logger)({
  level: process.env.LOG_LEVEL || 'debug',
  delimiter: " | ",
  prefix: 'perf'
});

const logReqPerf = function (req, res, next) {
  // Credit for inspiration: http://www.sheshbabu.com/posts/measuring-response-times-of-express-route-handlers/
  perfLogger.info(` log request starts for ${req.method} ${req.originalUrl}:`, {
    method: req.method,
    original_url: req.originalUrl,
    ga_id: req.cookies._ga,
  });
  const startNs = process.hrtime.bigint();
  res.on(`finish`, () => {
    const endNs = process.hrtime.bigint();
    perfLogger.info(` log response ends for ${req.method} ${req.originalUrl}:`, {
      method: req.method,
      original_url: req.originalUrl,
      ga_id: req.cookies._ga,
      time_lapse_ns: endNs - startNs,
      start_ns: startNs,
      end_ns: endNs
    });
  });
  next();
};

let docCounter = 0;
let allDocCounter = 0;
function isEmpty(value) {
  return typeof value == 'string' && !value.trim() || typeof value == 'undefined' || value === null;
}

const useStiki = !isEmpty(process.env.STIKI_MYSQL);
const useOauth = !isEmpty(process.env.MEDIAWIKI_CONSUMER_SECRET) && !isEmpty(process.env.MEDIAWIKI_CONSUMER_KEY);
console.log(`XXX process.env.MEDIAWIKI_CONSUMER_SECRET = ${process.env.MEDIAWIKI_CONSUMER_SECRET}, process.env.MEDIAWIKI_CONSUMER_KEY = ${process.env.MEDIAWIKI_CONSUMER_KEY}`)
// Import and Set Nuxt.js options
const config = require('../nuxt.config.js');
config.dev = !(process.env.NODE_ENV === 'production');

function computeOresField(oresJson, wiki, revId) {
{'fraction_non_alphanumeric': 0.15024893314367, 'fraction_numerical': 0.0035561877667140826, 'mean_word_length': 4.090497737556561, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1303895', 'n_tokens_mistral': 1807, 'n_tokens_neox': 1778, 'n_words': 482}