<commit_before>import path from 'path';
import express from 'express';
import webpack from 'webpack';
import webpackMiddleware from 'webpack-dev-middleware';
import webpackHotMiddleware from 'webpack-hot-middleware';
import wrap from 'express-async-wrap';
import fs from 'fs-promise';

import search from '../common/search';
import webpackConfig from './webpack.config.babel';
import macros from '../common/macros';

const app = express();


// Http to https redirect. 
app.use(function (req, res, next) {
  
  var remoteIp = req.connection.remoteAddress;


  // If this is https request, done. 
  if (req.protocol === 'https') {
    next()
  }

  // If we are behind a cloudflare proxy and cloudflare served a https response, done. 
  else if (req.headers['x-forwarded-proto'] && req.headers['x-forwarded-proto'] === 'https') {
    next()
  }

  // This is development mode
  else if (macros.DEV) {
    next()
  }

  // This is prod and still on http, redirect to https. 
  else {
    // Cache the http to https redirect for 2 months. 
    res.setHeader('Cache-Control', 'public, max-age=5256000');
    console.log('redirecting to https')
    res.redirect('https://' + req.get('host') + req.originalUrl);
  }
})



let searchPromise = null;

async function getSearch() {
  if (searchPromise) {
    return searchPromise;
  }

  const termDumpPromise = fs.readFile('./public/data/getTermDump/neu.edu/201810.json').then((body) => {
    return JSON.parse(body);
  });

  const searchIndexPromise = fs.readFile('./public/data/getSearchIndex/neu.edu/201810.json').then((body) => {
    return JSON.parse(body);
  });


  const employeeMapPromise = fs.readFile('./public/data/employeeMap.json').then((body) => {
    return JSON.parse(body);
  });

  const employeesSearchIndexPromise = fs.readFile('./public/data/employeesSearchIndex.json').then((body) => {
    return JSON.parse(body);
  });

  try {
    searchPromise = Promise.all([termDumpPromise, searchIndexPromise, employeeMapPromise, employeesSearchIndexPromise]).then((...args) => {
      return search.create(...args[0]);
    });
  }
  catch (e) {
    console.error("Error:", e)
    console.error('Not starting search backend.')
    return null;
  }

  return searchPromise;
}

app.get('/search', wrap(async (req, res) => {
  if (!req.query.query || typeof req.query.query !== 'string' || req.query.query.length > 100) {
    console.error('Need query.');
    res.send('Need query param.');
    return;
  }

  let minIndex = 0;
  if (req.query.minIndex) {
    minIndex = req.query.minIndex;
  }

  let maxIndex = 10;
  if (req.query.maxIndex) {
    maxIndex = req.query.maxIndex;
  }


  const index = await getSearch();
<commit_msg>load search as soon as backend starts<commit_after>import path from 'path';
import express from 'express';
import webpack from 'webpack';
import webpackMiddleware from 'webpack-dev-middleware';
import webpackHotMiddleware from 'webpack-hot-middleware';
import wrap from 'express-async-wrap';
import fs from 'fs-promise';

import search from '../common/search';
import webpackConfig from './webpack.config.babel';
import macros from '../common/macros';

const app = express();


// Http to https redirect. 
app.use(function (req, res, next) {
  
  var remoteIp = req.connection.remoteAddress;


  // If this is https request, done. 
  if (req.protocol === 'https') {
    next()
  }

  // If we are behind a cloudflare proxy and cloudflare served a https response, done. 
  else if (req.headers['x-forwarded-proto'] && req.headers['x-forwarded-proto'] === 'https') {
    next()
  }

  // This is development mode
  else if (macros.DEV) {
    next()
  }

  // This is prod and still on http, redirect to https. 
  else {
    // Cache the http to https redirect for 2 months. 
    res.setHeader('Cache-Control', 'public, max-age=5256000');
    console.log('redirecting to https')
    res.redirect('https://' + req.get('host') + req.originalUrl);
  }
})



let searchPromise = null;

async function getSearch() {
  if (searchPromise) {
    return searchPromise;
  }

  const termDumpPromise = fs.readFile('./public/data/getTermDump/neu.edu/201810.json').then((body) => {
    return JSON.parse(body);
  });

  const searchIndexPromise = fs.readFile('./public/data/getSearchIndex/neu.edu/201810.json').then((body) => {
    return JSON.parse(body);
  });


  const employeeMapPromise = fs.readFile('./public/data/employeeMap.json').then((body) => {
    return JSON.parse(body);
  });

  const employeesSearchIndexPromise = fs.readFile('./public/data/employeesSearchIndex.json').then((body) => {
    return JSON.parse(body);
  });

  try {
    searchPromise = Promise.all([termDumpPromise, searchIndexPromise, employeeMapPromise, employeesSearchIndexPromise]).then((...args) => {
      return search.create(...args[0]);
    });
  }
  catch (e) {
    console.error("Error:", e)
    console.error('Not starting search backend.')
    return null;
  }

  return searchPromise;
}

// Load the index as soon as the app starts. 
getSearch();

app.get('/search', wrap(async (req, res) => {
  if (!req.query.query || typeof req.query.query !== 'string' || req.query.query.length > 100) {
    console.error('Need query.');
    res.send('Need query param.');
    return;
  }

  let minIndex = 0;
  if (req.query.minIndex) {
    minIndex = req.query.minIndex;
  }

  let maxIndex = 10;
  if (req.query.maxIndex) {
    maxIndex = req.query.maxIndex;
  }


  const index = await getSearch();

{'fraction_non_alphanumeric': 0.15474880821415474, 'fraction_numerical': 0.009900990099009901, 'mean_word_length': 3.4385679414157853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499596', 'n_tokens_mistral': 1789, 'n_tokens_neox': 1700, 'n_words': 428}