<commit_before>var urljoin = require('url-join');
var path = require('path');
var etag = require('etag');
var parseUrl = require('url').parse;
var _ = require('lodash');
var instrument = require('../instrument');
var htmlprep = require('htmlprep');
var sha1 = require('sha1');
var debug = require('debug')('4front:http-host:pipe-response');

var DEFAULT_CLIENT_CONFIG_VAR = '__4front__';

module.exports = function(options) {
  _.defaults(options, {
    htmlprep: true,
    contentType: 'text/html',
    baseUrlPlaceholder: 'https://__baseurl__',
    noVersionAssetPatterns: []
  });

  return function(req, res, next) {
    var versionId = req.ext.virtualAppVersion.versionId;
    var versionName = req.ext.virtualAppVersion.name;

    if (!options.clientConfigVar) {
      options.clientConfigVar = req.app.settings.clientConfigVar;
    }

    // The Content-Type may have already been set by http-headers plugin
    if (!res.getHeader('Content-Type')) {
      res.set('Content-Type', options.contentType);
    }

    res.set('Virtual-App-Version', versionId);
    res.set('Virtual-App-Page', req.ext.webPagePath);

    if (!_.isEmpty(versionName)) {
      res.set('Virtual-App-Version-Name', versionName);
    }

    // The Cache-Control may have already been explicitly
    // set by the http-headers plugin.
    if (!res.getHeader('Cache-Control')) {
      var cacheControl = null;
      if (options.maxAge) {
        cacheControl = 'max-age=' + options.maxAge;
      } else if (options.cacheControl) {
        cacheControl = options.cacheControl;
      } else {
        // TODO: Should this be public, max-age=31536000, no-cache
        cacheControl = 'no-cache';
      }
      res.setHeader('Cache-Control', cacheControl);
    }

    populateClientConfig(req, res, versionId, versionName);

    // Only use etags for 200 OK GET requests
    var doConditionalGet = req.method === 'GET' &&
      (!_.isNumber(res.statusCode) || res.statusCode === 200) &&
      !req.query.__instrument;

    if (doConditionalGet) {
      conditionalGet(req, res, next);
    } else {
      pipeFromStorage(req, res, next);
    }
  };

  function conditionalGet(req, res, next) {
    var etagHeader = generateETag(req, res);
    if (etagHeader) {
      // If the etag matches the if-none-match header,
      // return a 304 Not Modified response.
      if (req.get('if-none-match') === etagHeader) {
        // According to the RFC, the server should still send back the same
        // headers that would appear in a 200 response.
        // https://tools.ietf.org/html/rfc7232#section-4.1
        res.set('Cache-Control', 'no-cache');
        res.set('ETag', etagHeader);
        return res.status(304).end();
      }
      res.setHeader('ETag', etagHeader);
    }

    // TODO: Temporary test to conditionally enable the content cache.
    req.ext.contentCacheEnabled =
      _.isFunction(req.app.settings.enableContentCache) &&
      req.app.settings.enableContentCache(req) === true;

    if (!req.ext.contentCacheEnabled) {
      debug('content cache is not enabled');
      return pipeFromStorage(req, res, next);
    }

    // First try reading the page from the cache keyed off the etag
    debug('check if content exists in cache with key %s', etagHeader);
    req.app.settings.contentCache.exists(etagHeader, function(err, exists) {
      if (err) return next(err);

      if (exists) {
        // TODO: Use a custom CloudWatchMetric to keep track of content cache hit rate.
        debug('webpage exists in cache');
        req.ext.cacheHit = true;
        return req.app.settings.contentCache.readStream(etagHeader).pipe(res);
      }

      debug('webpage does not exist in cache, load from storage');
      req.ext.cacheHit = false;
      pipeFromStorage(req, res, next);
    });
  }

  function pipeFromStorage(req, res) {
    // If req.ext.webPageStream is a function, then evaluate it to get
    // the actual stream. This is used to lazy fetch the stream in case
    // we don't actually need it, like when the etag is a match.
    var stream;
    if (_.isFunction(req.ext.webPageStream)) {
      stream = req.ext.webPageStream();
    } else {
      stream = req.ext.webPageStream;
    }

    if (options.contentType === 'text/html' && options.htmlprep !== false) {
      var htmlOptions = buildHtmlOptions(req, res);
      debug('piping page stream through htmlprep');

      var htmlPrepStream = htmlprep(htmlOptions);
      htmlPrepStream.on('start', function() {
        instrument.start(req, 'htmlprep');
      })
      .on('end', function() {
        instrument.finish(req, 'htmlprep');
      });

      stream = stream.pipe(htmlPrepStream);
    }

    // If there's an etag header, write it to the cache on the way out. The output
    // includes all the transformations made by htmlprep.
    var etagHeader = res.getHeader('etag');
    if (etagHeader && req.ext.contentCacheEnabled === true) {
      debug('writing page to content cache');
      stream = stream.pipe(req.app.settings.contentCache.writeThrough(etagHeader));
    }

    stream.pipe(res);
  }

  function generateETag(req, res) {
    // Don't set an etag if there is a user context or if this is the developer sandbox.
    if (req.ext.user || req.ext.virtualEnv === 'local') return null;

    // The etag is determined by hashing the stringified JSON versionId, and env variables.
    // Taken together they represent a unique hash of the html page response. The reason
    // the env variables are included is that they can change outside of a new version deployment.
    // Include the minimal amount of contextual data to maximize the cache hit rate.
    return etag(JSON.stringify({
      versionId: req.ext.virtualAppVersion.versionId,
      env: req.ext.env,
      webPagePath: req.ext.webPagePath,
      statusCode: res.statusCode || 200,
      buildType: req.ext.buildType || 'release'
    }));
  }

  function populateClientConfig(req, res, versionId, versionName) {
    if (!req.ext.clientConfig) {
      req.ext.clientConfig = {};
    }<commit_msg>Add more content cache logging<commit_after>var urljoin = require('url-join');
var path = require('path');
var etag = require('etag');
var parseUrl = require('url').parse;
var _ = require('lodash');
var instrument = require('../instrument');
var htmlprep = require('htmlprep');
var sha1 = require('sha1');
var debug = require('debug')('4front:http-host:pipe-response');

var DEFAULT_CLIENT_CONFIG_VAR = '__4front__';

module.exports = function(options) {
  _.defaults(options, {
    htmlprep: true,
    contentType: 'text/html',
    baseUrlPlaceholder: 'https://__baseurl__',
    noVersionAssetPatterns: []
  });

  return function(req, res, next) {
    var versionId = req.ext.virtualAppVersion.versionId;
    var versionName = req.ext.virtualAppVersion.name;

    if (!options.clientConfigVar) {
      options.clientConfigVar = req.app.settings.clientConfigVar;
    }

    // The Content-Type may have already been set by http-headers plugin
    if (!res.getHeader('Content-Type')) {
      res.set('Content-Type', options.contentType);
    }

    res.set('Virtual-App-Version', versionId);
    res.set('Virtual-App-Page', req.ext.webPagePath);

    if (!_.isEmpty(versionName)) {
      res.set('Virtual-App-Version-Name', versionName);
    }

    // The Cache-Control may have already been explicitly
    // set by the http-headers plugin.
    if (!res.getHeader('Cache-Control')) {
      var cacheControl = null;
      if (options.maxAge) {
        cacheControl = 'max-age=' + options.maxAge;
      } else if (options.cacheControl) {
        cacheControl = options.cacheControl;
      } else {
        // TODO: Should this be public, max-age=31536000, no-cache
        cacheControl = 'no-cache';
      }
      res.setHeader('Cache-Control', cacheControl);
    }

    populateClientConfig(req, res, versionId, versionName);

    // Only use etags for 200 OK GET requests
    var doConditionalGet = req.method === 'GET' &&
      (!_.isNumber(res.statusCode) || res.statusCode === 200) &&
      !req.query.__instrument;

    if (doConditionalGet) {
      conditionalGet(req, res, next);
    } else {
      pipeFromStorage(req, res, next);
    }
  };

  function conditionalGet(req, res, next) {
    var etagHeader = generateETag(req, res);
    if (etagHeader) {
      // If the etag matches the if-none-match header,
      // return a 304 Not Modified response.
      if (req.get('if-none-match') === etagHeader) {
        // According to the RFC, the server should still send back the same
        // headers that would appear in a 200 response.
        // https://tools.ietf.org/html/rfc7232#section-4.1
        res.set('Cache-Control', 'no-cache');
        res.set('ETag', etagHeader);
        return res.status(304).end();
      }
      res.setHeader('ETag', etagHeader);
    }

    // TODO: Temporary test to conditionally enable the content cache.
    req.ext.contentCacheEnabled =
      _.isFunction(req.app.settings.enableContentCache) &&
      req.app.settings.enableContentCache(req) === true;

    if (!req.ext.contentCacheEnabled) {
      debug('content cache is not enabled');
      return pipeFromStorage(req, res, next);
    }

    // First try reading the page from the cache keyed off the etag
    debug('check if content exists in cache with key %s', etagHeader);
    req.app.settings.contentCache.exists(etagHeader, function(err, exists) {
      if (err) return next(err);

      if (exists) {
        req.app.settings.logger.info('webpage read from cache', {
          type: 'contentCache',
          hit: true,
          key: etagHeader
        });
        req.ext.cacheHit = true;
        return req.app.settings.contentCache.readStream(etagHeader).pipe(res);
      }

      req.app.settings.logger.info('webpage does not exist in cache, load from storage', {
        type: 'contentCache',
        hit: false,
        key: etagHeader
      });
      req.ext.cacheHit = false;
      pipeFromStorage(req, res, next);
    });
  }

  function pipeFromStorage(req, res) {
    // If req.ext.webPageStream is a function, then evaluate it to get
    // the actual stream. This is used to lazy fetch the stream in case
    // we don't actually need it, like when the etag is a match.
    var stream;
    if (_.isFunction(req.ext.webPageStream)) {
      stream = req.ext.webPageStream();
    } else {
      stream = req.ext.webPageStream;
    }

    if (options.contentType === 'text/html' && options.htmlprep !== false) {
      var htmlOptions = buildHtmlOptions(req, res);
      debug('piping page stream through htmlprep');

      var htmlPrepStream = htmlprep(htmlOptions);
      htmlPrepStream.on('start', function() {
        instrument.start(req, 'htmlprep');
      })
      .on('end', function() {
        instrument.finish(req, 'htmlprep');
      });

      stream = stream.pipe(htmlPrepStream);
    }

    // If there's an etag header, write it to the cache on the way out. The output
    // includes all the transformations made by htmlprep.
    var etagHeader = res.getHeader('etag');
    if (etagHeader && req.ext.contentCacheEnabled === true) {
      req.app.settings.logger.info('writing webpage to cache', {
        type: 'contentCache',
        key: etagHeader
      });
      stream = stream.pipe(req.app.settings.contentCache.writeThrough(etagHeader));
    }

    stream.pipe(res);
  }

  function generateETag(req, res) {
    // Don't set an etag if there is a user context or if this is the developer sandbox.
    if (req.ext.user || req.ext.virtualEnv === 'local') return null;

    // The etag is determined by hashing the stringified JSON versionId, and env variables.
    // Taken together they represent a unique hash of the html page response. The reason
    // the env variables are included is that they can change outside of a new version deployment.
    // Include the minimal amount of contextual data to maximize the cache hit rate.
    return etag(JSON.stringify({
      versionId: req.ext.virtualAppVersion.versionId,
      env: req.ext.env,
      webPagePath: req.ext.webPagePath,
      statusCode: res.statusCode || 200,
      buildType: req.ext.buildType || 'release'
    }));
  }

  function populateClientConfig(req, res, versionId, versionName) {
    if (!req.ext.clientConfig) {
      req.ext.clientConfig = {};
    }
{'fraction_non_alphanumeric': 0.1251320386771756, 'fraction_numerical': 0.005850329081010807, 'mean_word_length': 2.86436420722135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2739431', 'n_tokens_mistral': 3756, 'n_tokens_neox': 3605, 'n_words': 1014}