<commit_before>var fs = require('fs');
var qs = require('querystring');
var url = require('url');

var request = require('request');
var b = require('equalitie-bundler');

var _t = require('./translations');

// The path on the RR to report a bundle that has been completed after the
// request *from* the RR timed out.
const RR_COMPLETE = '/complete';

// The header used to signify that the request for http://site.com was rewritten
// from one for https://site.com.  Note that Nodejs' http.IncomingMessage.headers
// object stores headers with all lowercase keys.
const REWRITTEN_HEADER = 'x-ceno-rewritten';

function bs_log(msg) {
  console.log('[BUNDLE SERVER] ' + msg);
}

// POST to the RR to prompt it to accept a completed bundle.
function reportCompleteBundle(config, data, wasRewritten, cb) {
  var headers = {};
  headers[REWRITTEN_HEADER] = wasRewritten;
  request({
    url: config.requestReceiver + RR_COMPLETE,
    method: 'POST',
    json: data,
    headers: headers
  }, cb);
}

module.exports = function (config) {
  return function (req, res) {
  ////////////////////////////
  // Handle bundle requests //
  ////////////////////////////
  var requestedURL = qs.parse(url.parse(req.url).query).url;
  requestedURL = (new Buffer(requestedURL, 'base64')).toString();
  bs_log(_t.__('Got request to bundle %s', requestedURL));
  var rewrittenHeaderValue = req.headers[REWRITTEN_HEADER];
  var wasRewritten = typeof(rewrittenHeaderValue ) !== 'undefined' && rewrittenHeaderValue === 'true';
  bs_log(_t.__('Request was rewritten: %s', wasRewritten.toString()));
  var disconnected = false; // A flag set when the request is closed.

  // Rewrite requests for http://site.com back to https://site.com
  if (wasRewritten) {
    var index = requestedURL.indexOf('http://');
    if (index !== 0) {
      requestedURL = 'https://' + requestedURL;
    } else {
      requestedURL = requestedURL.replace('http://', 'https://');
    }
  }

  req.on('close', function () {
    disconnected = true;
    bs_log(_t.__('Request ended prematurely'));
  });

  // Bundle as many resources in requested pages as possible, without any
  // request manipulation
  var bundler = new b.Bundler(requestedURL);

  if (config.useProxy) {
    bundler.on('originalRequest', b.proxyTo(config.proxyAddress));
    bundler.on('resourceRequest', b.proxyTo(config.proxyAddress));
  }

  bundler.on('originalReceived', b.replaceImages);
  bundler.on('originalReceived', b.replaceCSSFiles);
  bundler.on('originalReceived', b.replaceJSFiles);
  bundler.on('originalReceived', b.replaceURLCalls);
  bundler.on('resourceReceived', b.bundleCSSRecursively);

  res.writeHead(200, {'Content-Type': 'application/json'});

  bundler.bundle(function (err, bundle) {
    if (err) {
      if (!disconnected) {
        res.statusCode = 500;
        res.write(JSON.stringify({
          error: err.message
        }));
        res.end();
      }
      bs_log(_t.__('Encountered error creating bundle for %s', requestedURL));
      bs_log(_t.__('Error: %s', err.message));
    } else { // !err
      var data = {
        created: new Date(),
        url: requestedURL,
        bundle: bundle
      };
      bs_log(_t.__('Successfully created bundle for %s', requestedURL));
      if (!disconnected) {
        res.write(JSON.stringify(data));
        res.end();
        bs_log(_t.__('Sent bundle to RR'));
      } else { // disconnected
        // If the connection to the RR is closed before we can send the bundle,
        // rather than letting the bundling effort go to waste, we will POST to
        // the RR to prompt it to accept the now complete bundle.
        bs_log(_t.__('Reporting bundle completion to RR'));
        reportCompleteBundle(config, data, wasRewritten, function () {
          res.end();
          bs_log(_t.__('Sent POST to RR to prompt it to accept bundle'));
        });
      }
    }
  });
}; // End HTTP request handler
}; // End module-level closure over `config`<commit_msg>Construct the bundler in such a way that it will only produce text of articles to the RSS reader<commit_after>var fs = require('fs');
var qs = require('querystring');
var url = require('url');

var request = require('request');
var b = require('equalitie-bundler');

var _t = require('./translations');

// The path on the RR to report a bundle that has been completed after the
// request *from* the RR timed out.
const RR_COMPLETE = '/complete';

// The header used to signify that the request for http://site.com was rewritten
// from one for https://site.com.  Note that Nodejs' http.IncomingMessage.headers
// object stores headers with all lowercase keys.
const REWRITTEN_HEADER = 'x-ceno-rewritten';

// The header set by the RSS reader to identify itself. When this header is set
// to the value 'true', we should strip out images, css, and js from the bundle.
const RSS_READER_HEADER = 'x-rss-reader';

function bs_log(msg) {
  console.log('[BUNDLE SERVER] ' + msg);
}

// POST to the RR to prompt it to accept a completed bundle.
function reportCompleteBundle(config, data, wasRewritten, cb) {
  var headers = {};
  headers[REWRITTEN_HEADER] = wasRewritten;
  request({
    url: config.requestReceiver + RR_COMPLETE,
    method: 'POST',
    json: data,
    headers: headers
  }, cb);
}

/**
 * Construct the URL that the client has requested for bundling.
 * @param {string} reqUrl - The value of `req.url`
 * @param {bool} wasRewritten - Whether the request was rewritten from HTTPS to HTTP
 * @return {string} the exact url the bundler should request
 */
function constructRequestUrl(reqUrl, wasRewritten) {
  var url = qs.parse(url.parse(reqUrl).query).url;
  url = (new Buffer(url, 'base64')).toString();
  url = wasRewritten ? url.replace('http://', 'https://') : url;
  return url;
}

/**
 * Determine if the URL being requested was rewritten from HTTPS to HTTP
 * @param {IncomingMessage} request - The client's request
 * @return {bool} True if the rewritten header is set to 'true' else false
 */
function requestWasRewritten(request) {
  var headerValue = request.headers[REWRITTEN_HEADER];
  return typeof headerValue !== 'undefined' && headerValue === 'true';
}


/**
 * Determine if the request came from the RSS Reader.
 * @param {IncomingMessage} request - The client's request
 * @return {bool} True if the request has the X-Rss-Reader header set to 'true', else false
 */
function requestFromReader(request) {
  var headerValue = request.headers[RSS_READER_HEADER];
  return typeof headerValue !== 'undefined' && headerValue === 'true';
}

/**
 * Create a bundler object that will use configured proxy settings to fetch a given URL and
 * bundle the contents. If the request comes from the RSS reader, it will produce only the
 * text of the retrieved document.
 * @param {string} url - The URL to fetch
 * @param {object} config - The configuration options for the bundle server
 * @param {bool} reqfromReader - Whether the request comes from the RSS reader or not
 * @return {Bundler} A bundler object that will fetch the requested resource
 */
function makeBundler(url, config, reqFromReader) {
  var bundler = new b.Bundler(url);
  if (config.useProxy) {
    bundler.on('originalRequest', b.proxyTo(config.proxyAddress));
    bundler.on('resourceRequest', b.proxyTo(config.proxyAddress));
  }
  if (reqFromReader) {
    bundler.on('originalReceived', b.replaceLinks(function () {
      return '';
    }));
    bundler.on('resourceReceived', function (reqFn, options, body, diffs, response, callback) {
      diffs[options.url] = '';
      callback(null, diffs);
    });
  } else {
    bundler.on('originalReceived', b.replaceImages);
    bundler.on('originalReceived', b.replaceCSSFiles);
    bundler.on('originalReceived', b.replaceJSFiles);
    bundler.on('originalReceived', b.replaceURLCalls);
    bundler.on('resourceReceived', b.bundleCSSRecursively);
  }
  return bundler;
}

function handler(config) {
  return function (req, res) {
    var disconnected = false; // A flag set when the request is closed.
    var requestedUrl = constructRequestUrl(req.url, requestWasRewritten(req));
  
    req.on('close', function () {
      disconnected = true;
      bs_log(_t.__('Request ended prematurely'));
    });
    res.writeHead(200, {'Content-Type': 'application/json'});
  
    var bundler = makeBundler(requestedUrl, config, reqFromReader(req));
    bundler.bundle(function (err, bundle) {
      if (err) {
        if (!disconnected) {
          res.statusCode = 500;
          res.write(JSON.stringify({
            error: err.message
          }));
          res.end();
        }
        bs_log(_t.__('Encountered error creating bundle for %s', requestedURL));
        bs_log(_t.__('Error: %s', err.message));
      } else { // !err
        var data = {
          created: new Date(),
          url: requestedURL,
          bundle: bundle
        };
        bs_log(_t.__('Successfully created bundle for %s', requestedURL));
        if (!disconnected) {
          res.write(JSON.stringify(data));
          res.end();
          bs_log(_t.__('Sent bundle to RR'));
        } else { // disconnected
          bs_log(_t.__('Reporting bundle completion to RR'));
          reportCompleteBundle(config, data, wasRewritten, function () {
            res.end();
            bs_log(_t.__('Sent POST to RR to prompt it to accept bundle'));
          });
        }
      }
    });
  };
}

module.exports = handler;
{'fraction_non_alphanumeric': 0.13459293068676362, 'fraction_numerical': 0.001804479354633266, 'mean_word_length': 3.198752228163993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4789060', 'n_tokens_mistral': 2845, 'n_tokens_neox': 2727, 'n_words': 865}