<commit_before>// Newrelic *must* be the first module loaded. Do not move this require module!
if ( process.env.NEW_RELIC_HOME ) {
  require( 'newrelic' );
}

var express = require('express'),
    fs = require('fs'),
    path = require('path'),
    helmet = require( "helmet" ),
    nunjucks = require('nunjucks'),
    nunjucksEnv = new nunjucks.Environment(new nunjucks.FileSystemLoader( __dirname + '/views' )),
    app = express(),
    lessMiddleware = require( 'less-middleware' ),
    requirejsMiddleware = require( 'requirejs-middleware' ),
    config = require( './lib/config' ),
    Project,
    filter,
    sanitizer = require( './lib/sanitizer' ),
    FileStore = require( './lib/file-store.js' ),
    metrics = require('./lib/metrics.js'),
    utils,
    middleware = require( './lib/middleware' ),
    stores = {},
    APP_HOSTNAME = config.hostname,
    WWW_ROOT =  __dirname + '/public',
    port = config.PORT,
    makeapiConfig = {
      apiURL: config.MAKE_ENDPOINT,
      auth: config.MAKE_USERNAME + ":" + config.MAKE_PASSWORD
    };

nunjucksEnv.express( app );

function setupStore( storeConfig ) {
  var store = FileStore.create( storeConfig.type, storeConfig.options );
  if ( store.requiresFileSystem ) {
    app.use( express.static( store.root, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) );
  }
  return store;
}

if ( config.USE_WEBFAKER ) {
  var webfaker = require( "webfaker" );

  webfaker.start({
    port: port + 1,
    username: config.MAKE_USERNAME,
    password: config.<PASSWORD>,
    isAdminCheck: false
  }, function() {
    console.log( "Started Webfaker services on http://localhost ports: FakeAPI=%s, Fogin=%s, Fubble=%s",
                 port + 1, port + 2, port + 3 );

    process.on( "exit", function() {
      webfaker.stop();
    });
  });
}

app.locals({
  config: {
    audience: config.AUDIENCE,
    ga_account: config.GA_ACCOUNT,
    ga_domain: config.GA_DOMAIN,
    user_bar: config.USER_BAR
  }
});

app.configure( function() {
  var tmpDir = path.normalize( require( "os" ).tmpDir() + "/mozilla.butter/" );

  app.use( express.logger( config.logger ) );
  if ( !!config.FORCE_SSL ) {
    app.use( helmet.hsts() );
    app.enable( "trust proxy" );
  }
  app.use( express.compress() )
    .use( lessMiddleware({
      once: config.OPTIMIZE_CSS,
      dest: tmpDir,
      src: WWW_ROOT,
      compress: config.OPTIMIZE_CSS,
      yuicompress: config.OPTIMIZE_CSS,
      optimization: config.OPTIMIZE_CSS ? 0 : 2
    }))
    .use( requirejsMiddleware({
      src: WWW_ROOT,
      dest: tmpDir,
      once: config.OPTIMIZE_JS,
      modules: {
        "/src/butter.js": {
          include: [ "butter" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js",
          paths: {
            "make-api": path.resolve( __dirname, "node_modules/makeapi/public/js/make-api" ),
            "sso-include": path.resolve( __dirname, "node_modules/webmaker-sso/include" )
          }
        },
        "/src/embed.js": {
          include: [ "embed" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js",
        },
        "/templates/assets/editors/editorhelper.js": {
          include: [ "../templates/assets/editors/editorhelper" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js"
        }
      },
      defaults: {
        baseUrl: WWW_ROOT + "/src/",
        findNestedDependencies: true,
        optimize: "none",
        preserveLicenseComments: false,
        wrap: {
          startFile: __dirname + "/tools/wrap.start",
          endFile: __dirname + "/tools/wrap.end"
        }
      }
    }))
    .use( express.static( tmpDir, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) )
    .use( express.static( WWW_ROOT, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) );

  // File Store types and options come from JSON config file.
  stores.publish = setupStore( config.publishStore );

  app.use( express.bodyParser() )
    .use( express.cookieParser() )
    .use( express.cookieSession( config.session ) )
    .use( express.csrf() )
    /* Show Zeus who's boss
     * This only affects requests under /api and /persona, not static files
     * because the static file writes the response header before we hit this middleware
     */
    .use( function( req, res, next ) {
      res.header( 'Cache-Control', 'no-store' );
      return next();
    })
    .use( app.router )
    .use( function( err, req, res, next) {
      if ( !err.status ) {
        err.status = 500;
      }

      res.status( err.status );
      res.render( 'error.html', { message: err.message, status: err.status });
    })
    .use( function( req, res, next ) {
      res.status( 404 );
      res.render( 'error.html', { message: "This page doesn't exist", status: 404 });
    });

  utils = require( './lib/utils' )({
    EMBED_HOSTNAME: config.EMBED_HOSTNAME ? config.EMBED_HOSTNAME : APP_HOSTNAME,
    EMBED_SUFFIX: '_'
  }, stores );

  Project = require( './lib/project' )( config.database, makeapiConfig, utils );
  filter = require( './lib/filter' )( Project.isDBOnline );
});<commit_msg>bug885486 - add xframe<commit_after>// Newrelic *must* be the first module loaded. Do not move this require module!
if ( process.env.NEW_RELIC_HOME ) {
  require( 'newrelic' );
}

var express = require('express'),
    fs = require('fs'),
    path = require('path'),
    helmet = require( "helmet" ),
    nunjucks = require('nunjucks'),
    nunjucksEnv = new nunjucks.Environment(new nunjucks.FileSystemLoader( __dirname + '/views' )),
    app = express(),
    lessMiddleware = require( 'less-middleware' ),
    requirejsMiddleware = require( 'requirejs-middleware' ),
    config = require( './lib/config' ),
    Project,
    filter,
    sanitizer = require( './lib/sanitizer' ),
    FileStore = require( './lib/file-store.js' ),
    metrics = require('./lib/metrics.js'),
    utils,
    middleware = require( './lib/middleware' ),
    stores = {},
    APP_HOSTNAME = config.hostname,
    WWW_ROOT =  __dirname + '/public',
    port = config.PORT,
    makeapiConfig = {
      apiURL: config.MAKE_ENDPOINT,
      auth: config.MAKE_USERNAME + ":" + config.MAKE_PASSWORD
    };

nunjucksEnv.express( app );

function setupStore( storeConfig ) {
  var store = FileStore.create( storeConfig.type, storeConfig.options );
  if ( store.requiresFileSystem ) {
    app.use( express.static( store.root, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) );
  }
  return store;
}

if ( config.USE_WEBFAKER ) {
  var webfaker = require( "webfaker" );

  webfaker.start({
    port: port + 1,
    username: config.MAKE_USERNAME,
    password: config.<PASSWORD>,
    isAdminCheck: false
  }, function() {
    console.log( "Started Webfaker services on http://localhost ports: FakeAPI=%s, Fogin=%s, Fubble=%s",
                 port + 1, port + 2, port + 3 );

    process.on( "exit", function() {
      webfaker.stop();
    });
  });
}

app.locals({
  config: {
    audience: config.AUDIENCE,
    ga_account: config.GA_ACCOUNT,
    ga_domain: config.GA_DOMAIN,
    user_bar: config.USER_BAR
  }
});

app.configure( function() {
  var tmpDir = path.normalize( require( "os" ).tmpDir() + "/mozilla.butter/" );

  app.use( express.logger( config.logger ) );
  if ( !!config.FORCE_SSL ) {
    app.use( helmet.hsts() );
    app.enable( "trust proxy" );
  }
  app.use( express.compress() )
    .use( lessMiddleware({
      once: config.OPTIMIZE_CSS,
      dest: tmpDir,
      src: WWW_ROOT,
      compress: config.OPTIMIZE_CSS,
      yuicompress: config.OPTIMIZE_CSS,
      optimization: config.OPTIMIZE_CSS ? 0 : 2
    }))
    .use( requirejsMiddleware({
      src: WWW_ROOT,
      dest: tmpDir,
      once: config.OPTIMIZE_JS,
      modules: {
        "/src/butter.js": {
          include: [ "butter" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js",
          paths: {
            "make-api": path.resolve( __dirname, "node_modules/makeapi/public/js/make-api" ),
            "sso-include": path.resolve( __dirname, "node_modules/webmaker-sso/include" )
          }
        },
        "/src/embed.js": {
          include: [ "embed" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js",
        },
        "/templates/assets/editors/editorhelper.js": {
          include: [ "../templates/assets/editors/editorhelper" ],
          mainConfigFile: WWW_ROOT + "/src/popcorn.js"
        }
      },
      defaults: {
        baseUrl: WWW_ROOT + "/src/",
        findNestedDependencies: true,
        optimize: "none",
        preserveLicenseComments: false,
        wrap: {
          startFile: __dirname + "/tools/wrap.start",
          endFile: __dirname + "/tools/wrap.end"
        }
      }
    }))
    .use( express.static( tmpDir, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) )
    .use( express.static( WWW_ROOT, JSON.parse( JSON.stringify( config.staticMiddleware ) ) ) );

  // File Store types and options come from JSON config file.
  stores.publish = setupStore( config.publishStore );

  app.use( express.bodyParser() )
    .use( express.cookieParser() )
    .use( express.cookieSession( config.session ) )
    .use( express.csrf() )
    .use( helmet.xframe() )
    /* Show Zeus who's boss
     * This only affects requests under /api and /persona, not static files
     * because the static file writes the response header before we hit this middleware
     */
    .use( function( req, res, next ) {
      res.header( 'Cache-Control', 'no-store' );
      return next();
    })
    .use( app.router )
    .use( function( err, req, res, next) {
      if ( !err.status ) {
        err.status = 500;
      }

      res.status( err.status );
      res.render( 'error.html', { message: err.message, status: err.status });
    })
    .use( function( req, res, next ) {
      res.status( 404 );
      res.render( 'error.html', { message: "This page doesn't exist", status: 404 });
    });

  utils = require( './lib/utils' )({
    EMBED_HOSTNAME: config.EMBED_HOSTNAME ? config.EMBED_HOSTNAME : APP_HOSTNAME,
    EMBED_SUFFIX: '_'
  }, stores );

  Project = require( './lib/project' )( config.database, makeapiConfig, utils );
  filter = require( './lib/filter' )( Project.isDBOnline );
});
{'fraction_non_alphanumeric': 0.14928746928746928, 'fraction_numerical': 0.0035380835380835383, 'mean_word_length': 2.608510638297872, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5338020', 'n_tokens_mistral': 3382, 'n_tokens_neox': 3313, 'n_words': 722}