<commit_before>const config = require("./config").getProperties();
require("./logging").installConsoleHandler();
const mozlog = require("./logging").mozlog("server");
const accepts = require("accepts");
const path = require('path');
const { readFileSync, existsSync } = require('fs');
const Cookies = require("cookies");

const { Shot } = require("./servershot");
const {
  checkLogin,
  registerLogin,
  setState,
  checkState,
  tradeCode,
  getAccountId,
  registerAccount,
  fetchProfileData,
  saveProfileData,
  disconnectDevice,
  retrieveAccount
} = require("./users");
const dbschema = require("./dbschema");
const express = require("express");
const bodyParser = require('body-parser');
const contentDisposition = require("content-disposition");
const csrf = require("csurf");
const morgan = require("morgan");
const linker = require("./linker");
const { randomBytes } = require("./helpers");
const errors = require("./errors");
const buildTime = require("./build-time").string;
const ua = require("universal-analytics");
const urlParse = require("url").parse;
const http = require("http");
const https = require("https");
const gaActivation = require("./ga-activation");
const genUuid = require("nodify-uuid");
const AWS = require("aws-sdk");
const statsd = require("./statsd");
const { notFound } = require("./pages/not-found/server");
const { cacheTime, setCache } = require("./caching");
const { captureRavenException, sendRavenMessage,
        addRavenRequestHandler, addRavenErrorHandler } = require("./ravenclient");
const { errorResponse, simpleResponse, jsResponse } = require("./responses");
const selfPackage = require("./package.json");
const { b64EncodeJson, b64DecodeJson } = require("./b64");
const l10n = require("./l10n");

const PROXY_HEADER_WHITELIST = {
  "content-type": true,
  "content-encoding": true,
  "content-length": true,
  "last-modified": true,
  "etag": true,
  "date": true,
  "accept-ranges": true,
  "content-range": true,
  "retry-after": true,
  "via": true
};

if (config.useS3) {
  // Test a PUT to s3 because configuring this requires using the aws web interface
  // If the permissions are not set up correctly, then we want to know that asap
  var s3bucket = new AWS.S3({params: {Bucket: config.s3BucketName}});
  mozlog.info("creating-s3-bucket", {msg: `creating ${config.s3BucketName}`, bucketName: config.s3BucketName});

  // createBucket is a horribly named api; it creates a local object to access
  // an existing bucket
  s3bucket.createBucket(function() {
    var params = {Key: 'test', Body: 'Hello!'};
    s3bucket.upload(params, function(error, data) {
      if (error) {
        mozlog.warn("test-upload-error", {msg: "Error uploading data during test", error, bucketName: config.s3BucketName});
      } else {
        mozlog.info("test-upload-success", {msg: `Successfully uploaded data to ${config.s3BucketName}/test`, bucketName: config.s3BucketName})
      }
    });
  });
}

function initDatabase() {
  let forceDbVersion = config.db.forceDbVersion;
  if (forceDbVersion) {
    let hadError = false;
    // eslint-disable-next-line promise/catch-or-return
    dbschema.forceDbVersion(forceDbVersion).then(() => {
    },
    (e) => {
      hadError = true;
      mozlog.error("database-version-update-error", {msg: "Error forcing database version:", dbVersion: forceDbVersion, error: e});
    }).then(() => {
      mozlog.info("database-version-update-exit", {msg: "Exiting after downgrade"});
      process.exit(hadError ? 2 : 0);
    });
    return;
  }
  let promise;
  if (config.disableControllerTasks) {
    mozlog.info("database-version-update-skipped", {msg: "Note: this server will not perform database initialization"});
    promise = dbschema.createKeygrip();
  } else {
    promise = dbschema.createTables().then(() => {
      return dbschema.createKeygrip();
    }).then(() => {
      return Shot.upgradeSearch();
    });
  }
  promise.catch((e) => {
    mozlog.error("database-version-update-error", {msg: "Error initializing database:", error: e, stack: e.stack});
    captureRavenException(e);
    // Give Raven/etc a chance to work before exit:
    setTimeout(() => {
      process.exit(1);
    }, 500);
  });
}

initDatabase();

const csrfProtection = csrf({cookie: true});

const app = express();

app.set('trust proxy', true);

// Disable x-powered-by header
app.disable("x-powered-by");

const CONTENT_NAME = config.contentOrigin || '';

function addHSTS(req, res) {
  // Note: HSTS will only produce warning on a localhost self-signed cert
  if (req.protocol === "https" && !config.localhostSsl) {
    let time = 24 * 60 * 60 * 1000; // 24 hours
    res.header(
      "Strict-Transport-Security",
      `max-age=${time}`);
  }
}

addRavenRequestHandler(app);

if (config.expectProtocol) {
  if (!/^https?$/.test(config.expectProtocol)) {
    throw new Error(`Error, bad EXPECT_PROTOCOL: ${config.expectProtocol}`);
  }
  app.use((req, res, next) => {
    req.headers["x-forwarded-proto"] = config.expectProtocol;
    next();
  });
}

app.use((req, res, next) => {
  genUuid.generate(genUuid.V_RANDOM, function(err, uuid) {
    if (!err) {
      let dsn = config.sentryPublicDSN;
      if (dsn) {
        dsn = dsn.replace(/^https?:\/\/[^@]*@/, "").replace(/\/.*/, "");
      } else {
        dsn = "";
      }
      req.cspNonce = uuid;
      res.header(
        "Content-Security-Policy",
        `default-src 'self'; img-src 'self' www.google-analytics.com ${CONTENT_NAME} data:; script-src 'self' www.google-analytics.com 'nonce-${uuid}'; style-src 'self' 'unsafe-inline' https://code.cdn.mozilla.net; connect-src 'self' www.google-analytics.com ${dsn}; font-src https://code.cdn.mozilla.net; frame-ancestors 'none'; object-src 'none';`);
      res.header("X-Frame-Options", "DENY");
      res.header("X-Content-Type-Options", "nosniff");
      addHSTS(req, res);
      next();
    } else {
      errorResponse(res, "Error creating nonce:", err);
    }
  });
});

function isApiUrl(url) {
  return url.startsWith("/api") || url === "/event";
}

app.use((req, res, next) => {
  if (isApiUrl(req.url)) {
    // All API requests are CORS-enabled
    res.header("Access-Control-Allow-Origin", "*");
    res.header("Access-Control-Allow-Methods", "POST, GET, PUT, OPTIONS");
    res.header("Access-Control-Allow-Headers", "Cookie, Content-Type, User-Agent");
    if (req.method === "OPTIONS") {
      res.type("text");
      res.send("");
      return;
    }
  }
  next();
});

app.use(bodyParser.urlencoded({extended: false}));
app.use(bodyParser.json({limit: '25mb'}));

app.use("/static", express.static(path.join(__dirname, "static"), {
  index: false,
  maxAge: cacheTime ? cacheTime * 1000 : null
}));

let xpidir = path.join(__dirname, "..", "xpi");
app.use("/xpi", express.static(xpidir, {index: false}));

app.use("/homepage", express.static(path.join(__dirname, "static/homepage"), {
  index: false
}));

app.use(morgan("combined"));

app.use(function(req, res, next) {
  let authHeader = req.headers['x-screenshots-auth'];
  let authInfo = {};
  let cookies = new Cookies(req, res, {keys: dbschema.getKeygrip()});
  if (authHeader) {
    authInfo = decodeAuthHeader(authHeader);
  } else {
    authInfo.deviceId = cookies.get("user", {signed: true});
    authInfo.accountId = cookies.get("accountid", {signed: true});
    let abTests = cookies.get("abtests", {signed: true});
    if (abTests) {
      abTests = b64DecodeJson(abTests);
      authInfo.abTests = abTests;
    }
  }
  if (authInfo.deviceId) {
    req.deviceId = authInfo.deviceId;
    req.userAnalytics = ua(config.gaId, req.deviceId, {strictCidFormat: false});
    if (config.debugGoogleAnalytics) {
      req.userAnalytics = req.userAnalytics.debug();
    }
  }
  if (authInfo.accountId) {
    req.accountId = authInfo.accountId;
  }
  req.cookies = cookies;
  req.cookies._csrf = cookies.get("_csrf"); // csurf expects a property
  req.abTests = authInfo.abTests || {};
  const host = req.headers.host === config.contentOrigin ? config.contentOrigin : config.siteOrigin;
  req.backend = `${req.protocol}://${host}`;
  req.config = config;
  next();
});

function decodeAuthHeader(header) {
  /** Decode a string header in the format {deviceId}:{deviceIdSig};abtests={b64thing}:{sig} */
  // Since it's treated as opaque, we'll use a fragile regex
  let keygrip = dbschema.getKeygrip();
  let match = /^([^:]{1,255}):([^;]{1,255});abTests=([^:]{1,1500}):(.{0,255})$/.exec(header);
  if (!match) {
    // FIXME: log, Sentry error
    return {};
  }
  let deviceId = match[1];
  let deviceIdSig = match[2];
  let abTestsEncoded = match[3];
  let abTestsEncodedSig = match[4];
  if (!(keygrip.verify(deviceId, deviceIdSig) && keygrip.verify(abTestsEncoded, abTestsEncodedSig))) {
    // FIXME: log, Sentry error
    return {};
  }
  let abTests = b64DecodeJson(abTestsEncoded);
  return {deviceId, abTests};
}

app.use(function(req, res, next) {
  req.staticLink = linker.staticLink.bind(null, {
    cdn: req.config.cdn
  });
  let base = `${req.protocol}://${config.contentOrigin}`;
  linker.imageLinkWithHost = linker.imageLink.bind(null, base);
  next();
});

app.use(function(req, res, next) {
  const languages = req.headers['Accept-Language'] ?
    accepts(req.headers['Accept-Language']).languages :
    ['en-US'];
  l10n.init(languages).then(() => {
    req.getText = l10n.getText;
    req.userLocales = l10n.userLangs;
    return l10n.getStrings().then(strings => {
      req.messages = strings;
      next();
    });
  }).catch(err => {
    mozlog.info("l10n-error", {msg: "Error loading FTL files", description: err});
    next();
  });
});


app.param("id", function(req, res, next, id) {
  if (/^[a-zA-Z0-9]{16}$/.test(id)) {
    next();
    return;
  }
  next(new Error("invalid id"));
});
<commit_msg>Correctly get language headers from request (#3152) (#3169)<commit_after>const config = require("./config").getProperties();
require("./logging").installConsoleHandler();
const mozlog = require("./logging").mozlog("server");
const accepts = require("accepts");
const path = require('path');
const { readFileSync, existsSync } = require('fs');
const Cookies = require("cookies");

const { Shot } = require("./servershot");
const {
  checkLogin,
  registerLogin,
  setState,
  checkState,
  tradeCode,
  getAccountId,
  registerAccount,
  fetchProfileData,
  saveProfileData,
  disconnectDevice,
  retrieveAccount
} = require("./users");
const dbschema = require("./dbschema");
const express = require("express");
const bodyParser = require('body-parser');
const contentDisposition = require("content-disposition");
const csrf = require("csurf");
const morgan = require("morgan");
const linker = require("./linker");
const { randomBytes } = require("./helpers");
const errors = require("./errors");
const buildTime = require("./build-time").string;
const ua = require("universal-analytics");
const urlParse = require("url").parse;
const http = require("http");
const https = require("https");
const gaActivation = require("./ga-activation");
const genUuid = require("nodify-uuid");
const AWS = require("aws-sdk");
const statsd = require("./statsd");
const { notFound } = require("./pages/not-found/server");
const { cacheTime, setCache } = require("./caching");
const { captureRavenException, sendRavenMessage,
        addRavenRequestHandler, addRavenErrorHandler } = require("./ravenclient");
const { errorResponse, simpleResponse, jsResponse } = require("./responses");
const selfPackage = require("./package.json");
const { b64EncodeJson, b64DecodeJson } = require("./b64");
const l10n = require("./l10n");

const PROXY_HEADER_WHITELIST = {
  "content-type": true,
  "content-encoding": true,
  "content-length": true,
  "last-modified": true,
  "etag": true,
  "date": true,
  "accept-ranges": true,
  "content-range": true,
  "retry-after": true,
  "via": true
};

if (config.useS3) {
  // Test a PUT to s3 because configuring this requires using the aws web interface
  // If the permissions are not set up correctly, then we want to know that asap
  var s3bucket = new AWS.S3({params: {Bucket: config.s3BucketName}});
  mozlog.info("creating-s3-bucket", {msg: `creating ${config.s3BucketName}`, bucketName: config.s3BucketName});

  // createBucket is a horribly named api; it creates a local object to access
  // an existing bucket
  s3bucket.createBucket(function() {
    var params = {Key: 'test', Body: '<PASSWORD>!'};
    s3bucket.upload(params, function(error, data) {
      if (error) {
        mozlog.warn("test-upload-error", {msg: "Error uploading data during test", error, bucketName: config.s3BucketName});
      } else {
        mozlog.info("test-upload-success", {msg: `Successfully uploaded data to ${config.s3BucketName}/test`, bucketName: config.s3BucketName})
      }
    });
  });
}

function initDatabase() {
  let forceDbVersion = config.db.forceDbVersion;
  if (forceDbVersion) {
    let hadError = false;
    // eslint-disable-next-line promise/catch-or-return
    dbschema.forceDbVersion(forceDbVersion).then(() => {
    },
    (e) => {
      hadError = true;
      mozlog.error("database-version-update-error", {msg: "Error forcing database version:", dbVersion: forceDbVersion, error: e});
    }).then(() => {
      mozlog.info("database-version-update-exit", {msg: "Exiting after downgrade"});
      process.exit(hadError ? 2 : 0);
    });
    return;
  }
  let promise;
  if (config.disableControllerTasks) {
    mozlog.info("database-version-update-skipped", {msg: "Note: this server will not perform database initialization"});
    promise = dbschema.createKeygrip();
  } else {
    promise = dbschema.createTables().then(() => {
      return dbschema.createKeygrip();
    }).then(() => {
      return Shot.upgradeSearch();
    });
  }
  promise.catch((e) => {
    mozlog.error("database-version-update-error", {msg: "Error initializing database:", error: e, stack: e.stack});
    captureRavenException(e);
    // Give Raven/etc a chance to work before exit:
    setTimeout(() => {
      process.exit(1);
    }, 500);
  });
}

initDatabase();

const csrfProtection = csrf({cookie: true});

const app = express();

app.set('trust proxy', true);

// Disable x-powered-by header
app.disable("x-powered-by");

const CONTENT_NAME = config.contentOrigin || '';

function addHSTS(req, res) {
  // Note: HSTS will only produce warning on a localhost self-signed cert
  if (req.protocol === "https" && !config.localhostSsl) {
    let time = 24 * 60 * 60 * 1000; // 24 hours
    res.header(
      "Strict-Transport-Security",
      `max-age=${time}`);
  }
}

addRavenRequestHandler(app);

if (config.expectProtocol) {
  if (!/^https?$/.test(config.expectProtocol)) {
    throw new Error(`Error, bad EXPECT_PROTOCOL: ${config.expectProtocol}`);
  }
  app.use((req, res, next) => {
    req.headers["x-forwarded-proto"] = config.expectProtocol;
    next();
  });
}

app.use((req, res, next) => {
  genUuid.generate(genUuid.V_RANDOM, function(err, uuid) {
    if (!err) {
      let dsn = config.sentryPublicDSN;
      if (dsn) {
        dsn = dsn.replace(/^https?:\/\/[^@]*@/, "").replace(/\/.*/, "");
      } else {
        dsn = "";
      }
      req.cspNonce = uuid;
      res.header(
        "Content-Security-Policy",
        `default-src 'self'; img-src 'self' www.google-analytics.com ${CONTENT_NAME} data:; script-src 'self' www.google-analytics.com 'nonce-${uuid}'; style-src 'self' 'unsafe-inline' https://code.cdn.mozilla.net; connect-src 'self' www.google-analytics.com ${dsn}; font-src https://code.cdn.mozilla.net; frame-ancestors 'none'; object-src 'none';`);
      res.header("X-Frame-Options", "DENY");
      res.header("X-Content-Type-Options", "nosniff");
      addHSTS(req, res);
      next();
    } else {
      errorResponse(res, "Error creating nonce:", err);
    }
  });
});

function isApiUrl(url) {
  return url.startsWith("/api") || url === "/event";
}

app.use((req, res, next) => {
  if (isApiUrl(req.url)) {
    // All API requests are CORS-enabled
    res.header("Access-Control-Allow-Origin", "*");
    res.header("Access-Control-Allow-Methods", "POST, GET, PUT, OPTIONS");
    res.header("Access-Control-Allow-Headers", "Cookie, Content-Type, User-Agent");
    if (req.method === "OPTIONS") {
      res.type("text");
      res.send("");
      return;
    }
  }
  next();
});

app.use(bodyParser.urlencoded({extended: false}));
app.use(bodyParser.json({limit: '25mb'}));

app.use("/static", express.static(path.join(__dirname, "static"), {
  index: false,
  maxAge: cacheTime ? cacheTime * 1000 : null
}));

let xpidir = path.join(__dirname, "..", "xpi");
app.use("/xpi", express.static(xpidir, {index: false}));

app.use("/homepage", express.static(path.join(__dirname, "static/homepage"), {
  index: false
}));

app.use(morgan("combined"));

app.use(function(req, res, next) {
  let authHeader = req.headers['x-screenshots-auth'];
  let authInfo = {};
  let cookies = new Cookies(req, res, {keys: dbschema.getKeygrip()});
  if (authHeader) {
    authInfo = decodeAuthHeader(authHeader);
  } else {
    authInfo.deviceId = cookies.get("user", {signed: true});
    authInfo.accountId = cookies.get("accountid", {signed: true});
    let abTests = cookies.get("abtests", {signed: true});
    if (abTests) {
      abTests = b64DecodeJson(abTests);
      authInfo.abTests = abTests;
    }
  }
  if (authInfo.deviceId) {
    req.deviceId = authInfo.deviceId;
    req.userAnalytics = ua(config.gaId, req.deviceId, {strictCidFormat: false});
    if (config.debugGoogleAnalytics) {
      req.userAnalytics = req.userAnalytics.debug();
    }
  }
  if (authInfo.accountId) {
    req.accountId = authInfo.accountId;
  }
  req.cookies = cookies;
  req.cookies._csrf = cookies.get("_csrf"); // csurf expects a property
  req.abTests = authInfo.abTests || {};
  const host = req.headers.host === config.contentOrigin ? config.contentOrigin : config.siteOrigin;
  req.backend = `${req.protocol}://${host}`;
  req.config = config;
  next();
});

function decodeAuthHeader(header) {
  /** Decode a string header in the format {deviceId}:{deviceIdSig};abtests={b64thing}:{sig} */
  // Since it's treated as opaque, we'll use a fragile regex
  let keygrip = dbschema.getKeygrip();
  let match = /^([^:]{1,255}):([^;]{1,255});abTests=([^:]{1,1500}):(.{0,255})$/.exec(header);
  if (!match) {
    // FIXME: log, Sentry error
    return {};
  }
  let deviceId = match[1];
  let deviceIdSig = match[2];
  let abTestsEncoded = match[3];
  let abTestsEncodedSig = match[4];
  if (!(keygrip.verify(deviceId, deviceIdSig) && keygrip.verify(abTestsEncoded, abTestsEncodedSig))) {
    // FIXME: log, Sentry error
    return {};
  }
  let abTests = b64DecodeJson(abTestsEncoded);
  return {deviceId, abTests};
}

app.use(function(req, res, next) {
  req.staticLink = linker.staticLink.bind(null, {
    cdn: req.config.cdn
  });
  let base = `${req.protocol}://${config.contentOrigin}`;
  linker.imageLinkWithHost = linker.imageLink.bind(null, base);
  next();
});

app.use(function(req, res, next) {
  const languages = accepts(req).languages() || ['en-US'];
  l10n.init(languages).then(() => {
    req.getText = l10n.getText;
    req.userLocales = l10n.userLangs;
    return l10n.getStrings().then(strings => {
      req.messages = strings;
      next();
    });
  }).catch(err => {
    mozlog.info("l10n-error", {msg: "Error loading FTL files", description: err});
    next();
  });
});


app.param("id", function(req, res, next, id) {
  if (/^[a-zA-Z0-9]{16}$/.test(id)) {
    next();
    return;
  }
  next(new Error("invalid id"));
});

{'fraction_non_alphanumeric': 0.17959476054031928, 'fraction_numerical': 0.009414654113794515, 'mean_word_length': 3.8426660059464814, 'pattern_counts': {'":': 20, '<': 4, '<?xml version=': 0, '>': 30, 'https://': 4, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1264659', 'n_tokens_mistral': 6637, 'n_tokens_neox': 6477, 'n_words': 1386}