<commit_before>import * as fs from 'fs';
import * as http from 'http';
import * as path from 'path';
import * as express from 'express';
import { NextFunction, Request, Response } from 'express';
import Model from './lib/model';
import API from './lib/api';
import Logger from './lib/logger';
import {
  isLeaderServer,
  getElapsedSeconds,
  ClientError,
  APIError,
} from './lib/utility';
import { importSentences } from './lib/model/db/import-sentences';
import { getConfig } from './config-helper';
import authRouter from './auth-router';
import { router as adminRouter } from './admin';
import fetchLegalDocument from './fetch-legal-document';

const FULL_CLIENT_PATH = path.join(__dirname, '../web');

const CSP_HEADER = [
  `default-src 'none'`,
  `style-src 'self' 'unsafe-inline' https://fonts.googleapis.com`,
  `img-src 'self' www.google-analytics.com`,
  `media-src data: blob: https://*.amazonaws.com https://*.amazon.com`,
  // Note: we allow unsafe-eval locally for certain webpack functionality.
  `script-src 'self' 'unsafe-eval' 'sha256-it/hVbE0ffRQjkt+hTb6/JM7wKrTSMEK4CHF4s42Zu8=' https://www.google-analytics.com/analytics.js https://pontoon.mozilla.org/pontoon.js`,
  `font-src 'self' https://fonts.gstatic.com`,
  `connect-src 'self' https://pontoon.mozilla.org/graphql`,
].join(';');

export default class Server {
  app: express.Application;
  server: http.Server;
  model: Model;
  api: API;
  logger: Logger;
  isLeader: boolean;

  constructor(options?: { bundleCrossLocaleMessages: boolean }) {
    options = { bundleCrossLocaleMessages: true, ...options };
    this.model = new Model();
    this.api = new API(this.model);
    this.logger = new Logger();
    this.isLeader = null;

    // Make console.log output json.
    if (getConfig().PROD) {
      this.logger.overrideConsole();
    }

    const app = (this.app = express());

    app.use((request, response, next) => {
      // redirect to omit trailing slashes
      if (request.path.substr(-1) == '/' && request.path.length > 1) {
        const query = request.url.slice(request.path.length);
        response.redirect(301, request.path.slice(0, -1) + query);
      } else {
        next();
      }
    });

    app.use(authRouter);
    app.use('/api/v1', this.api.getRouter());
    app.use(adminRouter);

    const staticOptions = {
      setHeaders: (response: express.Response) => {
        // Only use CSP locally. In production, Apache handles CSP headers.
        // See path: nubis/puppet/web.pp
        !getConfig().PROD &&
          response.set('Content-Security-Policy', CSP_HEADER);
      },
    };

    app.use(express.static(FULL_CLIENT_PATH, staticOptions));

    app.use(
      '/contribute.json',
      express.static(path.join(__dirname, '..', 'contribute.json'))
    );

    if (options.bundleCrossLocaleMessages) {
      this.setupCrossLocaleRoute();
    }

    this.setupPrivacyAndTermsRoutes();

    app.use(
      /(.*)/,
      express.static(FULL_CLIENT_PATH + '/index.html', staticOptions)
    );

    app.use(
      (
        error: Error,
        request: Request,
        response: Response,
        next: NextFunction
      ) => {
        console.log(error.message, error.stack);
        const isAPIError = error instanceof APIError;
        if (!isAPIError) {
          console.error(request.url, error.message, error.stack);
        }
        response
          .status(error instanceof ClientError ? 400 : 500)
          .json({ message: isAPIError ? error.message : '' });
      }
    );
  }

  private setupCrossLocaleRoute() {
    const localesPath = path.join(FULL_CLIENT_PATH, 'locales');
    const crossLocaleMessages = fs
      .readdirSync(localesPath)
      .reduce((obj: any, locale: string) => {
        const filePath = path.join(localesPath, locale, 'cross-locale.ftl');
        if (fs.existsSync(filePath)) {
          obj[locale] = fs.readFileSync(filePath, 'utf-8');
        }
        return obj;
      }, {});

    this.app.get('/cross-locale-messages.json', (request, response) => {
      response.json(crossLocaleMessages);
    });
  }

  private setupPrivacyAndTermsRoutes() {
    this.app.get(
      '/privacy/:locale.html',
      async ({ params: { locale } }, response) => {
        response.send(await fetchLegalDocument('Privacy_Notice', locale));
      }
    );
    this.app.get(
      '/terms/:locale.html',
      async ({ params: { locale } }, response) => {
        response.send(await fetchLegalDocument('Terms', locale));
      }
    );
  }

  /**
   * Log application level messages in a common format.
   */
  private print(...args: any[]) {
    args.unshift('APPLICATION --');
    console.log.apply(console, args);
  }

  /**
   * Check if we are the chosen leader server (for performance maintenance).
   */
  private async checkLeader(): Promise<boolean> {
    if (this.isLeader !== null) {
      return this.isLeader;
    }

    try {
      const config = getConfig();
      console.log('config', JSON.stringify(config));
      this.isLeader = await isLeaderServer(
        config.ENVIRONMENT,
        config.RELEASE_VERSION
      );<commit_msg>throw out log<commit_after>import * as fs from 'fs';
import * as http from 'http';
import * as path from 'path';
import * as express from 'express';
import { NextFunction, Request, Response } from 'express';
import Model from './lib/model';
import API from './lib/api';
import Logger from './lib/logger';
import {
  isLeaderServer,
  getElapsedSeconds,
  ClientError,
  APIError,
} from './lib/utility';
import { importSentences } from './lib/model/db/import-sentences';
import { getConfig } from './config-helper';
import authRouter from './auth-router';
import { router as adminRouter } from './admin';
import fetchLegalDocument from './fetch-legal-document';

const FULL_CLIENT_PATH = path.join(__dirname, '../web');

const CSP_HEADER = [
  `default-src 'none'`,
  `style-src 'self' 'unsafe-inline' https://fonts.googleapis.com`,
  `img-src 'self' www.google-analytics.com`,
  `media-src data: blob: https://*.amazonaws.com https://*.amazon.com`,
  // Note: we allow unsafe-eval locally for certain webpack functionality.
  `script-src 'self' 'unsafe-eval' 'sha256-it/hVbE0ffRQjkt+hTb6/JM7wKrTSMEK4CHF4s42Zu8=' https://www.google-analytics.com/analytics.js https://pontoon.mozilla.org/pontoon.js`,
  `font-src 'self' https://fonts.gstatic.com`,
  `connect-src 'self' https://pontoon.mozilla.org/graphql`,
].join(';');

export default class Server {
  app: express.Application;
  server: http.Server;
  model: Model;
  api: API;
  logger: Logger;
  isLeader: boolean;

  constructor(options?: { bundleCrossLocaleMessages: boolean }) {
    options = { bundleCrossLocaleMessages: true, ...options };
    this.model = new Model();
    this.api = new API(this.model);
    this.logger = new Logger();
    this.isLeader = null;

    // Make console.log output json.
    if (getConfig().PROD) {
      this.logger.overrideConsole();
    }

    const app = (this.app = express());

    app.use((request, response, next) => {
      // redirect to omit trailing slashes
      if (request.path.substr(-1) == '/' && request.path.length > 1) {
        const query = request.url.slice(request.path.length);
        response.redirect(301, request.path.slice(0, -1) + query);
      } else {
        next();
      }
    });

    app.use(authRouter);
    app.use('/api/v1', this.api.getRouter());
    app.use(adminRouter);

    const staticOptions = {
      setHeaders: (response: express.Response) => {
        // Only use CSP locally. In production, Apache handles CSP headers.
        // See path: nubis/puppet/web.pp
        !getConfig().PROD &&
          response.set('Content-Security-Policy', CSP_HEADER);
      },
    };

    app.use(express.static(FULL_CLIENT_PATH, staticOptions));

    app.use(
      '/contribute.json',
      express.static(path.join(__dirname, '..', 'contribute.json'))
    );

    if (options.bundleCrossLocaleMessages) {
      this.setupCrossLocaleRoute();
    }

    this.setupPrivacyAndTermsRoutes();

    app.use(
      /(.*)/,
      express.static(FULL_CLIENT_PATH + '/index.html', staticOptions)
    );

    app.use(
      (
        error: Error,
        request: Request,
        response: Response,
        next: NextFunction
      ) => {
        console.log(error.message, error.stack);
        const isAPIError = error instanceof APIError;
        if (!isAPIError) {
          console.error(request.url, error.message, error.stack);
        }
        response
          .status(error instanceof ClientError ? 400 : 500)
          .json({ message: isAPIError ? error.message : '' });
      }
    );
  }

  private setupCrossLocaleRoute() {
    const localesPath = path.join(FULL_CLIENT_PATH, 'locales');
    const crossLocaleMessages = fs
      .readdirSync(localesPath)
      .reduce((obj: any, locale: string) => {
        const filePath = path.join(localesPath, locale, 'cross-locale.ftl');
        if (fs.existsSync(filePath)) {
          obj[locale] = fs.readFileSync(filePath, 'utf-8');
        }
        return obj;
      }, {});

    this.app.get('/cross-locale-messages.json', (request, response) => {
      response.json(crossLocaleMessages);
    });
  }

  private setupPrivacyAndTermsRoutes() {
    this.app.get(
      '/privacy/:locale.html',
      async ({ params: { locale } }, response) => {
        response.send(await fetchLegalDocument('Privacy_Notice', locale));
      }
    );
    this.app.get(
      '/terms/:locale.html',
      async ({ params: { locale } }, response) => {
        response.send(await fetchLegalDocument('Terms', locale));
      }
    );
  }

  /**
   * Log application level messages in a common format.
   */
  private print(...args: any[]) {
    args.unshift('APPLICATION --');
    console.log.apply(console, args);
  }

  /**
   * Check if we are the chosen leader server (for performance maintenance).
   */
  private async checkLeader(): Promise<boolean> {
    if (this.isLeader !== null) {
      return this.isLeader;
    }

    try {
      const config = getConfig();
      this.isLeader = await isLeaderServer(
        config.ENVIRONMENT,
        config.RELEASE_VERSION
      );
{'fraction_non_alphanumeric': 0.15128533438392594, 'fraction_numerical': 0.005121638924455826, 'mean_word_length': 2.7593483894853756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 14, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2056942', 'n_tokens_mistral': 3215, 'n_tokens_neox': 3136, 'n_words': 680}