<commit_before>const express = require('express');
const puppeteer = require('puppeteer'); // eslint-disable-line
const validate = require('express-validation');
const Joi = require('joi');
const Raven = require('raven');
const bodyParser = require('body-parser');
const cors = require('cors');
const httpCommon = require('_http_common');
const _ = require('lodash');

const validation = {
  screenshot: {
    body: {
      format: Joi.string().valid(['png', 'pdf']),
      target: Joi.string()
        .uri()
        .required(),
      title: Joi.string(),
      selector: Joi.string().required(),
    },
  },
};

if (process.env.SENTRY_DSN) {
  Raven.config(process.env.SENTRY_DSN).install();
}

const captureException = (error, runId = '') => {
  console.error(`Run ID: ${runId} -`, error);
  if (process.env.SENTRY_DSN) Raven.captureException(error);
  throw error;
};

const setContext = (contextData, callback) => {
  if (process.env.SENTRY_DSN) {
    Raven.context(() => {
      Raven.setContext(contextData);
      callback();
    });
  } else {
    callback();
  }
};

const app = express();
let browser;

app.use(bodyParser.json());
app.use(cors());

(async () => {
  try {
    browser = await puppeteer.launch({
      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
    });
    console.log('Exporter started...');
    app.listen(process.env.PORT || 3001, '0.0.0.0');
  } catch (err) {
    captureException(err);
  }
})();


function adaptTitle(title) {
  let r = '';
  [...title].forEach((c) => {
    r += (httpCommon._checkInvalidHeaderChar(c) ? '' : c); // eslint-disable-line
  });
  return r;
}

const takeScreenshot = (req, runId) => new Promise((resolve, reject) => {
  const {
    target, format, title, selector, clip,
  } = req.body;
  console.log('Starting run: ', runId, ' - ', target);
  setContext({ target, format, title }, async () => {
    try {
      // Create a new incognito browser context.
      const context = await browser.createIncognitoBrowserContext();
      const page = await context.newPage();
      page.setDefaultNavigationTimeout(100000);

      page.on('pageerror', e => captureException(e, runId));
      page.on('error', e => captureException(e, runId));

      const token = req.header('access_token');
      const locale = req.header('locale');
      const dest = `${target}?access_token=${token}&locale=${locale}`;
      await page.goto(dest, { waitUntil: 'networkidle2', timeout: 0 });

      const selectors = (selector || '').split(',');
      if (selectors.length) {
        await Promise.all(selectors.map(async (s) => {
          try {
            await page.waitFor(s);
          } catch (error) {
            captureException(error, runId);
          }
        }));
      } else {
        await page.waitFor(5000);
      }

      let screenshot;
      const screenshotOptions = {
        encoding: 'base64',
        format: 'A4',
        omitBackground: false,
      };

      switch (format) {
        case 'png': {
          if (clip === undefined) {
            screenshotOptions.fullPage = true;
          } else {
            screenshotOptions.clip = clip;
          }
          screenshot = await page.screenshot(screenshotOptions);
          resolve(screenshot);
          break;
        }
        case 'pdf': {
          screenshot = await page.pdf({
            format: 'A4',
            printBackground: true,
          });
          const data = screenshot.toString('base64');
          resolve(data);
          break;
        }
        // no default
      }

      await page.close();
      await context.close();
    } catch (err) {
      captureException(err, runId);
      throw err;
    }
  });
});

const MAX_RETRIES = 2;

const sendScreenshotResponse = ({
  res, format, data, title,
}) => {
  switch (format) {
    case 'png': {
      res.setHeader('Content-Length', data.length);
      res.setHeader('Content-Type', 'image/png');
      res.setHeader(
        'Content-Disposition',
        `attachment; filename=${adaptTitle(title)}.png`
      );
      res.send(data);
      break;
    }
    case 'pdf': {
      res.setHeader('Content-Length', data.length);
      res.setHeader('Content-Type', 'application/pdf');
      res.setHeader(<commit_msg>Debugging exporter-service http call  [#1795]<commit_after>const express = require('express');
const puppeteer = require('puppeteer'); // eslint-disable-line
const validate = require('express-validation');
const Joi = require('joi');
const Raven = require('raven');
const bodyParser = require('body-parser');
const cors = require('cors');
const httpCommon = require('_http_common');
const _ = require('lodash');

const validation = {
  screenshot: {
    body: {
      format: Joi.string().valid(['png', 'pdf']),
      target: Joi.string()
        .uri()
        .required(),
      title: Joi.string(),
      selector: Joi.string().required(),
    },
  },
};

if (process.env.SENTRY_DSN) {
  Raven.config(process.env.SENTRY_DSN).install();
}

const captureException = (error, runId = '') => {
  console.error(`Run ID: ${runId} -`, error);
  if (process.env.SENTRY_DSN) Raven.captureException(error);
  throw error;
};

const setContext = (contextData, callback) => {
  if (process.env.SENTRY_DSN) {
    Raven.context(() => {
      Raven.setContext(contextData);
      callback();
    });
  } else {
    callback();
  }
};

const app = express();
let browser;

app.use(bodyParser.json());
app.use(cors());

(async () => {
  try {
    browser = await puppeteer.launch({
      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
    });
    console.log('Exporter started...');
    app.listen(process.env.PORT || 3001, '0.0.0.0');
  } catch (err) {
    captureException(err);
  }
})();


function adaptTitle(title) {
  let r = '';
  [...title].forEach((c) => {
    r += (httpCommon._checkInvalidHeaderChar(c) ? '' : c); // eslint-disable-line
  });
  return r;
}

const takeScreenshot = (req, runId) => new Promise((resolve, reject) => {
  const {
    target, format, title, selector, clip,
  } = req.body;
  console.log('Starting run: ', runId, ' - ', target);
  setContext({ target, format, title }, async () => {
    try {
      // Create a new incognito browser context.
      const context = await browser.createIncognitoBrowserContext();
      const page = await context.newPage();
      page.setDefaultNavigationTimeout(100000);

      page.on('pageerror', e => captureException(e, runId));
      page.on('error', e => captureException(e, runId));

      const token = req.header('access_token');
      const locale = req.header('locale');
      const dest = `${target}?access_token=${token}&locale=${locale}`;
      await page.goto(dest, { waitUntil: 'networkidle2', timeout: 0 });

      const selectors = (selector || '').split(',');
      if (selectors.length) {
        await Promise.all(selectors.map(async (s) => {
          try {
            await page.waitFor(s);
          } catch (error) {
            captureException(error, runId);
          }
        }));
      } else {
        await page.waitFor(5000);
      }

      let screenshot;
      const screenshotOptions = {
        encoding: 'base64',
        format: 'A4',
        omitBackground: false,
      };

      switch (format) {
        case 'png': {
          if (clip === undefined) {
            screenshotOptions.fullPage = true;
          } else {
            screenshotOptions.clip = clip;
          }
          screenshot = await page.screenshot(screenshotOptions);
          resolve(screenshot);
          break;
        }
        case 'pdf': {
          screenshot = await page.pdf({
            format: 'A4',
            printBackground: true,
          });
          const data = screenshot.toString('base64');
          resolve(data);
          break;
        }
        // no default
      }

      await page.close();
      await context.close();
    } catch (err) {
      captureException(err, runId);
      reject(err);
    }
  });
});

const MAX_RETRIES = 2;

const sendScreenshotResponse = ({
  res, format, data, title,
}) => {
  switch (format) {
    case 'png': {
      res.setHeader('Content-Length', data.length);
      res.setHeader('Content-Type', 'image/png');
      res.setHeader(
        'Content-Disposition',
        `attachment; filename=${adaptTitle(title)}.png`
      );
      res.send(data);
      break;
    }
    case 'pdf': {
      res.setHeader('Content-Length', data.length);
      res.setHeader('Content-Type', 'application/pdf');
      res.setHeader(
{'fraction_non_alphanumeric': 0.16666666666666666, 'fraction_numerical': 0.006788389513108614, 'mean_word_length': 2.502049180327869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4465790', 'n_tokens_mistral': 2696, 'n_tokens_neox': 2648, 'n_words': 538}