<commit_before>const express = require("express")
const puppeteer = require("puppeteer")
const validate = require("express-validation")
const Joi = require("joi")
const Raven = require("raven")
const bodyParser = require("body-parser")
const cors = require("cors")

const validation = {
  screenshot: {
    body: {
      format: Joi.string().valid(["png", "pdf"]),
      target: Joi.string()
        .uri()
        .required(),
      title: Joi.string(),
      selector: Joi.string().required()
    }
  }
}

if (process.env.SENTRY_DSN) {
  Raven.config(process.env.SENTRY_DSN).install()
}

const captureException = error => {
  console.error(error)
  if (process.env.SENTRY_DSN) Raven.captureException(error)
}

const setContext = (contextData, callback) => {
  if (process.env.SENTRY_DSN) {
    Raven.context(function() {
      Raven.setContext(contextData)
      callback()
    })
  } else {
    callback()
  }
}

const app = express()
let browser

app.use(bodyParser.json())
app.use(cors());

(async () => {
  try {
    browser = await puppeteer.launch({
      args: ["--no-sandbox", "--disable-setuid-sandbox"]
    })
  } catch (err) {
    captureException(err);
  }
  console.log("Exporter started...")
  app.listen(process.env.PORT || 3001, "0.0.0.0")
})()

app.post("/screenshot", validate(validation.screenshot), async (req, res) => {
  const { target, format, title, selector, clip } = req.body

  setContext({ target, format, title }, async () => {
    try {
      // Create a new incognito browser context.
      const context = await browser.createIncognitoBrowserContext()
      const page = await context.newPage()
      page.setDefaultNavigationTimeout(100000);

      page.on("pageerror", captureException)
      page.on("error", e => captureException(e))

      const token = req.header("access_token")
      const locale = req.header("locale")
      const dest = `${target}?access_token=${token}&locale=${locale}`
      await page.goto(dest, { waitUntil: "networkidle2", timeout: 0 })

      const selectors = (selector || "").split(",")
      if (selectors.length) {
        selectors.forEach(async selector => {
          await page.waitFor(selector)
        })
      } else {
        await page.waitFor(5000)
      }

      let screenshot
      let screenshotOptions = {
        encoding: "base64",
        format: "A4",
        omitBackground: false<commit_msg>Increment timeout for exporter service to 100000 (second attempt)<commit_after>const express = require("express")
const puppeteer = require("puppeteer")
const validate = require("express-validation")
const Joi = require("joi")
const Raven = require("raven")
const bodyParser = require("body-parser")
const cors = require("cors")

const validation = {
  screenshot: {
    body: {
      format: Joi.string().valid(["png", "pdf"]),
      target: Joi.string()
        .uri()
        .required(),
      title: Joi.string(),
      selector: Joi.string().required()
    }
  }
}

if (process.env.SENTRY_DSN) {
  Raven.config(process.env.SENTRY_DSN).install()
}

const captureException = error => {
  console.error(error)
  if (process.env.SENTRY_DSN) Raven.captureException(error)
}

const setContext = (contextData, callback) => {
  if (process.env.SENTRY_DSN) {
    Raven.context(function() {
      Raven.setContext(contextData)
      callback()
    })
  } else {
    callback()
  }
}

const app = express()
let browser

app.use(bodyParser.json())
app.use(cors());

(async () => {
  try {
    browser = await puppeteer.launch({
      args: ["--no-sandbox", "--disable-setuid-sandbox"]
    })
  } catch (err) {
    captureException(err);
  }
  console.log("Exporter started...")
  app.listen(process.env.PORT || 3001, "0.0.0.0")
})()

app.post("/screenshot", validate(validation.screenshot), async (req, res) => {
  const { target, format, title, selector, clip } = req.body

  setContext({ target, format, title }, async () => {
    try {
      // Create a new incognito browser context.
      const context = await browser.createIncognitoBrowserContext()
      const page = await context.newPage()
      page.setDefaultNavigationTimeout(100000);

      page.on("pageerror", captureException)
      page.on("error", e => captureException(e))

      const token = req.header("access_token")
      const locale = req.header("locale")
      const dest = `${target}?access_token=${token}&locale=${locale}`
      await page.goto(dest, { waitUntil: "networkidle2", timeout: 100000 })

      const selectors = (selector || "").split(",")
      if (selectors.length) {
        selectors.forEach(async selector => {
          await page.waitFor(selector)
        })
      } else {
        await page.waitFor(5000)
      }

      let screenshot
      let screenshotOptions = {
        encoding: "base64",
        format: "A4",
        omitBackground: false
{'fraction_non_alphanumeric': 0.1546755131660792, 'fraction_numerical': 0.011818370308936347, 'mean_word_length': 3.105531914893617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '608955', 'n_tokens_mistral': 1558, 'n_tokens_neox': 1505, 'n_words': 319}