<commit_before>//
// Strider Worker extension for Browser Stack tests
//


var browserstack = require('browserstack')
var fs = require('fs')
var path = require('path')
var request = require('request')

// # of tries for application-under-test webserver to start up on specified port
// We test once per second
var HTTP_CHECK_RETRIES = 10

// Interval in between HTTP checks in ms
var HTTP_CHECK_INTERVAL = 1000

// Port on which the application-under-test webserver should bind to on localhost.
// Browser Stack Connector will tunnel from this to BrowserStack Cloud for tests
var HTTP_PORT

// Path to BrowserStack Connector PID file
var PIDFILE = path.join(__dirname, "tunnel.pid")

var connectorProc

var cleanupRun = false

// Read & parse a JSON file
function getJson(filename, cb) {
  fs.readFile(filename, function(err, data) {
    if (err) return cb(err, null)
    try {
      var json = JSON.parse(data)
      cb(null, json)
    } catch(e) {
      cb(e, null)
    }
  })
}

// Is browserStack configured for this context?
function browserStackConfigured(ctx) {
  var browserStackAPIKey = ctx.jobData.repo_config.browserstack_api_key
  var browserStackUsername = ctx.jobData.repo_config.browserstack_username
  var browserStackPassword = ctx.jobData.repo_config.browserstack_password

  if (browserStackAPIKey === undefined || browserStackPassword === undefined || browserStackUsername === undefined) {
    return false
  }

  return true
}

// This will shut down the tunnel
function cleanup(ctx, cb) {
  if (!browserStackConfigured(ctx)) {
    return cb(0)
  }
  cleanupRun = true
  var msg = "Shutting down BrowserStack Connector"
  console.log(msg)
  ctx.striderMessage(msg)
  connectorProc.kill("SIGINT")
  // Give BrowserStack Connector 5 seconds to gracefully stop before sending SIGKILL
  setTimeout(function() {
    connectorProc.kill("SIGKILL")
    fs.unlink(PIDFILE)
    msg = "BrowserStack Connector successfully shut down"
    console.log(msg)
    ctx.striderMessage(msg)

    return cb(0)
  }, 5000)
}

// then we start the BrowserStack test process.
function test(ctx, cb) {
  HTTP_PORT = ctx.browsertestPort || 8031

  var browserStackAPIKey = ctx.jobData.repo_config.browserstack_api_key
  var browserStackUsername = ctx.jobData.repo_config.browserstack_username
  var browserStackPassword = ctx.jobData.repo_config.browserstack_password
  var browserStackBrowsers = ctx.jobData.repo_config.browserstack_browsers
  if (browserStackBrowsers === undefined || browserStackBrowsers.length === 0) {
    // XXX Some default
    browserStackBrowsers = [
      {
        browserName:"chrome",
        version:"",
        platform:"Windows 2008"
      }
    ]
  }
  // Only start browserstack if 
  if (!browserStackConfigured(ctx)) {
    return cb(0)
  }
  var startPhaseDone = false
  // Run 
  ctx.striderMessage("trying BrowserStack tests...")
  // The project webserver should be available via HTTP once started.
  // This section implements a check which will attempt to make a HTTP request for /
  // expecting a 200 response code. It will try HTTP_CHECK_RETRIES times, waiting 1 second
  // between checks. If it fails after HTTP_CHECK_RETRIES times, the server process will be killed
  // and the test failed.
  var tries = 0
  ctx.striderMessage("Waiting for webserver to come up on localhost:" + HTTP_PORT)
  var intervalId = setInterval(function() {
    // Check for http response on http://localhost:HTTP_PORT/
    request("http://localhost:"+HTTP_PORT+"/", function(err, response) {
      if (startPhaseDone) {
        clearInterval(intervalId)
        return
      }
      if (!err && response.statusCode) {
        ctx.striderMessage("Got HTTP response on localhost:" + HTTP_PORT + " indicating server is up")
        startPhaseDone = true
        clearInterval(intervalId)
        serverUp()
      } else {
        tries++
        console.log("Error on localhost:%d: %s", HTTP_PORT, err)
        if (tries >= HTTP_CHECK_RETRIES) {
          var msg = ("HTTP check on localhost:" + HTTP_PORT + " failed after " + tries
            + " retries, server not up - failing test")
          ctx.striderMessage(msg)
          clearInterval(intervalId)
          startPhaseDone = true
          return cb(1)
        }
      }
    })
  }, HTTP_CHECK_INTERVAL)

  // Start the BrowserStack Connector. Returns childProcess object.
  function startConnector(username, password, apiKey, exitCb) {
    // Check for existing tunnel PID
    fs.readFile(PIDFILE, function(err, data) {
      if (!err && data) {
        console.debug("strider-browserstack: found existing PIDfile, killing process %s", data)
        try {
          process.kill(data, "SIGKILL")
        } catch(e) {
          console.log("exception: " + e)

        }<commit_msg>update comments & prettify code<commit_after>//
// Strider Worker extension for Browser Stack tests
//


var browserstack = require('browserstack')
var fs = require('fs')
var path = require('path')
var request = require('request')

// # of tries for application-under-test webserver to start up on specified port
// We test once per second
var HTTP_CHECK_RETRIES = 10

// Interval in between HTTP checks in ms
var HTTP_CHECK_INTERVAL = 1000

// Port on which the application-under-test webserver should bind to on localhost.
// Browser Stack Connector will tunnel from this to BrowserStack Cloud for tests
var HTTP_PORT

// Path to BrowserStack Connector PID file
var PIDFILE = path.join(__dirname, "tunnel.pid")

var connectorProc

var cleanupRun = false

// Read & parse a JSON file
function getJson(filename, cb) {
  fs.readFile(filename, function(err, data) {
    if (err) return cb(err, null)
    try {
      var json = JSON.parse(data)
      cb(null, json)
    } catch(e) {
      cb(e, null)
    }
  })
}

// Is browserStack configured for this context?
function browserStackConfigured(ctx) {
  var browserStackAPIKey = ctx.jobData.repo_config.browserstack_api_key
  var browserStackUsername = ctx.jobData.repo_config.browserstack_username
  var browserStackPassword = ctx.jobData.repo_config.browserstack_password

  if (browserStackAPIKey === undefined || browserStackPassword === undefined || browserStackUsername === undefined) {
    return false
  }

  return true
}

// This will shut down the tunnel in a nice way
function cleanup(ctx, cb) {
  if (!browserStackConfigured(ctx)) {
    return cb(0)
  }
  cleanupRun = true
  var msg = "Shutting down BrowserStack Connector"
  console.log(msg)
  ctx.striderMessage(msg)
  connectorProc.kill("SIGINT")
  // Give BrowserStack Connector 5 seconds to gracefully stop before sending SIGKILL
  setTimeout(function() {
    connectorProc.kill("SIGKILL")
    fs.unlink(PIDFILE)
    msg = "BrowserStack Connector successfully shut down"
    console.log(msg)
    ctx.striderMessage(msg)

    return cb(0)
  }, 5000)
}

// BrowserStack test process.
function test(ctx, cb) {
  // Only start BrowserStack if it is configured for this project.

  if (!browserStackConfigured(ctx)) {
    return cb(0)
  }

  HTTP_PORT = ctx.browsertestPort || 8031

  var browserStackAPIKey = ctx.jobData.repo_config.browserstack_api_key
  var browserStackUsername = ctx.jobData.repo_config.browserstack_username
  var browserStackPassword = ctx.jobData.repo_config.browserstack_password
  var browserStackBrowsers = ctx.jobData.repo_config.browserstack_browsers
  if (browserStackBrowsers === undefined || browserStackBrowsers.length === 0) {
    // XXX Some default
    browserStackBrowsers = [
      {
        browserName:"chrome",
        version:"",
        platform:"Windows 2008"
      }
    ]
  }
  var startPhaseDone = false
  // Run 
  ctx.striderMessage("trying BrowserStack tests...")
  // The project webserver should be available via HTTP once started.
  // This section implements a check which will attempt to make a HTTP request for /
  // expecting a 200 response code. It will try HTTP_CHECK_RETRIES times, waiting 1 second
  // between checks. If it fails after HTTP_CHECK_RETRIES times, the server process will be killed
  // and the test failed.
  var tries = 0
  ctx.striderMessage("Waiting for webserver to come up on localhost:" + HTTP_PORT)
  var intervalId = setInterval(function() {
    // Check for http response on http://localhost:HTTP_PORT/
    request("http://localhost:"+HTTP_PORT+"/", function(err, response) {
      if (startPhaseDone) {
        clearInterval(intervalId)
        return
      }
      if (!err && response.statusCode) {
        ctx.striderMessage("Got HTTP response on localhost:" + HTTP_PORT + " indicating server is up")
        startPhaseDone = true
        clearInterval(intervalId)
        serverUp()
      } else {
        tries++
        console.log("Error on localhost:%d: %s", HTTP_PORT, err)
        if (tries >= HTTP_CHECK_RETRIES) {
          var msg = ("HTTP check on localhost:" + HTTP_PORT + " failed after " + tries
            + " retries, server not up - failing test")
          ctx.striderMessage(msg)
          clearInterval(intervalId)
          startPhaseDone = true
          return cb(1)
        }
      }
    })
  }, HTTP_CHECK_INTERVAL)

  // Start the BrowserStack Connector, killing previous one if PID file found. Write out PID file for new process.
  // Wait until Connector ready.
  function startConnector(username, password, apiKey, exitCb) {
    // Check for existing tunnel PID
    fs.readFile(PIDFILE, function(err, data) {
      if (!err && data) {
        console.debug("strider-browserstack: found existing PIDfile, killing process %s", data)
        try {
          process.kill(data, "SIGKILL")
        } catch(e) {
          console.log("exception: " + e)

        }
{'fraction_non_alphanumeric': 0.09556384742951907, 'fraction_numerical': 0.0060116086235489224, 'mean_word_length': 3.206190061028771, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '849497', 'n_tokens_mistral': 2868, 'n_tokens_neox': 2762, 'n_words': 854}