<commit_before>
assert            = require("assert")
Assert            = require("./assert")
createTabs        = require("./tabs")
Console           = require("./console")
Cookies           = require("./cookies")
{ EventEmitter }  = require("events")
EventLoop         = require("./eventloop")
{ format }        = require("util")
File              = require("fs")
HTML5             = require("html5")
Interact          = require("./interact")
HTML              = require("jsdom").dom.level3.html
Mime              = require("mime")
ms                = require("ms")
Q                 = require("q")
Path              = require("path")
Resources         = require("./resources")
Storages          = require("./storage")
URL               = require("url")
XPath             = require("jsdom").dom.level3.xpath


# DOM extensions.
require("./jsdom_patches")
require("./forms")
require("./dom_focus")
require("./dom_iframe")


# Browser options you can set when creating new browser, or on browser instance.
BROWSER_OPTIONS = ["debug", "features", "headers", "htmlParser", "waitDuration",
                   "proxy", "referer", "silent", "site", "userAgent",
                   "maxRedirects", "language"]

MOUSE_EVENT_NAMES = ["mousedown", "mousemove", "mouseup"]


# Use the browser to open up new windows and load documents.
#
# The browser maintains state for cookies and local storage.
class Browser extends EventEmitter
  constructor: (options = {}) ->
    browser = this
    @cookies = new Cookies()
    @_storages = new Storages()
    @_interact = Interact.use(this)
    # The window that is currently in scope, some JS functions need this, e.g.
    # when closing a window, you need to determine whether caller (window in
    # scope) is same as window.opener
    @_windowInScope = null

    # Used for assertions
    @assert = new Assert(this)


    # -- Console/Logging --

    # Shared by all windows.
    @console = new Console(this)

    # Message written to window.console.  Level is log, info, error, etc.
    #
    # Errors go to stderr, unless slient mode is on.
    # Debug go to stdout, if debug mode is on.
    # All other messages go to stdout, unless silent is on.
    @on "console", (level, message)->
      unless browser.silent
        switch level
          when "error"
            process.stderr.write(message + "\n")
          when "debug"
            if browser.debug
              process.stdout.write(message + "\n")
          else
            process.stdout.write(message + "\n")

    # Message written to browser.log.
    @on "log", (message)->
      if browser.debug
        process.stdout.write("Zombie: #{message}\n")


    # -- Resources --

    # Start with no this referer.
    @referer = null
    # All the resources loaded by this browser.
    @resources = new Resources(this)

    @on "request", (request)->

    @on "response", (request, response)->
      browser.log "#{request.method} #{request.url} => #{response.statusCode}"

    @on "redirect", (request, response)->
      browser.log "#{request.method} #{request.url} => #{response.statusCode} #{response.url}"

    # Document loaded.
    @on "loaded", (document)->
      browser.log "Loaded document", document.location.href


    # -- Tabs/Windows --

    # Open tabs.
    @tabs = createTabs(this)

    # Window has been opened
    @on "opened", (window)->
      browser.log "Opened window", window.location.href, window.name || ""

    # Window has been closed
    @on "closed", (window)->
      browser.log "Closed window", window.location.href, window.name || ""

    # Window becomes inactive
    @on "active", (window)->
      onfocus = window.document.createEvent("HTMLEvents")
      onfocus.initEvent("focus", false, false)
      window.dispatchEvent(onfocus)
      if element = window.document.activeElement
        onfocus = window.document.createEvent("HTMLEvents")
        onfocus.initEvent("focus", false, false)
        element.dispatchEvent(onfocus)

    # Window becomes inactive
    @on "inactive", (window)->
      if element = window.document.activeElement
        onblur = window.document.createEvent("HTMLEvents")
        onblur.initEvent("blur", false, false)
        element.dispatchEvent(onblur)
      onblur = window.document.createEvent("HTMLEvents")
      onblur.initEvent("blur", false, false)
      window.dispatchEvent(onblur)


    # -- Event loop --

    # The browser event loop.
    @eventLoop = new EventLoop(this)

    # Returns all errors reported while loading this window.
    @errors = []

    # Make sure we don't blow up Node when we get a JS error, but dump error to console.  Also, catch any errors
    # reported while processing resources/JavaScript.
    @on "error", (error)->
      browser.errors.push(error)
      browser.console.error(error.message, error.stack)

    @on "done", (timedOut)->
      if timedOut
        browser.log "Event loop timed out"
      else
        browser.log "Event loop is empty"

    @on "timeout", (fn, delay)->
      browser.log "Fired timeout after #{delay}ms delay"

    @on "interval", (fn, interval)->
      browser.log "Fired interval every #{interval}ms"

    @on "link", (url, target)->
      browser.log "Follow link to #{url}"

    @on "submit", (url, target)->
      browser.log "Submit form to #{url}"

    # Sets the browser options.
    for name in BROWSER_OPTIONS
      if options.hasOwnProperty(name)
        @[name] = options[name]
      else if Browser.default.hasOwnProperty(name)
        @[name] = Browser.default[name]


  # Returns true if the given feature is enabled.
  #
  # If the feature is listed, then it is enabled.  If the feature is listed
  # with "no-" prefix, then it is disabled.  If the feature is missing, return
  # the default value.
  hasFeature: (name, ifMissing = true)->
    if @features
      features = @features.split(/\s+/)
      if ~features.indexOf(name)
        return true
      if ~features.indexOf("no-#{name}")
        return false
    return ifMissing


  # Changes the browser options, and calls the function with a callback (reset).  When you're done processing, call the
  # reset function to bring options back to their previous values.
  #<commit_msg>Added back extension mechanism<commit_after>
assert            = require("assert")
Assert            = require("./assert")
createTabs        = require("./tabs")
Console           = require("./console")
Cookies           = require("./cookies")
{ EventEmitter }  = require("events")
EventLoop         = require("./eventloop")
{ format }        = require("util")
File              = require("fs")
HTML5             = require("html5")
Interact          = require("./interact")
HTML              = require("jsdom").dom.level3.html
Mime              = require("mime")
ms                = require("ms")
Q                 = require("q")
Path              = require("path")
Resources         = require("./resources")
Storages          = require("./storage")
URL               = require("url")
XPath             = require("jsdom").dom.level3.xpath


# DOM extensions.
require("./jsdom_patches")
require("./forms")
require("./dom_focus")
require("./dom_iframe")


# Browser options you can set when creating new browser, or on browser instance.
BROWSER_OPTIONS = ["debug", "features", "headers", "htmlParser", "waitDuration",
                   "proxy", "referer", "silent", "site", "userAgent",
                   "maxRedirects", "language"]

MOUSE_EVENT_NAMES = ["mousedown", "mousemove", "mouseup"]


# Use the browser to open up new windows and load documents.
#
# The browser maintains state for cookies and local storage.
class Browser extends EventEmitter
  constructor: (options = {}) ->
    browser = this
    @cookies = new Cookies()
    @_storages = new Storages()
    @_interact = Interact.use(this)
    # The window that is currently in scope, some JS functions need this, e.g.
    # when closing a window, you need to determine whether caller (window in
    # scope) is same as window.opener
    @_windowInScope = null

    # Used for assertions
    @assert = new Assert(this)


    # -- Console/Logging --

    # Shared by all windows.
    @console = new Console(this)

    # Message written to window.console.  Level is log, info, error, etc.
    #
    # Errors go to stderr, unless slient mode is on.
    # Debug go to stdout, if debug mode is on.
    # All other messages go to stdout, unless silent is on.
    @on "console", (level, message)->
      unless browser.silent
        switch level
          when "error"
            process.stderr.write(message + "\n")
          when "debug"
            if browser.debug
              process.stdout.write(message + "\n")
          else
            process.stdout.write(message + "\n")

    # Message written to browser.log.
    @on "log", (message)->
      if browser.debug
        process.stdout.write("Zombie: #{message}\n")


    # -- Resources --

    # Start with no this referer.
    @referer = null
    # All the resources loaded by this browser.
    @resources = new Resources(this)

    @on "request", (request)->

    @on "response", (request, response)->
      browser.log "#{request.method} #{request.url} => #{response.statusCode}"

    @on "redirect", (request, response)->
      browser.log "#{request.method} #{request.url} => #{response.statusCode} #{response.url}"

    # Document loaded.
    @on "loaded", (document)->
      browser.log "Loaded document", document.location.href


    # -- Tabs/Windows --

    # Open tabs.
    @tabs = createTabs(this)

    # Window has been opened
    @on "opened", (window)->
      browser.log "Opened window", window.location.href, window.name || ""

    # Window has been closed
    @on "closed", (window)->
      browser.log "Closed window", window.location.href, window.name || ""

    # Window becomes inactive
    @on "active", (window)->
      onfocus = window.document.createEvent("HTMLEvents")
      onfocus.initEvent("focus", false, false)
      window.dispatchEvent(onfocus)
      if element = window.document.activeElement
        onfocus = window.document.createEvent("HTMLEvents")
        onfocus.initEvent("focus", false, false)
        element.dispatchEvent(onfocus)

    # Window becomes inactive
    @on "inactive", (window)->
      if element = window.document.activeElement
        onblur = window.document.createEvent("HTMLEvents")
        onblur.initEvent("blur", false, false)
        element.dispatchEvent(onblur)
      onblur = window.document.createEvent("HTMLEvents")
      onblur.initEvent("blur", false, false)
      window.dispatchEvent(onblur)


    # -- Event loop --

    # The browser event loop.
    @eventLoop = new EventLoop(this)

    # Returns all errors reported while loading this window.
    @errors = []

    # Make sure we don't blow up Node when we get a JS error, but dump error to console.  Also, catch any errors
    # reported while processing resources/JavaScript.
    @on "error", (error)->
      browser.errors.push(error)
      browser.console.error(error.message, error.stack)

    @on "done", (timedOut)->
      if timedOut
        browser.log "Event loop timed out"
      else
        browser.log "Event loop is empty"

    @on "timeout", (fn, delay)->
      browser.log "Fired timeout after #{delay}ms delay"

    @on "interval", (fn, interval)->
      browser.log "Fired interval every #{interval}ms"

    @on "link", (url, target)->
      browser.log "Follow link to #{url}"

    @on "submit", (url, target)->
      browser.log "Submit form to #{url}"

    # Sets the browser options.
    for name in BROWSER_OPTIONS
      if options.hasOwnProperty(name)
        @[name] = options[name]
      else if Browser.default.hasOwnProperty(name)
        @[name] = Browser.default[name]

    # Last, run all extensions in order.
    for extension in Browser._extensions
      extension(this)


  # Register a browser extension.
  #
  # Browser extensions are called for each newly created browser object, and
  # can be used to change browser options, register listeners, add methods,
  # etc.
  @extend: (extension)->
    Browser._extensions.push(extension)

  @_extensions = []

  # Returns true if the given feature is enabled.
  #
  # If the feature is listed, then it is enabled.  If the feature is listed
  # with "no-" prefix, then it is disabled.  If the feature is missing, return
  # the default value.
  hasFeature: (name, ifMissing = true)->
    if @features
      features = @features.split(/\s+/)
      if ~features.indexOf(name)
        return true
      if ~features.indexOf("no-#{name}")
        return false
    return ifMissing


  # Changes the browser options, and calls the function with a callback (reset).  When you're done processing, call the
  # reset function to bring options back to their previous values.
  #
{'fraction_non_alphanumeric': 0.12440378450230667, 'fraction_numerical': 0.0006255375713503792, 'mean_word_length': 2.554752640355753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1276629', 'n_tokens_mistral': 3686, 'n_tokens_neox': 3551, 'n_words': 1147}