<commit_before>// Licensed to the Software Freedom Conservancy (SFC) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The SFC licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.openqa.selenium.htmlunit;

import static org.openqa.selenium.remote.CapabilityType.ACCEPT_SSL_CERTS;
import static org.openqa.selenium.remote.CapabilityType.PAGE_LOAD_STRATEGY;
import static org.openqa.selenium.remote.CapabilityType.SUPPORTS_FINDING_BY_CSS;
import static org.openqa.selenium.remote.CapabilityType.UNEXPECTED_ALERT_BEHAVIOUR;

import java.io.IOException;
import java.lang.reflect.Field;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import javax.net.ssl.SSLHandshakeException;

import org.openqa.selenium.Alert;
import org.openqa.selenium.By;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.HasCapabilities;
import org.openqa.selenium.InvalidCookieDomainException;
import org.openqa.selenium.InvalidSelectorException;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.NoAlertPresentException;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.NoSuchFrameException;
import org.openqa.selenium.NoSuchSessionException;
import org.openqa.selenium.NoSuchWindowException;
import org.openqa.selenium.Platform;
import org.openqa.selenium.Point;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.SearchContext;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.UnableToSetCookieException;
import org.openqa.selenium.UnexpectedAlertBehaviour;
import org.openqa.selenium.UnhandledAlertException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.interactions.HasInputDevices;
import org.openqa.selenium.interactions.Keyboard;
import org.openqa.selenium.interactions.Mouse;
import org.openqa.selenium.internal.FindsByClassName;
import org.openqa.selenium.internal.FindsByCssSelector;
import org.openqa.selenium.internal.FindsById;
import org.openqa.selenium.internal.FindsByLinkText;
import org.openqa.selenium.internal.FindsByName;
import org.openqa.selenium.internal.FindsByTagName;
import org.openqa.selenium.internal.FindsByXPath;
import org.openqa.selenium.internal.WrapsElement;
import org.openqa.selenium.logging.Logs;
import org.openqa.selenium.remote.BrowserType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.gargoylesoftware.css.parser.CSSException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.BrowserVersion.BrowserVersionBuilder;
import com.gargoylesoftware.htmlunit.CookieManager;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.ScriptResult;
import com.gargoylesoftware.htmlunit.SgmlPage;
import com.gargoylesoftware.htmlunit.StringWebResponse;
import com.gargoylesoftware.htmlunit.TopLevelWindow;
import com.gargoylesoftware.htmlunit.UnexpectedPage;
import com.gargoylesoftware.htmlunit.Version;
import com.gargoylesoftware.htmlunit.WaitingRefreshHandler;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebResponse;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.WebWindowEvent;
import com.gargoylesoftware.htmlunit.WebWindowListener;
import com.gargoylesoftware.htmlunit.WebWindowNotFoundException;
import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.FrameWindow;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlHtml;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.javascript.host.Element;
import com.gargoylesoftware.htmlunit.javascript.host.Location;
import com.gargoylesoftware.htmlunit.javascript.host.html.DocumentProxy;
import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLCollection;
import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLElement;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

import net.sourceforge.htmlunit.corejs.javascript.Context;
import net.sourceforge.htmlunit.corejs.javascript.IdScriptableObject;
import net.sourceforge.htmlunit.corejs.javascript.NativeArray;
import net.sourceforge.htmlunit.corejs.javascript.NativeObject;
import net.sourceforge.htmlunit.corejs.javascript.Scriptable;
import net.sourceforge.htmlunit.corejs.javascript.Undefined;

/**
 * An implementation of {@link WebDriver} that drives <a href="http://htmlunit.sourceforge.net/">HtmlUnit</a>,
 * which is a headless (GUI-less) browser simulator.
 * <p>The main supported browsers are Chrome, Firefox and Internet Explorer.
 */
public class HtmlUnitDriver implements WebDriver, JavascriptExecutor,
    FindsById, FindsByLinkText, FindsByXPath, FindsByName, FindsByCssSelector,
    FindsByTagName, FindsByClassName, HasCapabilities, HasInputDevices {

  private static final int sleepTime = 200;

  private WebClient webClient;
  private WebWindow currentWindow;
  private HtmlUnitAlert alert;

  // Fictive position just to implement the API
  private Point windowPosition = new Point(0, 0);
  private Dimension initialWindowDimension;

  private boolean enableJavascript;
  private ProxyConfig proxyConfig;
  private long implicitWait = 0;
  private long scriptTimeout = 0;
  private HtmlUnitKeyboard keyboard;
  private HtmlUnitMouse mouse;
  private boolean gotPage;
  private TargetLocator targetLocator = new HtmlUnitTargetLocator();
  private AsyncScriptExecutor asyncScriptExecutor;
  private UnexpectedAlertBehaviour unexpectedAlertBehaviour;
  private PageLoadStrategy pageLoadStrategy = PageLoadStrategy.NORMAL;
  private int elementsCounter;
  private Map<SgmlPage, Map<DomElement, HtmlUnitWebElement>> elementsMap = new WeakHashMap<>();

  public static final String INVALIDXPATHERROR = "The xpath expression '%s' cannot be evaluated";
  public static final String INVALIDSELECTIONERROR =
      "The xpath expression '%s' selected an object of type '%s' instead of a WebElement";

  public static final String BROWSER_LANGUAGE_CAPABILITY = "browserLanguage";
  public static final String DOWNLOAD_IMAGES_CAPABILITY = "downloadImages";
  public static final String JAVASCRIPT_ENABLED = "javascriptEnabled";

  /** The Lock for the {@link #mainCondition}, which waits at the end of {@link #runAsync(Runnable)}
   * till either and alert is triggered, or {@link Runnable} finishes. */
  private Lock conditionLock = new ReentrantLock();
  private Condition mainCondition = conditionLock.newCondition();
  private boolean runAsyncRunning;
  private RuntimeException exception;

  /**
   * Constructs a new instance with JavaScript disabled,
   * and the {@link BrowserVersion#getDefault() default} BrowserVersion.
   */
  public HtmlUnitDriver() {
    this(false);
  }

  /**
   * Constructs a new instance, specify JavaScript support
   * and using the {@link BrowserVersion#getDefault() default} BrowserVersion.
   *
   * @param enableJavascript whether to enable JavaScript support or not
   */
  public HtmlUnitDriver(boolean enableJavascript) {
    this(BrowserVersion.getDefault(), enableJavascript);
  }

  /**
   * Constructs a new instance with the specified {@link BrowserVersion} and the JavaScript support.
   *
   * @param version the browser version to use
   * @param enableJavascript whether to enable JavaScript support or not
   */
  public HtmlUnitDriver(BrowserVersion version, boolean enableJavascript) {
    this(version);
    setJavascriptEnabled(enableJavascript);
  }

  /**
   * Constructs a new instance with the specified {@link BrowserVersion}.
   *
   * @param version the browser version to use
   */
  public HtmlUnitDriver(BrowserVersion version) {
    webClient = createWebClient(version);
    alert = new HtmlUnitAlert(this);
    currentWindow = webClient.getCurrentWindow();
    initialWindowDimension = new Dimension(currentWindow.getOuterWidth(), currentWindow.getOuterHeight());
    unexpectedAlertBehaviour = UnexpectedAlertBehaviour.DISMISS_AND_NOTIFY;

    webClient.addWebWindowListener(new WebWindowListener() {
      @Override
      public void webWindowOpened(WebWindowEvent webWindowEvent) {
        // Ignore
      }

      @Override
      public void webWindowContentChanged(WebWindowEvent event) {
        elementsMap.remove(event.getOldPage());
        if (event.getWebWindow() != currentWindow) {
          return;
        }

        // Do we need to pick some new default content?
        switchToDefaultContentOfWindow(currentWindow);
      }

      @Override
      public void webWindowClosed(WebWindowEvent event) {
        elementsMap.remove(event.getOldPage());
        // Check if the event window refers to us or one of our parent windows
        // setup the currentWindow appropriately if necessary
        WebWindow curr = currentWindow;
        do {
          // Instance equality is okay in this case
          if (curr == event.getWebWindow()) {
            currentWindow = currentWindow.getTopWindow();
            return;
          }
          curr = curr.getParentWindow();
        } while (curr != currentWindow.getTopWindow());
      }
    });

    // Now put us on the home page, like a real browser
    get(webClient.getOptions().getHomePage());
    gotPage = false;
    resetKeyboardAndMouseState();
  }

  /**
   * The browserName is {@link BrowserType#HTMLUNIT} "htmlunit" and the browserVersion
   * denotes the required browser AND its version.
   * For example "chrome" for Chrome, "firefox-45" for Firefox 45
   * or "internet explorer" for IE.
   *
   * @param capabilities desired capabilities requested for the htmlunit driver session
   */
  public HtmlUnitDriver(Capabilities capabilities) {
    this(determineBrowserVersion(capabilities));

    setJavascriptEnabled(capabilities.getCapability(JAVASCRIPT_ENABLED) == null
        || capabilities.is(JAVASCRIPT_ENABLED));

    setProxySettings(Proxy.extractFrom(capabilities));

    setDownloadImages(capabilities.is(DOWNLOAD_IMAGES_CAPABILITY));
    
    unexpectedAlertBehaviour = (UnexpectedAlertBehaviour) capabilities.getCapability(UNEXPECTED_ALERT_BEHAVIOUR);
    if (unexpectedAlertBehaviour == null) {
      unexpectedAlertBehaviour = UnexpectedAlertBehaviour.DISMISS_AND_NOTIFY;
    }
    
    Boolean acceptSslCerts = (Boolean) capabilities.getCapability(ACCEPT_SSL_CERTS);
    if (acceptSslCerts == null) {
      acceptSslCerts = true;
    }
    setAcceptSslCertificates(acceptSslCerts);

    String pageLoadStrategyString = (String) capabilities.getCapability(PAGE_LOAD_STRATEGY);
    if ("none".equals(pageLoadStrategyString)) {
      pageLoadStrategy = PageLoadStrategy.NONE;
    }
    else if ("eager".equals(pageLoadStrategyString)) {
      pageLoadStrategy = PageLoadStrategy.EAGER;
    }
  }

  public HtmlUnitDriver(Capabilities desiredCapabilities, Capabilities requiredCapabilities) {
    this(new DesiredCapabilities(desiredCapabilities, requiredCapabilities));
  }

  static BrowserVersion determineBrowserVersion(Capabilities capabilities) {
    if (!BrowserType.HTMLUNIT.equals(capabilities.getBrowserName())) {
      throw new IllegalArgumentException("Browser name must be " + BrowserType.HTMLUNIT);
    }
    String browserName;
    String browserVersion;

    String rawVersion = capabilities.getVersion();
    String[] splitVersion = rawVersion == null ? new String[0] : rawVersion.split("-");
    if (splitVersion.length > 1) {
      browserName = splitVersion[0];
      browserVersion = splitVersion[1];
    } else {
      browserName = capabilities.getVersion();
      browserVersion = null;
    }

    BrowserVersion browserVersionObject;
    switch (browserName) {
      case BrowserType.CHROME:
        browserVersionObject = BrowserVersion.CHROME;
        break;

      case BrowserType.IE:
        browserVersionObject = BrowserVersion.INTERNET_EXPLORER;
        break;

      case BrowserType.EDGE:
          browserVersionObject = BrowserVersion.EDGE;
          break;

      case BrowserType.FIREFOX:
        try {
          int version = Integer.parseInt(browserVersion);
          switch (version) {
            case 52:
              browserVersionObject = BrowserVersion.FIREFOX_52;
              break;

            default:
              browserVersionObject = BrowserVersion.FIREFOX_60;
          }
        } catch (NumberFormatException e) {
            browserVersionObject = BrowserVersion.FIREFOX_60;
        }
        break;

      default:
        browserVersionObject = BrowserVersion.getDefault();
        break;
    }

    Object rawLanguage = capabilities.getCapability(BROWSER_LANGUAGE_CAPABILITY);
    if (rawLanguage instanceof String) {
      browserVersionObject = new BrowserVersionBuilder(browserVersionObject)
          .setBrowserLanguage((String) rawLanguage).build();
    }

    return browserVersionObject;
  }

  private WebClient createWebClient(BrowserVersion version) {
    WebClient client = newWebClient(version);
    WebClientOptions options = client.getOptions();
    options.setHomePage(WebClient.URL_ABOUT_BLANK.toString());
    options.setThrowExceptionOnFailingStatusCode(false);
    options.setPrintContentOnFailingStatusCode(false);
    options.setJavaScriptEnabled(enableJavascript);
    options.setRedirectEnabled(true);
    options.setUseInsecureSSL(true);

    // Ensure that we've set the proxy if necessary
    if (proxyConfig != null) {
      options.setProxyConfig(proxyConfig);
    }

    client.setRefreshHandler(new WaitingRefreshHandler());

    return modifyWebClient(client);
  }

  /**
   * @return to process or not to proceed
   */
  boolean isProcessAlert() {
    if (asyncScriptExecutor != null) {
      String text = alert.getText();
      alert.dismiss();
      asyncScriptExecutor.alertTriggered(text);
      return false;
    }
    conditionLock.lock();
    mainCondition.signal();
    conditionLock.unlock();
    return true;
  }

  void runAsync(Runnable r) {
    boolean loadStrategyWait = pageLoadStrategy != PageLoadStrategy.NONE;

    if (loadStrategyWait) {
      while (runAsyncRunning) {
        try {
          Thread.sleep(10);
        } catch (InterruptedException e) {
          throw new RuntimeException(e);
        }
      }
      conditionLock.lock();
      runAsyncRunning = true;
    }
    
    exception = null;
    new Thread(() -> {
      try {
        r.run();
      }
      catch (RuntimeException e) {
        exception = e;
      }
      finally {
        conditionLock.lock();
        mainCondition.signal();
        runAsyncRunning = false;
        conditionLock.unlock();
      }
    }).start();

    if (loadStrategyWait) {
      mainCondition.awaitUninterruptibly();
      conditionLock.unlock();
    }

    if (exception != null) {
      throw exception;
    }
  }

  void click(DomElement element, boolean directClick) {
    runAsync(() -> mouse.click(element, directClick));
  }

  void doubleClick(DomElement element) {
    runAsync(() -> mouse.doubleClick(element));
  }

  void mouseUp(DomElement element) {
    runAsync(() -> mouse.mouseUp(element));
  }

  void mouseMove(DomElement element) {
    runAsync(() -> mouse.mouseMove(element));
  }

  void mouseDown(DomElement element) {
    runAsync(() -> mouse.mouseDown(element));
  }

  void submit(HtmlUnitWebElement element) {
    runAsync(() -> element.submitImpl());
  }

  void sendKeys(HtmlUnitWebElement element, CharSequence... value) {
    runAsync(() -> keyboard.sendKeys(element, true, value));
  }

  /**
   * Get the simulated {@code BrowserVersion}.
   * @return the used {@code BrowserVersion}
   */
  public BrowserVersion getBrowserVersion() {
    return webClient.getBrowserVersion();
  }

  /**
   * Create the underlying WebClient, but don't set any fields on it.
   *
   * @param version Which browser to emulate
   * @return a new instance of WebClient.
   */
  protected WebClient newWebClient(BrowserVersion version) {
    return new WebClient(version);
  }

  /**
   * Child classes can override this method to customize the WebClient that the HtmlUnit driver
   * uses.
   *
   * @param client The client to modify
   * @return The modified client
   */
  protected WebClient modifyWebClient(WebClient client) {
    // Does nothing here to be overridden.
    return client;
  }

  /**
   * Set proxy for WebClient using Proxy.
   *
   * @param proxy The proxy preferences.
   */
  public void setProxySettings(Proxy proxy) {
    if (proxy == null || proxy.getProxyType() == Proxy.ProxyType.UNSPECIFIED) {
      return;
    }

    switch (proxy.getProxyType()) {
      case MANUAL:
        List<String> noProxyHosts = new ArrayList<>();
        String noProxy = proxy.getNoProxy();
        if (noProxy != null && !noProxy.isEmpty()) {
          String[] hosts = noProxy.split(",");
          for (String host : hosts) {
            if (host.trim().length() > 0) {
              noProxyHosts.add(host.trim());
            }
          }
        }

        String httpProxy = proxy.getHttpProxy();
        if (httpProxy != null && !httpProxy.isEmpty()) {
          String host = httpProxy;
          int port = 0;

          int index = httpProxy.indexOf(":");
          if (index != -1) {
            host = httpProxy.substring(0, index);
            port = Integer.parseInt(httpProxy.substring(index + 1));
          }

          setHTTPProxy(host, port, noProxyHosts);
        }

        String socksProxy = proxy.getSocksProxy();
        if (socksProxy != null && !socksProxy.isEmpty()) {
          String host = socksProxy;
          int port = 0;

          int index = socksProxy.indexOf(":");
          if (index != -1) {
            host = socksProxy.substring(0, index);
            port = Integer.parseInt(socksProxy.substring(index + 1));
          }

          setSocksProxy(host, port, noProxyHosts);
        }

        // sslProxy is not supported/implemented
        // ftpProxy is not supported/implemented

        break;

      case PAC:
        String pac = proxy.getProxyAutoconfigUrl();
        if (pac != null && !pac.isEmpty()) {
          setAutoProxy(pac);
        }
        break;

      default:
          break;
    }
  }

  /**
   * Sets HTTP proxy for WebClient
   *
   * @param host The hostname of HTTP proxy
   * @param port The port of HTTP proxy, 0 means HTTP proxy w/o port
   */
  public void setProxy(String host, int port) {
    setHTTPProxy(host, port, null);
  }

  /**
   * Sets HTTP proxy for WebClient with bypass proxy hosts
   *
   * @param host The hostname of HTTP proxy
   * @param port The port of HTTP proxy, 0 means HTTP proxy w/o port
   * @param noProxyHosts The list of hosts which need to bypass HTTP proxy
   */
  public void setHTTPProxy(String host, int port, List<String> noProxyHosts) {
    proxyConfig = new ProxyConfig();
    proxyConfig.setProxyHost(host);
    proxyConfig.setProxyPort(port);
    if (noProxyHosts != null && noProxyHosts.size() > 0) {
      for (String noProxyHost : noProxyHosts) {
        proxyConfig.addHostsToProxyBypass(noProxyHost);
      }
    }
    getWebClient().getOptions().setProxyConfig(proxyConfig);
  }

  /**
   * Sets SOCKS proxy for WebClient
   *
   * @param host The hostname of SOCKS proxy
   * @param port The port of SOCKS proxy, 0 means HTTP proxy w/o port
   */
  public void setSocksProxy(String host, int port) {
    setSocksProxy(host, port, null);
  }

  /**
   * Sets SOCKS proxy for WebClient with bypass proxy hosts
   *
   * @param host The hostname of SOCKS proxy
   * @param port The port of SOCKS proxy, 0 means HTTP proxy w/o port
   * @param noProxyHosts The list of hosts which need to bypass SOCKS proxy
   */
  public void setSocksProxy(String host, int port, List<String> noProxyHosts) {
    proxyConfig = new ProxyConfig();
    proxyConfig.setProxyHost(host);
    proxyConfig.setProxyPort(port);
    proxyConfig.setSocksProxy(true);
    if (noProxyHosts != null && noProxyHosts.size() > 0) {
      for (String noProxyHost : noProxyHosts) {
        proxyConfig.addHostsToProxyBypass(noProxyHost);
      }
    }
    getWebClient().getOptions().setProxyConfig(proxyConfig);
  }

  /**<commit_msg>Add HtmlUnitDriver Executor (#69)<commit_after>// Licensed to the Software Freedom Conservancy (SFC) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The SFC licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package org.openqa.selenium.htmlunit;

import static org.openqa.selenium.remote.CapabilityType.ACCEPT_SSL_CERTS;
import static org.openqa.selenium.remote.CapabilityType.PAGE_LOAD_STRATEGY;
import static org.openqa.selenium.remote.CapabilityType.SUPPORTS_FINDING_BY_CSS;
import static org.openqa.selenium.remote.CapabilityType.UNEXPECTED_ALERT_BEHAVIOUR;

import java.io.IOException;
import java.lang.reflect.Field;
import java.net.ConnectException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.UnknownHostException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;

import javax.net.ssl.SSLHandshakeException;

import org.openqa.selenium.Alert;
import org.openqa.selenium.By;
import org.openqa.selenium.Capabilities;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.HasCapabilities;
import org.openqa.selenium.InvalidCookieDomainException;
import org.openqa.selenium.InvalidSelectorException;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.NoAlertPresentException;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.NoSuchFrameException;
import org.openqa.selenium.NoSuchSessionException;
import org.openqa.selenium.NoSuchWindowException;
import org.openqa.selenium.Platform;
import org.openqa.selenium.Point;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.SearchContext;
import org.openqa.selenium.StaleElementReferenceException;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.UnableToSetCookieException;
import org.openqa.selenium.UnexpectedAlertBehaviour;
import org.openqa.selenium.UnhandledAlertException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebDriverException;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.interactions.HasInputDevices;
import org.openqa.selenium.interactions.Keyboard;
import org.openqa.selenium.interactions.Mouse;
import org.openqa.selenium.internal.FindsByClassName;
import org.openqa.selenium.internal.FindsByCssSelector;
import org.openqa.selenium.internal.FindsById;
import org.openqa.selenium.internal.FindsByLinkText;
import org.openqa.selenium.internal.FindsByName;
import org.openqa.selenium.internal.FindsByTagName;
import org.openqa.selenium.internal.FindsByXPath;
import org.openqa.selenium.internal.WrapsElement;
import org.openqa.selenium.logging.Logs;
import org.openqa.selenium.remote.BrowserType;
import org.openqa.selenium.remote.DesiredCapabilities;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.gargoylesoftware.css.parser.CSSException;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.BrowserVersion.BrowserVersionBuilder;
import com.gargoylesoftware.htmlunit.CookieManager;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.ProxyConfig;
import com.gargoylesoftware.htmlunit.ScriptResult;
import com.gargoylesoftware.htmlunit.SgmlPage;
import com.gargoylesoftware.htmlunit.StringWebResponse;
import com.gargoylesoftware.htmlunit.TopLevelWindow;
import com.gargoylesoftware.htmlunit.UnexpectedPage;
import com.gargoylesoftware.htmlunit.Version;
import com.gargoylesoftware.htmlunit.WaitingRefreshHandler;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebResponse;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.WebWindowEvent;
import com.gargoylesoftware.htmlunit.WebWindowListener;
import com.gargoylesoftware.htmlunit.WebWindowNotFoundException;
import com.gargoylesoftware.htmlunit.html.BaseFrameElement;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.DomNodeList;
import com.gargoylesoftware.htmlunit.html.FrameWindow;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlHtml;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.javascript.host.Element;
import com.gargoylesoftware.htmlunit.javascript.host.Location;
import com.gargoylesoftware.htmlunit.javascript.host.html.DocumentProxy;
import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLCollection;
import com.gargoylesoftware.htmlunit.javascript.host.html.HTMLElement;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

import net.sourceforge.htmlunit.corejs.javascript.Context;
import net.sourceforge.htmlunit.corejs.javascript.IdScriptableObject;
import net.sourceforge.htmlunit.corejs.javascript.NativeArray;
import net.sourceforge.htmlunit.corejs.javascript.NativeObject;
import net.sourceforge.htmlunit.corejs.javascript.Scriptable;
import net.sourceforge.htmlunit.corejs.javascript.Undefined;

/**
 * An implementation of {@link WebDriver} that drives <a href="http://htmlunit.sourceforge.net/">HtmlUnit</a>,
 * which is a headless (GUI-less) browser simulator.
 * <p>The main supported browsers are Chrome, Firefox and Internet Explorer.
 */
public class HtmlUnitDriver implements WebDriver, JavascriptExecutor,
    FindsById, FindsByLinkText, FindsByXPath, FindsByName, FindsByCssSelector,
    FindsByTagName, FindsByClassName, HasCapabilities, HasInputDevices {

  private static final int sleepTime = 200;

  private WebClient webClient;
  private WebWindow currentWindow;
  private HtmlUnitAlert alert;

  // Fictive position just to implement the API
  private Point windowPosition = new Point(0, 0);
  private Dimension initialWindowDimension;

  private boolean enableJavascript;
  private ProxyConfig proxyConfig;
  private long implicitWait = 0;
  private long scriptTimeout = 0;
  private HtmlUnitKeyboard keyboard;
  private HtmlUnitMouse mouse;
  private boolean gotPage;
  private TargetLocator targetLocator = new HtmlUnitTargetLocator();
  private AsyncScriptExecutor asyncScriptExecutor;
  private UnexpectedAlertBehaviour unexpectedAlertBehaviour;
  private PageLoadStrategy pageLoadStrategy = PageLoadStrategy.NORMAL;
  private int elementsCounter;
  private Map<SgmlPage, Map<DomElement, HtmlUnitWebElement>> elementsMap = new WeakHashMap<>();

  public static final String INVALIDXPATHERROR = "The xpath expression '%s' cannot be evaluated";
  public static final String INVALIDSELECTIONERROR =
      "The xpath expression '%s' selected an object of type '%s' instead of a WebElement";

  public static final String BROWSER_LANGUAGE_CAPABILITY = "browserLanguage";
  public static final String DOWNLOAD_IMAGES_CAPABILITY = "downloadImages";
  public static final String JAVASCRIPT_ENABLED = "javascriptEnabled";

  /** The Lock for the {@link #mainCondition}, which waits at the end of {@link #runAsync(Runnable)}
   * till either and alert is triggered, or {@link Runnable} finishes. */
  private Lock conditionLock = new ReentrantLock();
  private Condition mainCondition = conditionLock.newCondition();
  private boolean runAsyncRunning;
  private RuntimeException exception;
  private Executor executor = Executors.newCachedThreadPool();

  /**
   * Constructs a new instance with JavaScript disabled,
   * and the {@link BrowserVersion#getDefault() default} BrowserVersion.
   */
  public HtmlUnitDriver() {
    this(false);
  }

  /**
   * Constructs a new instance, specify JavaScript support
   * and using the {@link BrowserVersion#getDefault() default} BrowserVersion.
   *
   * @param enableJavascript whether to enable JavaScript support or not
   */
  public HtmlUnitDriver(boolean enableJavascript) {
    this(BrowserVersion.getDefault(), enableJavascript);
  }

  /**
   * Constructs a new instance with the specified {@link BrowserVersion} and the JavaScript support.
   *
   * @param version the browser version to use
   * @param enableJavascript whether to enable JavaScript support or not
   */
  public HtmlUnitDriver(BrowserVersion version, boolean enableJavascript) {
    this(version);
    setJavascriptEnabled(enableJavascript);
  }

  /**
   * Constructs a new instance with the specified {@link BrowserVersion}.
   *
   * @param version the browser version to use
   */
  public HtmlUnitDriver(BrowserVersion version) {
    webClient = createWebClient(version);
    alert = new HtmlUnitAlert(this);
    currentWindow = webClient.getCurrentWindow();
    initialWindowDimension = new Dimension(currentWindow.getOuterWidth(), currentWindow.getOuterHeight());
    unexpectedAlertBehaviour = UnexpectedAlertBehaviour.DISMISS_AND_NOTIFY;

    webClient.addWebWindowListener(new WebWindowListener() {
      @Override
      public void webWindowOpened(WebWindowEvent webWindowEvent) {
        // Ignore
      }

      @Override
      public void webWindowContentChanged(WebWindowEvent event) {
        elementsMap.remove(event.getOldPage());
        if (event.getWebWindow() != currentWindow) {
          return;
        }

        // Do we need to pick some new default content?
        switchToDefaultContentOfWindow(currentWindow);
      }

      @Override
      public void webWindowClosed(WebWindowEvent event) {
        elementsMap.remove(event.getOldPage());
        // Check if the event window refers to us or one of our parent windows
        // setup the currentWindow appropriately if necessary
        WebWindow curr = currentWindow;
        do {
          // Instance equality is okay in this case
          if (curr == event.getWebWindow()) {
            currentWindow = currentWindow.getTopWindow();
            return;
          }
          curr = curr.getParentWindow();
        } while (curr != currentWindow.getTopWindow());
      }
    });

    // Now put us on the home page, like a real browser
    get(webClient.getOptions().getHomePage());
    gotPage = false;
    resetKeyboardAndMouseState();
  }

  /**
   * The browserName is {@link BrowserType#HTMLUNIT} "htmlunit" and the browserVersion
   * denotes the required browser AND its version.
   * For example "chrome" for Chrome, "firefox-45" for Firefox 45
   * or "internet explorer" for IE.
   *
   * @param capabilities desired capabilities requested for the htmlunit driver session
   */
  public HtmlUnitDriver(Capabilities capabilities) {
    this(determineBrowserVersion(capabilities));

    setJavascriptEnabled(capabilities.getCapability(JAVASCRIPT_ENABLED) == null
        || capabilities.is(JAVASCRIPT_ENABLED));

    setProxySettings(Proxy.extractFrom(capabilities));

    setDownloadImages(capabilities.is(DOWNLOAD_IMAGES_CAPABILITY));
    
    unexpectedAlertBehaviour = (UnexpectedAlertBehaviour) capabilities.getCapability(UNEXPECTED_ALERT_BEHAVIOUR);
    if (unexpectedAlertBehaviour == null) {
      unexpectedAlertBehaviour = UnexpectedAlertBehaviour.DISMISS_AND_NOTIFY;
    }
    
    Boolean acceptSslCerts = (Boolean) capabilities.getCapability(ACCEPT_SSL_CERTS);
    if (acceptSslCerts == null) {
      acceptSslCerts = true;
    }
    setAcceptSslCertificates(acceptSslCerts);

    String pageLoadStrategyString = (String) capabilities.getCapability(PAGE_LOAD_STRATEGY);
    if ("none".equals(pageLoadStrategyString)) {
      pageLoadStrategy = PageLoadStrategy.NONE;
    }
    else if ("eager".equals(pageLoadStrategyString)) {
      pageLoadStrategy = PageLoadStrategy.EAGER;
    }
  }

  public HtmlUnitDriver(Capabilities desiredCapabilities, Capabilities requiredCapabilities) {
    this(new DesiredCapabilities(desiredCapabilities, requiredCapabilities));
  }

  static BrowserVersion determineBrowserVersion(Capabilities capabilities) {
    if (!BrowserType.HTMLUNIT.equals(capabilities.getBrowserName())) {
      throw new IllegalArgumentException("Browser name must be " + BrowserType.HTMLUNIT);
    }
    String browserName;
    String browserVersion;

    String rawVersion = capabilities.getVersion();
    String[] splitVersion = rawVersion == null ? new String[0] : rawVersion.split("-");
    if (splitVersion.length > 1) {
      browserName = splitVersion[0];
      browserVersion = splitVersion[1];
    } else {
      browserName = capabilities.getVersion();
      browserVersion = null;
    }

    BrowserVersion browserVersionObject;
    switch (browserName) {
      case BrowserType.CHROME:
        browserVersionObject = BrowserVersion.CHROME;
        break;

      case BrowserType.IE:
        browserVersionObject = BrowserVersion.INTERNET_EXPLORER;
        break;

      case BrowserType.EDGE:
          browserVersionObject = BrowserVersion.EDGE;
          break;

      case BrowserType.FIREFOX:
        try {
          int version = Integer.parseInt(browserVersion);
          switch (version) {
            case 52:
              browserVersionObject = BrowserVersion.FIREFOX_52;
              break;

            default:
              browserVersionObject = BrowserVersion.FIREFOX_60;
          }
        } catch (NumberFormatException e) {
            browserVersionObject = BrowserVersion.FIREFOX_60;
        }
        break;

      default:
        browserVersionObject = BrowserVersion.getDefault();
        break;
    }

    Object rawLanguage = capabilities.getCapability(BROWSER_LANGUAGE_CAPABILITY);
    if (rawLanguage instanceof String) {
      browserVersionObject = new BrowserVersionBuilder(browserVersionObject)
          .setBrowserLanguage((String) rawLanguage).build();
    }

    return browserVersionObject;
  }

  private WebClient createWebClient(BrowserVersion version) {
    WebClient client = newWebClient(version);
    WebClientOptions options = client.getOptions();
    options.setHomePage(WebClient.URL_ABOUT_BLANK.toString());
    options.setThrowExceptionOnFailingStatusCode(false);
    options.setPrintContentOnFailingStatusCode(false);
    options.setJavaScriptEnabled(enableJavascript);
    options.setRedirectEnabled(true);
    options.setUseInsecureSSL(true);

    // Ensure that we've set the proxy if necessary
    if (proxyConfig != null) {
      options.setProxyConfig(proxyConfig);
    }

    client.setRefreshHandler(new WaitingRefreshHandler());

    return modifyWebClient(client);
  }

  /**
   * @return to process or not to proceed
   */
  boolean isProcessAlert() {
    if (asyncScriptExecutor != null) {
      String text = alert.getText();
      alert.dismiss();
      asyncScriptExecutor.alertTriggered(text);
      return false;
    }
    conditionLock.lock();
    mainCondition.signal();
    conditionLock.unlock();
    return true;
  }

  void runAsync(Runnable r) {
    boolean loadStrategyWait = pageLoadStrategy != PageLoadStrategy.NONE;

    if (loadStrategyWait) {
      while (runAsyncRunning) {
        try {
          Thread.sleep(10);
        } catch (InterruptedException e) {
          throw new RuntimeException(e);
        }
      }
      conditionLock.lock();
      runAsyncRunning = true;
    }
    
    exception = null;
    Runnable wrapped = () -> {
      try {
        r.run();
      }
      catch (RuntimeException e) {
        exception = e;
      }
      finally {
        conditionLock.lock();
        mainCondition.signal();
        runAsyncRunning = false;
        conditionLock.unlock();
      }
    };
    executor.execute(wrapped);

    if (loadStrategyWait) {
      mainCondition.awaitUninterruptibly();
      conditionLock.unlock();
    }

    if (exception != null) {
      throw exception;
    }
  }

  void click(DomElement element, boolean directClick) {
    runAsync(() -> mouse.click(element, directClick));
  }

  void doubleClick(DomElement element) {
    runAsync(() -> mouse.doubleClick(element));
  }

  void mouseUp(DomElement element) {
    runAsync(() -> mouse.mouseUp(element));
  }

  void mouseMove(DomElement element) {
    runAsync(() -> mouse.mouseMove(element));
  }

  void mouseDown(DomElement element) {
    runAsync(() -> mouse.mouseDown(element));
  }

  void submit(HtmlUnitWebElement element) {
    runAsync(() -> element.submitImpl());
  }

  void sendKeys(HtmlUnitWebElement element, CharSequence... value) {
    runAsync(() -> keyboard.sendKeys(element, true, value));
  }

  /**
   * Get the simulated {@code BrowserVersion}.
   * @return the used {@code BrowserVersion}
   */
  public BrowserVersion getBrowserVersion() {
    return webClient.getBrowserVersion();
  }

  /**
   * Create the underlying WebClient, but don't set any fields on it.
   *
   * @param version Which browser to emulate
   * @return a new instance of WebClient.
   */
  protected WebClient newWebClient(BrowserVersion version) {
    return new WebClient(version);
  }

  /**
   * Child classes can override this method to customize the WebClient that the HtmlUnit driver
   * uses.
   *
   * @param client The client to modify
   * @return The modified client
   */
  protected WebClient modifyWebClient(WebClient client) {
    // Does nothing here to be overridden.
    return client;
  }

  /**
   * Set proxy for WebClient using Proxy.
   *
   * @param proxy The proxy preferences.
   */
  public void setProxySettings(Proxy proxy) {
    if (proxy == null || proxy.getProxyType() == Proxy.ProxyType.UNSPECIFIED) {
      return;
    }

    switch (proxy.getProxyType()) {
      case MANUAL:
        List<String> noProxyHosts = new ArrayList<>();
        String noProxy = proxy.getNoProxy();
        if (noProxy != null && !noProxy.isEmpty()) {
          String[] hosts = noProxy.split(",");
          for (String host : hosts) {
            if (host.trim().length() > 0) {
              noProxyHosts.add(host.trim());
            }
          }
        }

        String httpProxy = proxy.getHttpProxy();
        if (httpProxy != null && !httpProxy.isEmpty()) {
          String host = httpProxy;
          int port = 0;

          int index = httpProxy.indexOf(":");
          if (index != -1) {
            host = httpProxy.substring(0, index);
            port = Integer.parseInt(httpProxy.substring(index + 1));
          }

          setHTTPProxy(host, port, noProxyHosts);
        }

        String socksProxy = proxy.getSocksProxy();
        if (socksProxy != null && !socksProxy.isEmpty()) {
          String host = socksProxy;
          int port = 0;

          int index = socksProxy.indexOf(":");
          if (index != -1) {
            host = socksProxy.substring(0, index);
            port = Integer.parseInt(socksProxy.substring(index + 1));
          }

          setSocksProxy(host, port, noProxyHosts);
        }

        // sslProxy is not supported/implemented
        // ftpProxy is not supported/implemented

        break;

      case PAC:
        String pac = proxy.getProxyAutoconfigUrl();
        if (pac != null && !pac.isEmpty()) {
          setAutoProxy(pac);
        }
        break;

      default:
          break;
    }
  }

  /**
   * Sets HTTP proxy for WebClient
   *
   * @param host The hostname of HTTP proxy
   * @param port The port of HTTP proxy, 0 means HTTP proxy w/o port
   */
  public void setProxy(String host, int port) {
    setHTTPProxy(host, port, null);
  }

  /**
   * Sets HTTP proxy for WebClient with bypass proxy hosts
   *
   * @param host The hostname of HTTP proxy
   * @param port The port of HTTP proxy, 0 means HTTP proxy w/o port
   * @param noProxyHosts The list of hosts which need to bypass HTTP proxy
   */
  public void setHTTPProxy(String host, int port, List<String> noProxyHosts) {
    proxyConfig = new ProxyConfig();
    proxyConfig.setProxyHost(host);
    proxyConfig.setProxyPort(port);
    if (noProxyHosts != null && noProxyHosts.size() > 0) {
      for (String noProxyHost : noProxyHosts) {
        proxyConfig.addHostsToProxyBypass(noProxyHost);
      }
    }
    getWebClient().getOptions().setProxyConfig(proxyConfig);
  }

  /**
   * Sets SOCKS proxy for WebClient
   *
   * @param host The hostname of SOCKS proxy
   * @param port The port of SOCKS proxy, 0 means HTTP proxy w/o port
   */
  public void setSocksProxy(String host, int port) {
    setSocksProxy(host, port, null);
  }

  /**
   * Sets SOCKS proxy for WebClient with bypass proxy hosts
   *
   * @param host The hostname of SOCKS proxy
   * @param port The port of SOCKS proxy, 0 means HTTP proxy w/o port
   * @param noProxyHosts The list of hosts which need to bypass SOCKS proxy
   */
  public void setSocksProxy(String host, int port, List<String> noProxyHosts) {
    proxyConfig = new ProxyConfig();
    proxyConfig.setProxyHost(host);
    proxyConfig.setProxyPort(port);
    proxyConfig.setSocksProxy(true);
    if (noProxyHosts != null && noProxyHosts.size() > 0) {
      for (String noProxyHost : noProxyHosts) {
        proxyConfig.addHostsToProxyBypass(noProxyHost);
      }
    }
    getWebClient().getOptions().setProxyConfig(proxyConfig);
  }

  /**
   * Sets the {@link Executor} to be used for submitting async tasks to
   * @param executor the {@link Executor} to use
   */
  public void setExecutor(Executor executor) {
    if (executor == null) {
      throw new IllegalArgumentException("executor cannot be null");
    }
    this.executor = executor;
  }

  /**
{'fraction_non_alphanumeric': 0.09266669664133807, 'fraction_numerical': 0.002158176340991862, 'mean_word_length': 3.715179139283443, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3871205', 'n_tokens_mistral': 13033, 'n_tokens_neox': 12628, 'n_words': 3034}