<commit_before># -*- coding: utf-8 -*-

import fcntl
import glob
import json
import os
import random
import shutil
import socket
import subprocess
import sys
import tempfile
import time

TEST_DIR = os.path.normpath(os.path.dirname(os.path.realpath(os.path.join(__file__, ".."))))


def browser_path(headless=True):
    """Return path to CDP browser.

    Support the following locations:
     - /usr/lib*/chromium-browser/headless_shell (chromium-headless RPM), if
       headless is true
     - "chromium-browser" in $PATH (distro package)
     - node_modules/chromium/lib/chromium/chrome-linux/chrome (npm install chromium)

    Exit with an error if none is found.
    """
    if headless:
        g = glob.glob("/usr/lib*/chromium-browser/headless_shell")
        if g:
            return g[0]
    try:
        return subprocess.check_output(["which", "chromium-browser"]).strip()
    except subprocess.CalledProcessError:
        p = os.path.join(os.path.dirname(TEST_DIR), "node_modules/chromium/lib/chromium/chrome-linux/chrome")
        if os.access(p, os.X_OK):
            return p
        raise


def jsquote(str):
    return json.dumps(str)


class CDP:
    def __init__(self, lang=None, headless=True, verbose=False, trace=False, inject_helpers=[]):
        self.lang = lang
        self.timeout = 60
        self.valid = False
        self.headless = headless
        self.verbose = verbose
        self.trace = trace
        self.inject_helpers = inject_helpers
        self._driver = None
        self._browser = None
        self._browser_home = None
        self._cdp_port_lockfile = None

    def invoke(self, fn, **kwargs):
        """Call a particular CDP method such as Runtime.evaluate

        Use command() for arbitrary JS code.
        """
        trace = self.trace and not kwargs.get("no_trace", False)
        try:
            del kwargs["no_trace"]
        except KeyError:
            pass

        cmd = fn + "(" + json.dumps(kwargs) + ")"

        # frame support for Runtime.evaluate(): map frame name to
        # executionContextId and insert into argument object; this must not be quoted
        # see "Frame tracking" in cdp-driver.js for how this works
        if fn == 'Runtime.evaluate' and self.cur_frame:
            cmd = "%s, contextId: getFrameExecId(%s)%s" % (cmd[:-2], jsquote(self.cur_frame), cmd[-2:])

        if trace:
            print("-> " + kwargs.get('trace', cmd))

        # avoid having to write the "client." prefix everywhere
        cmd = "client." + cmd
        res = self.command(cmd)
        if trace:
            if "result" in res:
                print("<- " + repr(res["result"]))
            else:
                print("<- " + repr(res))
        return res

    def command(self, cmd):
        if not self._driver:
            self.start()
        self._driver.stdin.write(cmd + "\n")
        line = self._driver.stdout.readline()
        if not line:
            self.kill()
            raise RuntimeError("CDP broken")
        try:
            res = json.loads(line)
        except ValueError:
            print(line.strip())
            raise

        if "error" in res:
            if self.trace:
                print("<- raise " + res["error"])
            raise RuntimeError(res["error"])
        return res["result"]

    def claim_port(self, port):
        f = None
        try:
            f = open(os.path.join(tempfile.gettempdir(), ".cdp-%i.lock" % port), "w")
            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
            self._cdp_port_lockfile = f
            return True
        except (IOError, OSError):
            if f:
                f.close()
            return False

    def find_cdp_port(self):
        """Find an unused port and claim it through lock file"""

        for retry in range(100):
            # don't use the default CDP port 9222 to avoid interfering with running browsers
            port = random.randint (9223, 10222)
            if self.claim_port(port):
                return port
        else:
            raise RuntimeError("unable to find free port")

    def start(self):
        environ = os.environ.copy()
        if self.lang:
            environ["LC_ALL"] = self.lang
        self.cur_frame = None

        # allow attaching to external browser
        cdp_port = None
        if "TEST_CDP_PORT" in os.environ:
            p = int(os.environ["TEST_CDP_PORT"])
            if self.claim_port(p):
                # can fail when a test starts multiple browsers; only show the first one
                cdp_port = p

        if not cdp_port:
            # start browser on a new port
            cdp_port = self.find_cdp_port()
            self._browser_home = tempfile.mkdtemp()
            environ = os.environ.copy()
            environ["HOME"] = self._browser_home
            environ["LC_ALL"] = "C.utf8"
            # this might be set for the tests themselves, but we must isolate caching between tests
            try:
                del environ["XDG_CACHE_HOME"]
            except KeyError:
                pass

            exe = browser_path(self.headless)

            if self.headless:
                argv = [exe,  "--headless"]
            else:
                argv = [os.path.join(TEST_DIR, "common/xvfb-wrapper"), exe]

            # sandboxing does not work in Docker container
            self._browser = subprocess.Popen(
                argv + ["--disable-gpu", "--no-sandbox", "--disable-setuid-sandbox",
                    "--disable-namespace-sandbox", "--disable-seccomp-filter-sandbox",
                    "--disable-sandbox-denial-logging",
                    "--remote-debugging-port=%i" % cdp_port, "about:blank"],
                env=environ, close_fds=True)
            if self.verbose:
                sys.stderr.write("Started %s (pid %i) on port %i\n" % (exe, self._browser.pid, cdp_port))

        # wait for CDP to be up
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        for retry in range(300):
            try:
                s.connect(('127.0.0.1', cdp_port))
                break
            except socket.error:<commit_msg>test: Increase Chromium window size<commit_after># -*- coding: utf-8 -*-

import fcntl
import glob
import json
import os
import random
import shutil
import socket
import subprocess
import sys
import tempfile
import time

TEST_DIR = os.path.normpath(os.path.dirname(os.path.realpath(os.path.join(__file__, ".."))))


def browser_path(headless=True):
    """Return path to CDP browser.

    Support the following locations:
     - /usr/lib*/chromium-browser/headless_shell (chromium-headless RPM), if
       headless is true
     - "chromium-browser" in $PATH (distro package)
     - node_modules/chromium/lib/chromium/chrome-linux/chrome (npm install chromium)

    Exit with an error if none is found.
    """
    if headless:
        g = glob.glob("/usr/lib*/chromium-browser/headless_shell")
        if g:
            return g[0]
    try:
        return subprocess.check_output(["which", "chromium-browser"]).strip()
    except subprocess.CalledProcessError:
        p = os.path.join(os.path.dirname(TEST_DIR), "node_modules/chromium/lib/chromium/chrome-linux/chrome")
        if os.access(p, os.X_OK):
            return p
        raise


def jsquote(str):
    return json.dumps(str)


class CDP:
    def __init__(self, lang=None, headless=True, verbose=False, trace=False, inject_helpers=[]):
        self.lang = lang
        self.timeout = 60
        self.valid = False
        self.headless = headless
        self.verbose = verbose
        self.trace = trace
        self.inject_helpers = inject_helpers
        self._driver = None
        self._browser = None
        self._browser_home = None
        self._cdp_port_lockfile = None

    def invoke(self, fn, **kwargs):
        """Call a particular CDP method such as Runtime.evaluate

        Use command() for arbitrary JS code.
        """
        trace = self.trace and not kwargs.get("no_trace", False)
        try:
            del kwargs["no_trace"]
        except KeyError:
            pass

        cmd = fn + "(" + json.dumps(kwargs) + ")"

        # frame support for Runtime.evaluate(): map frame name to
        # executionContextId and insert into argument object; this must not be quoted
        # see "Frame tracking" in cdp-driver.js for how this works
        if fn == 'Runtime.evaluate' and self.cur_frame:
            cmd = "%s, contextId: getFrameExecId(%s)%s" % (cmd[:-2], jsquote(self.cur_frame), cmd[-2:])

        if trace:
            print("-> " + kwargs.get('trace', cmd))

        # avoid having to write the "client." prefix everywhere
        cmd = "client." + cmd
        res = self.command(cmd)
        if trace:
            if "result" in res:
                print("<- " + repr(res["result"]))
            else:
                print("<- " + repr(res))
        return res

    def command(self, cmd):
        if not self._driver:
            self.start()
        self._driver.stdin.write(cmd + "\n")
        line = self._driver.stdout.readline()
        if not line:
            self.kill()
            raise RuntimeError("CDP broken")
        try:
            res = json.loads(line)
        except ValueError:
            print(line.strip())
            raise

        if "error" in res:
            if self.trace:
                print("<- raise " + res["error"])
            raise RuntimeError(res["error"])
        return res["result"]

    def claim_port(self, port):
        f = None
        try:
            f = open(os.path.join(tempfile.gettempdir(), ".cdp-%i.lock" % port), "w")
            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
            self._cdp_port_lockfile = f
            return True
        except (IOError, OSError):
            if f:
                f.close()
            return False

    def find_cdp_port(self):
        """Find an unused port and claim it through lock file"""

        for retry in range(100):
            # don't use the default CDP port 9222 to avoid interfering with running browsers
            port = random.randint (9223, 10222)
            if self.claim_port(port):
                return port
        else:
            raise RuntimeError("unable to find free port")

    def start(self):
        environ = os.environ.copy()
        if self.lang:
            environ["LC_ALL"] = self.lang
        self.cur_frame = None

        # allow attaching to external browser
        cdp_port = None
        if "TEST_CDP_PORT" in os.environ:
            p = int(os.environ["TEST_CDP_PORT"])
            if self.claim_port(p):
                # can fail when a test starts multiple browsers; only show the first one
                cdp_port = p

        if not cdp_port:
            # start browser on a new port
            cdp_port = self.find_cdp_port()
            self._browser_home = tempfile.mkdtemp()
            environ = os.environ.copy()
            environ["HOME"] = self._browser_home
            environ["LC_ALL"] = "C.utf8"
            # this might be set for the tests themselves, but we must isolate caching between tests
            try:
                del environ["XDG_CACHE_HOME"]
            except KeyError:
                pass

            exe = browser_path(self.headless)

            if self.headless:
                argv = [exe,  "--headless"]
            else:
                argv = [os.path.join(TEST_DIR, "common/xvfb-wrapper"), exe]

            # sandboxing does not work in Docker container
            self._browser = subprocess.Popen(
                argv + ["--disable-gpu", "--no-sandbox", "--disable-setuid-sandbox",
                    "--disable-namespace-sandbox", "--disable-seccomp-filter-sandbox",
                    "--disable-sandbox-denial-logging", "--window-size=1280x1200",
                    "--remote-debugging-port=%i" % cdp_port, "about:blank"],
                env=environ, close_fds=True)
            if self.verbose:
                sys.stderr.write("Started %s (pid %i) on port %i\n" % (exe, self._browser.pid, cdp_port))

        # wait for CDP to be up
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        for retry in range(300):
            try:
                s.connect(('127.0.0.1', cdp_port))
                break
            except socket.error:
{'fraction_non_alphanumeric': 0.11090332547361574, 'fraction_numerical': 0.0058541344824782505, 'mean_word_length': 1.819161127664451, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5480199', 'n_tokens_mistral': 3696, 'n_tokens_neox': 3528, 'n_words': 956}