<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/gobs/args"
	"github.com/gobs/pretty"
	"github.com/gobs/simplejson"
	"github.com/raff/godet"
)

func runCommand(commandString string) error {
	parts := args.GetArgs(commandString)
	cmd := exec.Command(parts[0], parts[1:]...)
	return cmd.Start()
}

func limit(s string, l int) string {
	if len(s) > l {
		return s[:l] + "..."
	}
	return s
}

func documentNode(remote *godet.RemoteDebugger, verbose bool) int {
	res, err := remote.GetDocument()
	if err != nil {
		log.Fatal("error getting document: ", err)
	}

	if verbose {
		pretty.PrettyPrint(res)
	}

	doc := simplejson.AsJson(res)
	return doc.GetPath("root", "nodeId").MustInt(-1)
}

func main() {
	var chromeapp string

	switch runtime.GOOS {
	case "darwin":
		for _, c := range []string{
			"/Applications/Google Chrome Canary.app",
			"/Applications/Google Chrome.app",
		} {
			// MacOS apps are actually folders
			if info, err := os.Stat(c); err == nil && info.IsDir() {
				chromeapp = fmt.Sprintf("open %q --args", c)
				break
			}
		}

	case "linux":
		for _, c := range []string{
			"headless_shell",
			"chromium",
			"google-chrome-beta",
			"google-chrome-unstable",
			"google-chrome-stable"} {
			if _, err := exec.LookPath(c); err == nil {
				chromeapp = c
				break
			}
		}

	case "windows":
	}

	if chromeapp != "" {
		if chromeapp == "headless_shell" {
			chromeapp += " --no-sandbox"
		} else {
			chromeapp += " --headless"
		}

		chromeapp += " --remote-debugging-port=9222 --disable-extensions --disable-gpu about:blank"
	}

	cmd := flag.String("cmd", chromeapp, "command to execute to start the browser")
	port := flag.String("port", "localhost:9222", "Chrome remote debugger port")
	verbose := flag.Bool("verbose", false, "verbose logging")
	version := flag.Bool("version", false, "display remote devtools version")
	listtabs := flag.Bool("tabs", false, "show list of open tabs")
	seltab := flag.Int("tab", 0, "select specified tab if available")
	newtab := flag.Bool("new", false, "always open a new tab")
	filter := flag.String("filter", "page", "filter tab list")
	domains := flag.Bool("domains", false, "show list of available domains")
	requests := flag.Bool("requests", false, "show request notifications")
	responses := flag.Bool("responses", false, "show response notifications")
	allEvents := flag.Bool("all-events", false, "enable all events")
	logev := flag.Bool("log", false, "show log/console messages")
	query := flag.String("query", "", "query against current document")
	eval := flag.String("eval", "", "evaluate expression")
	screenshot := flag.Bool("screenshot", false, "take a screenshot")
	pdf := flag.Bool("pdf", false, "save current page as PDF")
	control := flag.String("control", "", "control navigation (proceed,cancel,cancelIgnore)")
	block := flag.String("block", "", "block specified URLs or pattenrs. Use '|' as separator")
	html := flag.Bool("html", false, "get outer HTML for current page")
	setHtml := flag.String("set-html", "", "set outer HTML for current page")
	wait := flag.Bool("wait", false, "wait for more events")
	flag.Parse()

	if *cmd != "" {
		if err := runCommand(*cmd); err != nil {
			log.Println("cannot start browser", err)
		}
	}

	var remote *godet.RemoteDebugger
	var err error

	for i := 0; i < 10; i++ {
		if i > 0 {
			time.Sleep(500 * time.Millisecond)
		}

		remote, err = godet.Connect(*port, *verbose)
		if err == nil {
			break
		}

		log.Println("connect", err)
	}

	if err != nil {
		log.Fatal("cannot connect to browser")
	}

	defer remote.Close()

	done := make(chan bool)
	should_wait := true

	v, err := remote.Version()
	if err != nil {
		log.Fatal("cannot get version: ", err)
	}

	if *version {
		pretty.PrettyPrint(v)
	} else {
		log.Println("connected to", v.Browser, "protocol version", v.ProtocolVersion)
	}

	if *listtabs {
		tabs, err := remote.TabList(*filter)
		if err != nil {
			log.Fatal("cannot get list of tabs: ", err)
		}

		pretty.PrettyPrint(tabs)
		should_wait = false
	}

	if *domains {
		d, err := remote.GetDomains()
		if err != nil {
			log.Fatal("cannot get domains: ", err)
		}

		pretty.PrettyPrint(d)
		should_wait = false
	}

	remote.CallbackEvent(godet.EventClosed, func(params godet.Params) {
		log.Println("RemoteDebugger connection terminated.")
		done <- true
	})

	if *requests {
		remote.CallbackEvent("Network.requestWillBeSent", func(params godet.Params) {
			log.Println("requestWillBeSent",<commit_msg>There are times when I don't want to run in headless mode ( --headless=false ). Also added --protocol, that dumps the browser protocol schema<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"time"

	"github.com/gobs/args"
	"github.com/gobs/pretty"
	"github.com/gobs/simplejson"
	"github.com/raff/godet"
)

func runCommand(commandString string) error {
	parts := args.GetArgs(commandString)
	cmd := exec.Command(parts[0], parts[1:]...)
	return cmd.Start()
}

func limit(s string, l int) string {
	if len(s) > l {
		return s[:l] + "..."
	}
	return s
}

func documentNode(remote *godet.RemoteDebugger, verbose bool) int {
	res, err := remote.GetDocument()
	if err != nil {
		log.Fatal("error getting document: ", err)
	}

	if verbose {
		pretty.PrettyPrint(res)
	}

	doc := simplejson.AsJson(res)
	return doc.GetPath("root", "nodeId").MustInt(-1)
}

func main() {
	var chromeapp string

	switch runtime.GOOS {
	case "darwin":
		for _, c := range []string{
			"/Applications/Google Chrome Canary.app",
			"/Applications/Google Chrome.app",
		} {
			// MacOS apps are actually folders
			if info, err := os.Stat(c); err == nil && info.IsDir() {
				chromeapp = fmt.Sprintf("open %q --args", c)
				break
			}
		}

	case "linux":
		for _, c := range []string{
			"headless_shell",
			"chromium",
			"google-chrome-beta",
			"google-chrome-unstable",
			"google-chrome-stable"} {
			if _, err := exec.LookPath(c); err == nil {
				chromeapp = c
				break
			}
		}

	case "windows":
	}

	if chromeapp != "" {
		if chromeapp == "headless_shell" {
			chromeapp += " --no-sandbox"
		} else {
			chromeapp += " --headless"
		}

		chromeapp += " --remote-debugging-port=9222 --disable-extensions --disable-gpu about:blank"
	}

	cmd := flag.String("cmd", chromeapp, "command to execute to start the browser")
	headless := flag.Bool("headless", true, "headless mode")
	port := flag.String("port", "localhost:9222", "Chrome remote debugger port")
	verbose := flag.Bool("verbose", false, "verbose logging")
	version := flag.Bool("version", false, "display remote devtools version")
	protocol := flag.Bool("protocol", false, "display the DevTools protocol")
	listtabs := flag.Bool("tabs", false, "show list of open tabs")
	seltab := flag.Int("tab", 0, "select specified tab if available")
	newtab := flag.Bool("new", false, "always open a new tab")
	filter := flag.String("filter", "page", "filter tab list")
	domains := flag.Bool("domains", false, "show list of available domains")
	requests := flag.Bool("requests", false, "show request notifications")
	responses := flag.Bool("responses", false, "show response notifications")
	allEvents := flag.Bool("all-events", false, "enable all events")
	logev := flag.Bool("log", false, "show log/console messages")
	query := flag.String("query", "", "query against current document")
	eval := flag.String("eval", "", "evaluate expression")
	screenshot := flag.Bool("screenshot", false, "take a screenshot")
	pdf := flag.Bool("pdf", false, "save current page as PDF")
	control := flag.String("control", "", "control navigation (proceed,cancel,cancelIgnore)")
	block := flag.String("block", "", "block specified URLs or pattenrs. Use '|' as separator")
	html := flag.Bool("html", false, "get outer HTML for current page")
	setHtml := flag.String("set-html", "", "set outer HTML for current page")
	wait := flag.Bool("wait", false, "wait for more events")
	flag.Parse()

	if *cmd != "" {
		if !*headless {
			*cmd = strings.Replace(*cmd, " --headless ", " ", -1)
		}

		if err := runCommand(*cmd); err != nil {
			log.Println("cannot start browser", err)
		}
	}

	var remote *godet.RemoteDebugger
	var err error

	for i := 0; i < 10; i++ {
		if i > 0 {
			time.Sleep(500 * time.Millisecond)
		}

		remote, err = godet.Connect(*port, *verbose)
		if err == nil {
			break
		}

		log.Println("connect", err)
	}

	if err != nil {
		log.Fatal("cannot connect to browser")
	}

	defer remote.Close()

	done := make(chan bool)
	should_wait := true

	v, err := remote.Version()
	if err != nil {
		log.Fatal("cannot get version: ", err)
	}

	if *version {
		pretty.PrettyPrint(v)
	} else {
		log.Println("connected to", v.Browser, "protocol version", v.ProtocolVersion)
	}

	if *protocol {
		p, err := remote.Protocol()
		if err != nil {
			log.Fatal("cannot get protocol: ", err)
		}

		pretty.PrettyPrint(p)
		should_wait = false
	}

	if *listtabs {
		tabs, err := remote.TabList(*filter)
		if err != nil {
			log.Fatal("cannot get list of tabs: ", err)
		}

		pretty.PrettyPrint(tabs)
		should_wait = false
	}

	if *domains {
		d, err := remote.GetDomains()
		if err != nil {
			log.Fatal("cannot get domains: ", err)
		}

		pretty.PrettyPrint(d)
		should_wait = false
	}

	remote.CallbackEvent(godet.EventClosed, func(params godet.Params) {
		log.Println("RemoteDebugger connection terminated.")
		done <- true
	})

	if *requests {
		remote.CallbackEvent("Network.requestWillBeSent", func(params godet.Params) {
			log.Println("requestWillBeSent",
{'fraction_non_alphanumeric': 0.16566929133858269, 'fraction_numerical': 0.004094488188976378, 'mean_word_length': 3.7941620533467537, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1153584', 'n_tokens_mistral': 3558, 'n_tokens_neox': 3194, 'n_words': 869}