<commit_before>package driver

import (
	"os"
	"time"

	log "github.com/adl32x/sailfoot/log"
	"github.com/logrusorgru/aurora"

	"github.com/sclevine/agouti"
)

type WebDriver struct {
	driver *agouti.WebDriver
	page   *agouti.Page
	pages  []*agouti.Page
}

func (w *WebDriver) Start() {
	// w.driver = driver
	w.page, _ = w.driver.NewPage()
	w.pages = append(w.pages, w.page)
	w.page.SetImplicitWait(10000)
}

func (w *WebDriver) Init(driverType *string) {
	if *driverType == "chromeheadless" {
		w.driver = agouti.ChromeDriver(
			agouti.ChromeOptions("args", []string{"--headless", "--disable-gpu"}),
		)
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start with chromedriver: ", err)
		}
	} else if *driverType == "chrome" {
		w.driver = agouti.ChromeDriver()
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start chrome: ", err)
		}
	} else if *driverType == "phantomjs" {
		w.driver = agouti.PhantomJS()
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start phantomjs: ", err)
		}
	} else if *driverType == "firefox" {
		w.driver = agouti.GeckoDriver()
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start geckodriver: ", err)
		}
	} else {
		log.Error("Unsupported driver %s", *driverType)<commit_msg>Removes the chrome info-bars.<commit_after>package driver

import (
	"os"
	"time"

	log "github.com/adl32x/sailfoot/log"
	"github.com/logrusorgru/aurora"

	"github.com/sclevine/agouti"
)

type WebDriver struct {
	driver *agouti.WebDriver
	page   *agouti.Page
	pages  []*agouti.Page
}

func (w *WebDriver) Start() {
	// w.driver = driver
	w.page, _ = w.driver.NewPage()
	w.pages = append(w.pages, w.page)
	w.page.SetImplicitWait(10000)
}

func (w *WebDriver) Init(driverType *string) {
	if *driverType == "chromeheadless" {
		w.driver = agouti.ChromeDriver(
			agouti.ChromeOptions("args", []string{"--headless", "--disable-gpu"}),
		)
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start with chromedriver: ", err)
		}
	} else if *driverType == "chrome" {
		w.driver = agouti.ChromeDriver(
			agouti.ChromeOptions("args", []string{"--disable-infobars"}),
		)
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start chrome: ", err)
		}
	} else if *driverType == "phantomjs" {
		w.driver = agouti.PhantomJS()
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start phantomjs: ", err)
		}
	} else if *driverType == "firefox" {
		w.driver = agouti.GeckoDriver()
		if err := w.driver.Start(); err != nil {
			log.Error("Failed to start geckodriver: ", err)
		}
	} else {
		log.Error("Unsupported driver %s", *driverType)
{'fraction_non_alphanumeric': 0.1733232856066315, 'fraction_numerical': 0.0052750565184626974, 'mean_word_length': 3.6578947368421053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6470314', 'n_tokens_mistral': 1095, 'n_tokens_neox': 1014, 'n_words': 218}