<commit_before>package net.ruippeixotog.scalascraper

import java.io.PrintStream

import scala.collection.immutable.SortedMap

import net.ruippeixotog.scalascraper.ExampleMatchers._
import net.ruippeixotog.scalascraper.browser.{ HtmlUnitBrowser, JsoupBrowser }
import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.model.Element
import net.ruippeixotog.scalascraper.scraper.HtmlValidator
import net.ruippeixotog.scalascraper.util.EitherRightBias._
import net.ruippeixotog.scalascraper.util.ProxyUtils

object ExampleMatchers {
  val succ = HtmlValidator(text("head > title"), 1)(_.matches(".*Observador.*"))
  val errs = Seq(
    HtmlValidator(attr("content")("meta[name=viewport]"), 2)(_.matches(".*initial-scale=2\\.0.*")),
    HtmlValidator(elements("meta[name=viewport]"), 3)(_.nonEmpty))
}

object ProxyApp extends App {
  ProxyUtils.setProxy("localhost", 3128)
  val browser = JsoupBrowser()
  val doc = browser.get("http://observador.pt")

  println("=== OBSERVADOR HTTP & HTTPS PROXY ===")

  Thread.sleep(2000)

  ProxyUtils.removeProxy()

  // You should get a [java.net.SocketTimeoutException: connect timed out] if you are behind a proxy
  browser.get("http://observador.pt")<commit_msg>Do not use ProxyUtils in Examples<commit_after>package net.ruippeixotog.scalascraper

import java.io.PrintStream
import java.net.{ InetSocketAddress, Proxy }

import scala.collection.immutable.SortedMap

import net.ruippeixotog.scalascraper.ExampleMatchers._
import net.ruippeixotog.scalascraper.browser.{ HtmlUnitBrowser, JsoupBrowser }
import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.model.Element
import net.ruippeixotog.scalascraper.scraper.HtmlValidator
import net.ruippeixotog.scalascraper.util.EitherRightBias._

object ExampleMatchers {
  val succ = HtmlValidator(text("head > title"), 1)(_.matches(".*Observador.*"))
  val errs = Seq(
    HtmlValidator(attr("content")("meta[name=viewport]"), 2)(_.matches(".*initial-scale=2\\.0.*")),
    HtmlValidator(elements("meta[name=viewport]"), 3)(_.nonEmpty))
}

object ProxyApp extends App {
  val browser = new JsoupBrowser(proxy = new Proxy(Proxy.Type.SOCKS, new InetSocketAddress("localhost", 3128)))
  val doc = browser.get("http://observador.pt")

  println("=== OBSERVADOR HTTP & HTTPS PROXY ===")

  Thread.sleep(2000)

  // You should get a [java.net.SocketTimeoutException: connect timed out] if you are behind a proxy
  browser.get("http://observador.pt")
{'fraction_non_alphanumeric': 0.13310185185185186, 'fraction_numerical': 0.010030864197530864, 'mean_word_length': 5.752604166666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2369325', 'n_tokens_mistral': 937, 'n_tokens_neox': 904, 'n_words': 153}