<commit_before>package me.lorandszakacs.util.html

import java.io.File

import scala.collection.mutable.ListBuffer

import org.jsoup.Jsoup
import org.jsoup.nodes.Element

object HtmlParser {
  def apply(contents: String) = {
    new HtmlParser(contents)
  }

  def apply(file: File) = {

  }

  private final val BetweenQuotesRegex = ".*\"(.*)\".*".r
}

class HtmlParser private (val contents: String) {
  private lazy val document = Jsoup.parse(contents)

  def filterByClass(cls: String): List[String] = {
    val elements = document.getElementsByClass(cls)
    val buff = ListBuffer[Element]();
    val iterator = elements.iterator()
    while (iterator.hasNext()) {
      buff.append(iterator.next())
    }
    val result = buff.map(el => el.html()).toList
    result
  }

  def grabFirstLink(): Option[String] = {
    val links = document.getElementsByAttribute("href")
    val buff = ListBuffer[Element]();
    val iterator = links.iterator()
    while (iterator.hasNext()) {
      buff.append(iterator.next())
    }
    if (buff.length == 0)
      None
    else {
      val firstLink: String = buff(0).toString()
      try {
        val HtmlParser.BetweenQuotesRegex(result) = firstLink
        Some(result)
      } catch {
        case e: MatchError => None
      }
    }
  }
}<commit_msg>Added function that grabs all the links.<commit_after>package me.lorandszakacs.util.html

import java.io.File

import scala.collection.mutable.ListBuffer

import org.jsoup.Jsoup
import org.jsoup.nodes.Element

object HtmlParser {
  def apply(contents: String) = {
    new HtmlParser(contents)
  }

  def apply(file: File) = {

  }

  private final val BetweenQuotesRegex = ".*\"(.*)\".*".r
}

class HtmlParser private (val contents: String) {
  private lazy val document = Jsoup.parse(contents)

  def filterByClass(cls: String): List[String] = {
    val elements = document.getElementsByClass(cls)
    val buff = ListBuffer[Element]();
    val iterator = elements.iterator()
    while (iterator.hasNext()) {
      buff.append(iterator.next())
    }
    val result = buff.map(el => el.html()).toList
    result
  }

  def grabAllLinks(): Option[List[String]] = {
    val links = document.getElementsByAttribute("href")
    val buff = ListBuffer[Element]();
    val iterator = links.iterator()
    while (iterator.hasNext()) {
      buff.append(iterator.next())
    }

    if (buff.length == 0)
      None
    else {
      val result = ListBuffer[String]()
      buff.foreach(htmlLink => {
        try {
          val HtmlParser.BetweenQuotesRegex(link) = htmlLink.toString
          result.append(link)
        } catch {
          case e: MatchError => Unit
        }
      })

      if (result.length == 0)
        None
      else
        Some(result.toList)
    }
  }

  def grabFirstLink(): Option[String] = {
    val links = grabAllLinks
    links match {
      case None => None
      case Some(res) => Some(res.head)
    }
  }
}
{'fraction_non_alphanumeric': 0.1269191402251791, 'fraction_numerical': 0.001364721937905152, 'mean_word_length': 2.758974358974359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7657748', 'n_tokens_mistral': 931, 'n_tokens_neox': 910, 'n_words': 208}