<commit_before>package robots.protocol.inclusion

import java.net.URL

import scala.util.Try
import scala.xml.XML

/**
 * Abstract class that represents a sitemap.
 *
 * @author andrei
 */
sealed abstract class Sitemap {
  /**
   * The location (URL) of the sitemap.
   */
  val location: URL

  /**
   * All valid links at which the sitemap points to. A link is valid if it has
   * correct URL syntax and points to a page placed somewhere inside the
   * directory where the sitemap is located. It filters the URLs returned by the
   * `parseLinks` method.
   */
  final val links: Seq[URL] = filterURLs(parseLinks)

  /**
   * Returns all raw-URLs parsed from the sitemap.
   */
  protected def parseLinks: Seq[String]

  private def filterURLs(rawURLs: Seq[String]): Seq[URL] = {
    val rootDirectory = location.toString.reverse.dropWhile(_ != '/').reverse

    def makeURLs(urls: Seq[String]): Seq[URL] =
      urls.map(url => Try(new URL(url))).filter(_.isSuccess).map(_.get)

    def validURL(url: URL): Boolean =
      url.toString.startsWith(rootDirectory)

    makeURLs(rawURLs).filter(validURL)
  }
}

/**
 * Class that represents .xml sitemaps.
 */
final class SitemapXML(val location: URL, content: String) extends Sitemap {
  protected def parseLinks: Seq[String] =
    (XML.loadString(content) \ "url" \ "loc").map(_.text)
}

/**
 * Class that represents .rss sitemaps.
 */
final class SitemapRSS(val location: URL, content: String) extends Sitemap {
  protected def parseLinks: Seq[String] =
    (XML.loadString(content) \ "channel" \ "item" \ "link").map(_.text)
}

/**
 * Class that represents .txt sitemaps.
 */<commit_msg>Removed unhelpful documentation.<commit_after>package robots.protocol.inclusion

import java.net.URL

import scala.util.Try
import scala.xml.XML

/**
 * Abstract class that represents a sitemap.
 *
 * @author andrei
 */
sealed abstract class Sitemap {
  /**
   * The location (URL) of the sitemap.
   */
  val location: URL

  /**
   * All valid links at which the sitemap points to. A link is valid if it has
   * correct URL syntax and points to a page placed somewhere inside the
   * directory where the sitemap is located. It filters the URLs returned by the
   * `parseLinks` method.
   */
  final val links: Seq[URL] = filterURLs(parseLinks)

  /**
   * Returns all raw-URLs parsed from the sitemap.
   */
  protected def parseLinks: Seq[String]

  private def filterURLs(rawURLs: Seq[String]): Seq[URL] = {
    val rootDirectory = location.toString.reverse.dropWhile(_ != '/').reverse

    def makeURLs(urls: Seq[String]): Seq[URL] =
      urls.map(url => Try(new URL(url))).filter(_.isSuccess).map(_.get)

    def validURL(url: URL): Boolean =
      url.toString.startsWith(rootDirectory)

    makeURLs(rawURLs).filter(validURL)
  }
}

final class SitemapXML(val location: URL, content: String) extends Sitemap {
  protected def parseLinks: Seq[String] =
    (XML.loadString(content) \ "url" \ "loc").map(_.text)
}

final class SitemapRSS(val location: URL, content: String) extends Sitemap {
  protected def parseLinks: Seq[String] =
    (XML.loadString(content) \ "channel" \ "item" \ "link").map(_.text)
}

{'fraction_non_alphanumeric': 0.12306232204998419, 'fraction_numerical': 0.0, 'mean_word_length': 3.5171428571428573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5896708', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1024, 'n_words': 303}