<commit_before>package la.dp.ingestion3

import java.net.URL

import org.apache.http.client.utils.URIBuilder

/**
  * Creates a fully formed and valid URL for OAI request
  */
class OaiQueryUrlBuilder extends QueryUrlBuilder {
  /**
    * Builds an OAI request
    *
    * @param params Map of parameters needed to construct the URL
    *               OAI request verbs
    *               See https://www.openarchives.org/OAI/openarchivesprotocol.html#ProtocolMessages
    * @return URL
    */
  def buildQueryUrl(params: Map[String, String]): URL = {
    // Required properties, not sure if this is the right style
    assume(params.get("endpoint").get.nonEmpty)
    assume(params.get("verb").get.nonEmpty)
    assume(params.get("metadataPrefix").get.nonEmpty)

    val verb = params.get("verb").get
    val metadataPrefix = params.get("metadataPrefix").get
    val url = new URL(params.get("endpoint").get)

    // Build the URL
    val urlParams = new URIBuilder()
      .setScheme(url.getProtocol)
      .setHost(url.getHost)
      .setPath(url.getPath)
      .setParameter("verb", verb)

    params.get("resumptionToken") match {
      case Some(value) => urlParams.setParameter("resumptionToken", value)
      case None  => urlParams.setParameter("metadataPrefix", metadataPrefix)
    }

    urlParams.build.toURL
  }
}

trait QueryUrlBuilder {
  def buildQueryUrl (params: Map[String, String]): URL

}<commit_msg>Change logic for checking the presence of resumptionToken. Previous method was failing on match because if resumptionToken was not present the second .get method was called on a None object<commit_after>package la.dp.ingestion3

import java.net.URL

import org.apache.http.client.utils.URIBuilder

/**
  * Creates a fully formed and valid URL for OAI request
  */
class OaiQueryUrlBuilder extends QueryUrlBuilder {
  /**
    * Builds an OAI request
    *
    * @param params Map of parameters needed to construct the URL
    *               OAI request verbs
    *               See https://www.openarchives.org/OAI/openarchivesprotocol.html#ProtocolMessages
    * @return URL
    */
  def buildQueryUrl(params: Map[String, String]): URL = {
    // Required properties, not sure if this is the right style
    assume(params.get("endpoint").get.nonEmpty)
    assume(params.get("verb").get.nonEmpty)
    assume(params.get("metadataPrefix").get.nonEmpty)

    val verb = params.get("verb").get
    val metadataPrefix = params.get("metadataPrefix").get
    val url = new URL(params.get("endpoint").get)

    // Build the URL
    val urlParams = new URIBuilder()
      .setScheme(url.getProtocol)
      .setHost(url.getHost)
      .setPath(url.getPath)
      .setParameter("verb", verb)

    params.get("resumptionToken") match {
      case Some(v) if v.nonEmpty => urlParams.setParameter("resumptionToken", v)
      case None => urlParams.setParameter("metadataPrefix", metadataPrefix)
    }

    urlParams.build.toURL
  }
}

trait QueryUrlBuilder {
  def buildQueryUrl (params: Map[String, String]): URL

}
{'fraction_non_alphanumeric': 0.10942148760330579, 'fraction_numerical': 0.0006611570247933885, 'mean_word_length': 3.238095238095238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7316399', 'n_tokens_mistral': 895, 'n_tokens_neox': 882, 'n_words': 231}