<commit_before>package com.thetestpeople.trt.jenkins.importer

import java.net.URI

import scala.xml._

import com.thetestpeople.trt.utils.HasLogger
import com.thetestpeople.trt.utils.UriUtils._
import com.thetestpeople.trt.utils.http.Credentials
import com.thetestpeople.trt.utils.http._

class JenkinsScraperException(message: String, cause: Throwable = null) extends RuntimeException(message, cause)

class JenkinsScraper(
    http: Http,
    credentialsOpt: Option[Credentials],
    fetchConsole: Boolean) extends HasLogger {

  @throws[JenkinsScraperException]
  def getJenkinsJob(jobUrl: URI): JenkinsJob = {
    val jobXml = getJobXml(jobUrl)
    parseJobXml(jobUrl, jobXml)
  }

  private def parseJobXml(jobUrl: URI, jobXml: Elem): JenkinsJob =
    try
      new JenkinsJobXmlParser().parse(jobXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing Jenkins job XML from $jobUrl", e)
    }

  private def scrapeTestResult(buildUrl: URI): Option[OrdinaryTestResult] = {
    val testXml = getTestResultsXml(buildUrl)
    val testResult = parseTestResultsXml(testResultsUrl(buildUrl), testXml)
    testResult match {
      case result: OrdinaryTestResult   ⇒ Some(result)
      case result: MatrixTestResult     ⇒ processMatrixTestResult(result)
      case result: AggregatedTestResult ⇒ processAggregatedTestResult(result)
    }
  }

  private def parseTestResultsXml(testUrl: URI, testXml: Elem): TestResult =
    try
      new JenkinsTestResultXmlParser().parseTestResult(testXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing test result XML from $testUrl: ${e.getMessage}", e)
    }

  private def processAggregatedTestResult(testResult: AggregatedTestResult): Option[OrdinaryTestResult] = {
    val childResults =
      for {
        childUrl ← testResult.urls
        childTestXml = getTestResultsXml(childUrl)
      } yield parseOrdinaryTestResult(childTestXml)
    val aggregatedResults = childResults.reduce(_ combine _)
    Some(aggregatedResults)
  }

  private def processMatrixTestResult(testResult: MatrixTestResult): Option[OrdinaryTestResult] =
    for {
      childUrl ← testResult.urls.headOption // TODO: handle more than one matrix result, maybe auto-assign to a configuration?
      childTestXml = getTestResultsXml(childUrl)
    } yield parseOrdinaryTestResult(childTestXml)

  private def parseOrdinaryTestResult(testResultXml: Elem): OrdinaryTestResult =
    try
      new JenkinsTestResultXmlParser().parseOrdinaryTestResult(testResultXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing test result: ${e.getMessage}", e)
    }

  /**
   * @return None if build has no tests associated with it.
   */
  @throws[JenkinsScraperException]
  def scrapeBuild(buildUrl: URI, jobUrl: URI): Option[JenkinsBuild] = {
    logger.debug(s"scrapeBuild($buildUrl)")
    val buildXml = getBuildXml(buildUrl)
    val buildSummary = parseBuild(buildUrl, buildXml)
    if (buildSummary.hasTestReport) {
      scrapeTestResult(buildUrl)
      for {
        testResult ← scrapeTestResult(buildUrl)
        consoleTextOpt = if (fetchConsole) Some(getConsole(buildUrl)) else None
      } yield JenkinsBuild(jobUrl, buildSummary, testResult, consoleTextOpt)
    } else<commit_msg>unnecessary double scraping of test reports<commit_after>package com.thetestpeople.trt.jenkins.importer

import java.net.URI

import scala.xml._

import com.thetestpeople.trt.utils.HasLogger
import com.thetestpeople.trt.utils.UriUtils._
import com.thetestpeople.trt.utils.http.Credentials
import com.thetestpeople.trt.utils.http._

class JenkinsScraperException(message: String, cause: Throwable = null) extends RuntimeException(message, cause)

class JenkinsScraper(
    http: Http,
    credentialsOpt: Option[Credentials],
    fetchConsole: Boolean) extends HasLogger {

  @throws[JenkinsScraperException]
  def getJenkinsJob(jobUrl: URI): JenkinsJob = {
    val jobXml = getJobXml(jobUrl)
    parseJobXml(jobUrl, jobXml)
  }

  private def parseJobXml(jobUrl: URI, jobXml: Elem): JenkinsJob =
    try
      new JenkinsJobXmlParser().parse(jobXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing Jenkins job XML from $jobUrl", e)
    }

  private def scrapeTestResult(buildUrl: URI): Option[OrdinaryTestResult] = {
    val testXml = getTestResultsXml(buildUrl)
    val testResult = parseTestResultsXml(testResultsUrl(buildUrl), testXml)
    testResult match {
      case result: OrdinaryTestResult   ⇒ Some(result)
      case result: MatrixTestResult     ⇒ processMatrixTestResult(result)
      case result: AggregatedTestResult ⇒ processAggregatedTestResult(result)
    }
  }

  private def parseTestResultsXml(testUrl: URI, testXml: Elem): TestResult =
    try
      new JenkinsTestResultXmlParser().parseTestResult(testXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing test result XML from $testUrl: ${e.getMessage}", e)
    }

  private def processAggregatedTestResult(testResult: AggregatedTestResult): Option[OrdinaryTestResult] = {
    val childResults =
      for {
        childUrl ← testResult.urls
        childTestXml = getTestResultsXml(childUrl)
      } yield parseOrdinaryTestResult(childTestXml)
    val aggregatedResults = childResults.reduce(_ combine _)
    Some(aggregatedResults)
  }

  private def processMatrixTestResult(testResult: MatrixTestResult): Option[OrdinaryTestResult] =
    for {
      childUrl ← testResult.urls.headOption // TODO: handle more than one matrix result, maybe auto-assign to a configuration?
      childTestXml = getTestResultsXml(childUrl)
    } yield parseOrdinaryTestResult(childTestXml)

  private def parseOrdinaryTestResult(testResultXml: Elem): OrdinaryTestResult =
    try
      new JenkinsTestResultXmlParser().parseOrdinaryTestResult(testResultXml)
    catch {
      case e: ParseException ⇒
        throw new JenkinsScraperException(s"Problem parsing test result: ${e.getMessage}", e)
    }

  /**
   * @return None if build has no tests associated with it.
   */
  @throws[JenkinsScraperException]
  def scrapeBuild(buildUrl: URI, jobUrl: URI): Option[JenkinsBuild] = {
    logger.debug(s"scrapeBuild($buildUrl)")
    val buildXml = getBuildXml(buildUrl)
    val buildSummary = parseBuild(buildUrl, buildXml)
    if (buildSummary.hasTestReport) {
      for {
        testResult ← scrapeTestResult(buildUrl)
        consoleTextOpt = if (fetchConsole) Some(getConsole(buildUrl)) else None
      } yield JenkinsBuild(jobUrl, buildSummary, testResult, consoleTextOpt)
    } else
{'fraction_non_alphanumeric': 0.08264955989855288, 'fraction_numerical': 0.0, 'mean_word_length': 4.233411397345823, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6439055', 'n_tokens_mistral': 1998, 'n_tokens_neox': 1958, 'n_words': 469}