<commit_before>package la.dp.ingestion3

import java.io.File
import java.net.URL
import java.util.concurrent.TimeUnit

import la.dp.ingestion3.harvesters.OaiHarvester
import la.dp.ingestion3.utils.{FlatFileIO, Utils}

import scala.util.control.NonFatal

/**
  * Driver program for OAI harvest
  */
object OaiHarvesterMain extends App {

  /**
    * Entry point for running an OAI harvest
    *
    * @param args Output directory: String
    *             OAI URL: String
    *             Metadata Prefix: String oai_dc oai_qdc, mods, MODS21 etc.
    *             OAI Verb: String ListRecords, ListSets etc.
    */
  override def main(args: Array[String]): Unit = {

    val logger = org.apache.log4j.LogManager.getLogger("harvester")

    if(!args.length.equals(4)) {
      println("Bad Args: <OUT>, <OAI URL>, <METADATA PREFIX>, <OAI VERB>")
      sys.exit(-1)
    }
    val outDir: File = new File(args(0))
    val endpoint = new URL(args(1))
    val metadataPrefix = args(2)
    val verb = args(3)
    val fileIO = new FlatFileIO

    logger.debug(s"Saving records to ${outDir}")
    logger.debug(s"Harvesting from ${endpoint}")
    // Create the harvester and run
    val harvester: OaiHarvester = new OaiHarvester(endpoint, metadataPrefix, outDir, fileIO)

    val start = System.currentTimeMillis()
    try {
      harvester.runHarvest(verb)
    } catch {
      case NonFatal(e) => {
        logger.error(e.getMessage)
        logger.debug("Exiting...")
        System.exit(-1)
      }
    }
    val end = System.currentTimeMillis()

    val recordsHarvestedCount = Utils.countFiles(outDir, ".xml")
    val runtimeMs = (end - start) + 1

    printResults(runtimeMs, recordsHarvestedCount)

  }

  /**
    * Print the results of a harvest
    *
    * Example:
    *   Harvest count: 242924 records harvested
    *   Runtime: 4 minutes 24 seconds
    *   Throughput: 920 records/second<commit_msg>Typesafe Int comparision<commit_after>package la.dp.ingestion3

import java.io.File
import java.net.URL
import java.util.concurrent.TimeUnit

import la.dp.ingestion3.harvesters.OaiHarvester
import la.dp.ingestion3.utils.{FlatFileIO, Utils}

import scala.util.control.NonFatal

/**
  * Driver program for OAI harvest
  */
object OaiHarvesterMain extends App {

  /**
    * Entry point for running an OAI harvest
    *
    * @param args Output directory: String
    *             OAI URL: String
    *             Metadata Prefix: String oai_dc oai_qdc, mods, MODS21 etc.
    *             OAI Verb: String ListRecords, ListSets etc.
    */
  override def main(args: Array[String]): Unit = {

    val logger = org.apache.log4j.LogManager.getLogger("harvester")

    if(!args.length === 4 ) {
      logger.error("Bad number of args: <OUT>, <OAI URL>, <METADATA PREFIX>, <OAI VERB>")
      sys.exit(-1)
    }

    val outDir: File = new File(args(0))
    val endpoint = new URL(args(1))
    val metadataPrefix = args(2)
    val verb = args(3)
    // How to serialize the output
    val fileIO = new FlatFileIO

    logger.debug(s"Saving records to ${outDir}")
    logger.debug(s"Harvesting from ${endpoint}")

    // Create the harvester and run
    val harvester: OaiHarvester = new OaiHarvester(endpoint, metadataPrefix, outDir, fileIO)

    val start = System.currentTimeMillis()
    try {
      harvester.runHarvest(verb)
    } catch {
      case NonFatal(e) => {
        logger.error(e.getMessage)
        logger.debug("Exiting...")
        System.exit(-1)
      }
    }
    val end = System.currentTimeMillis()

    val recordsHarvestedCount = Utils.countFiles(outDir, ".xml")
    val runtimeMs = (end - start) + 1

    printResults(runtimeMs, recordsHarvestedCount)

  }

  /**
    * Print the results of a harvest
    *
    * Example:
    *   Harvest count: 242924 records harvested
    *   Runtime: 4 minutes 24 seconds
    *   Throughput: 920 records/second
{'fraction_non_alphanumeric': 0.10993274702534919, 'fraction_numerical': 0.013450594930160372, 'mean_word_length': 2.7326254826254828, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1787177', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1240, 'n_words': 321}