<commit_before>package applicant.etl

import applicant.nlp._
import java.text.DecimalFormat
import java.net.{URL, HttpURLConnection}
import scala.io._
import java.io.FileNotFoundException
import org.json4s._
import org.json4s.jackson.JsonMethods._

/**
 * Class to retrieve JSON data from URLs and convert them
 * to Scala maps for saving to Elasticsearch
 *
 */
object ApiMapper {
  /**
   * Retrieves data from github api and converts it to a map of [String,String]
   * @param github A string of the parsed applicant's GitHub URL
   * @return A [String,String] map of Github data from the Github api
   */
  def githubAPI(github : String) : Map[String,String] = {
    LinkParser.parseGithubProfile("https://api.github.com/users/", github) match {
      case Some(apiUrl) =>
        try {
          val jsonString = scala.io.Source.fromURL(apiUrl).mkString
          val parsedJson = parse(jsonString)
          object VarToString extends CustomSerializer[String](format => (
            {
              case JBool(false) => "false"
              case JBool(true) => "true"
            },
            {
              case format: String => JString(format)
            }
          ))
          implicit val formats = DefaultFormats + VarToString

          val gitJsonMap = parsedJson.extract[Map[String, String]]
          return gitJsonMap
        }
        catch {
          case ex: Exception => return Map()
        }
      case None =>
        return Map()
    }
  }
  /**
   * Given two locations, returns the distance between the two
   * Note: uses the unauthorized Google Maps API internally, may be cut off
   * after too many requests
   * @param location1 First location
   * @param location2 Second location
   * @return Distance between the two locations in meters
   */
  def googlemapsAPI(location1 : String, location2 : String) : Option[Int] = {
    val apiUrl = "https://maps.googleapis.com/maps/api/distancematrix/json?units=imperial&origins=" + location1.trim().replaceAll("\\s", "+") + "&destinations=" + location2.trim().replaceAll("\\s", "+")
    val jsonString = scala.io.Source.fromURL(apiUrl).mkString
    println(jsonString)
    val distance = ((((parse(jsonString) \ "rows")(0) \ "elements")(0) \ "distance") \ "value" )
    implicit val formats = DefaultFormats
    if (distance.isInstanceOf[JInt]) {
      return Some(distance.extract[Int])
    }
    else {
      return None
    }
  }
}
<commit_msg>Removed uneeded print<commit_after>package applicant.etl

import applicant.nlp._
import java.text.DecimalFormat
import java.net.{URL, HttpURLConnection}
import scala.io._
import java.io.FileNotFoundException
import org.json4s._
import org.json4s.jackson.JsonMethods._

/**
 * Class to retrieve JSON data from URLs and convert them
 * to Scala maps for saving to Elasticsearch
 *
 */
object ApiMapper {
  /**
   * Retrieves data from github api and converts it to a map of [String,String]
   * @param github A string of the parsed applicant's GitHub URL
   * @return A [String,String] map of Github data from the Github api
   */
  def githubAPI(github : String) : Map[String,String] = {
    LinkParser.parseGithubProfile("https://api.github.com/users/", github) match {
      case Some(apiUrl) =>
        try {
          val jsonString = scala.io.Source.fromURL(apiUrl).mkString
          val parsedJson = parse(jsonString)
          object VarToString extends CustomSerializer[String](format => (
            {
              case JBool(false) => "false"
              case JBool(true) => "true"
            },
            {
              case format: String => JString(format)
            }
          ))
          implicit val formats = DefaultFormats + VarToString

          val gitJsonMap = parsedJson.extract[Map[String, String]]
          return gitJsonMap
        }
        catch {
          case ex: Exception => return Map()
        }
      case None =>
        return Map()
    }
  }
  /**
   * Given two locations, returns the distance between the two
   * Note: uses the unauthorized Google Maps API internally, may be cut off
   * after too many requests
   * @param location1 First location
   * @param location2 Second location
   * @return Distance between the two locations in meters
   */
  def googlemapsAPI(location1 : String, location2 : String) : Option[Int] = {
    val apiUrl = "https://maps.googleapis.com/maps/api/distancematrix/json?units=imperial&origins=" + location1.trim().replaceAll("\\s", "+") + "&destinations=" + location2.trim().replaceAll("\\s", "+")
    val jsonString = scala.io.Source.fromURL(apiUrl).mkString
    val distance = ((((parse(jsonString) \ "rows")(0) \ "elements")(0) \ "distance") \ "value" )
    implicit val formats = DefaultFormats
    if (distance.isInstanceOf[JInt]) {
      return Some(distance.extract[Int])
    }
    else {
      return None
    }
  }
}

{'fraction_non_alphanumeric': 0.11138613861386139, 'fraction_numerical': 0.004125412541254125, 'mean_word_length': 2.679059180576631, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2755730', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1373, 'n_words': 393}