<commit_before>package com.ligadata.MetadataAPI.Utility

import java.io.File

import com.ligadata.MetadataAPI.MetadataAPIImpl

import scala.io.Source
import org.apache.log4j._
/**
 * Created by dhaval on 8/13/15.
 */
object ConfigService {
  private val userid: Option[String] = Some("metadataapi")
  val loggerName = this.getClass.getName
  lazy val logger = Logger.getLogger(loggerName)

 def uploadEngineConfig(input: String): String ={
   var response = ""
   var configFileDir: String = ""
   //val gitMsgFile = "https://raw.githubusercontent.com/ligadata-dhaval/Kamanja/master/HelloWorld_Msg_Def.json"
   if (input == "") {
     configFileDir = MetadataAPIImpl.GetMetadataAPIConfig.getProperty("CONFIG_FILES_DIR")
     if (configFileDir == null) {
       response = "CONFIG_FILES_DIR property missing in the metadata API configuration"
     } else {
       //verify the directory where messages can be present
       IsValidDir(configFileDir) match {
         case true => {
           //get all files with json extension
           val types: Array[File] = new java.io.File(configFileDir).listFiles.filter(_.getName.endsWith(".json"))
           types.length match {
             case 0 => {
               println("Configs not found at " + configFileDir)
               response="Configs not found at " + configFileDir
             }
             case option => {
               val configDefs = getUserInputFromMainMenu(types)
               for (configDef <- configDefs) {
                 response += MetadataAPIImpl.UploadConfig(configDef.toString, userid, "configuration")
               }
             }
           }
         }
         case false => {
           //println("Message directory is invalid.")
           response = "Config directory is invalid."
         }
       }
     }
   } else {
     //input provided
     var message = new File(input.toString)
     val configDef = Source.fromFile(message).mkString
     response = MetadataAPIImpl.UploadConfig(configDef.toString, userid, "configuration")
   }
   response
 }

  def uploadCompileConfig(input: String): String ={
    var response = ""
    var configFileDir: String = ""
    //val gitMsgFile = "https://raw.githubusercontent.com/ligadata-dhaval/Kamanja/master/HelloWorld_Msg_Def.json"
    if (input == "") {
      configFileDir = MetadataAPIImpl.GetMetadataAPIConfig.getProperty("CONFIG_FILES_DIR")
      if (configFileDir == null) {
        response = "CONFIG_FILES_DIR property missing in the metadata API configuration"
      } else {
        //verify the directory where messages can be present
        IsValidDir(configFileDir) match {
          case true => {
            //get all files with json extension
            val types: Array[File] = new java.io.File(configFileDir).listFiles.filter(_.getName.endsWith(".json"))
            types.length match {
              case 0 => {
                println("Configs not found at " + configFileDir)
                response="Configs not found at " + configFileDir
              }
              case option => {
                val configDefs = getUserInputFromMainMenu(types)
                for (configDef <- configDefs) {
                  response += MetadataAPIImpl.UploadModelsConfig(configDef.toString, userid, "configuration")
                }
              }
            }
          }
          case false => {
            //println("Message directory is invalid.")
            response = "Config directory is invalid."
          }
        }
      }
    } else {
      //input provided
      var message = new File(input.toString)
      val configDef = Source.fromFile(message).mkString
      response = MetadataAPIImpl.UploadModelsConfig(configDef.toString, userid, "configuration")
    }
    response
  }
  def dumpAllCfgObjects: String ={
    var response=""
    try{
      response= MetadataAPIImpl.GetAllCfgObjects("JSON", userid)
    }
    catch {
      case e: Exception => {
        response=e.getStackTrace.toString
      }
    }
    response

  }
  def removeEngineConfig: String ={
    var response="TO BE IMPLEMENTED"
    response
  }

  def IsValidDir(dirName: String): Boolean = {
    val iFile = new File(dirName)
    if (!iFile.exists) {
      println("The File Path (" + dirName + ") is not found: ")
      false
    } else if (!iFile.isDirectory) {
      println("The File Path (" + dirName + ") is not a directory: ")
      false
    } else
      true
  }

  def   getUserInputFromMainMenu(messages: Array[File]): Array[String] = {
    var listOfMsgDef: Array[String] = Array[String]()
    var srNo = 0
    println("\nPick a Config Definition file(s) from below choices\n")
    for (message <- messages) {
      srNo += 1
      println("[" + srNo + "]" + message)
    }
    print("\nEnter your choice(If more than 1 choice, please use commas to seperate them): \n")
    val userOptions: List[Int] = Console.readLine().filter(_ != '\n').split(',').filter(ch => (ch != null && ch != "")).map(_.trim.toInt).toList
    //check if user input valid. If not exit
    for (userOption <- userOptions) {
      userOption match {
        case userOption if (1 to srNo).contains(userOption) => {
          //find the file location corresponding to the message
          var message = messages(userOption - 1)
          //process message
          val messageDef = Source.fromFile(message).mkString
          listOfMsgDef = listOfMsgDef :+ messageDef
        }
        case _ => {
          println("Unknown option: ")
        }
      }
    }
    listOfMsgDef
  }
}
<commit_msg>Change cluster config method name from engine to cluster<commit_after>package com.ligadata.MetadataAPI.Utility

import java.io.File

import com.ligadata.MetadataAPI.MetadataAPIImpl

import scala.io.Source
import org.apache.log4j._
/**
 * Created by dhaval on 8/13/15.
 */
object ConfigService {
  private val userid: Option[String] = Some("metadataapi")
  val loggerName = this.getClass.getName
  lazy val logger = Logger.getLogger(loggerName)

 def uploadClusterConfig(input: String): String ={
   var response = ""
   var configFileDir: String = ""
   //val gitMsgFile = "https://raw.githubusercontent.com/ligadata-dhaval/Kamanja/master/HelloWorld_Msg_Def.json"
   if (input == "") {
     configFileDir = MetadataAPIImpl.GetMetadataAPIConfig.getProperty("CONFIG_FILES_DIR")
     if (configFileDir == null) {
       response = "CONFIG_FILES_DIR property missing in the metadata API configuration"
     } else {
       //verify the directory where messages can be present
       IsValidDir(configFileDir) match {
         case true => {
           //get all files with json extension
           val types: Array[File] = new java.io.File(configFileDir).listFiles.filter(_.getName.endsWith(".json"))
           types.length match {
             case 0 => {
               println("Configs not found at " + configFileDir)
               response="Configs not found at " + configFileDir
             }
             case option => {
               val configDefs = getUserInputFromMainMenu(types)
               for (configDef <- configDefs) {
                 response += MetadataAPIImpl.UploadConfig(configDef.toString, userid, "configuration")
               }
             }
           }
         }
         case false => {
           //println("Message directory is invalid.")
           response = "Config directory is invalid."
         }
       }
     }
   } else {
     //input provided
     var message = new File(input.toString)
     val configDef = Source.fromFile(message).mkString
     response = MetadataAPIImpl.UploadConfig(configDef.toString, userid, "configuration")
   }
   response
 }

  def uploadCompileConfig(input: String): String ={
    var response = ""
    var configFileDir: String = ""
    //val gitMsgFile = "https://raw.githubusercontent.com/ligadata-dhaval/Kamanja/master/HelloWorld_Msg_Def.json"
    if (input == "") {
      configFileDir = MetadataAPIImpl.GetMetadataAPIConfig.getProperty("CONFIG_FILES_DIR")
      if (configFileDir == null) {
        response = "CONFIG_FILES_DIR property missing in the metadata API configuration"
      } else {
        //verify the directory where messages can be present
        IsValidDir(configFileDir) match {
          case true => {
            //get all files with json extension
            val types: Array[File] = new java.io.File(configFileDir).listFiles.filter(_.getName.endsWith(".json"))
            types.length match {
              case 0 => {
                println("Configs not found at " + configFileDir)
                response="Configs not found at " + configFileDir
              }
              case option => {
                val configDefs = getUserInputFromMainMenu(types)
                for (configDef <- configDefs) {
                  response += MetadataAPIImpl.UploadModelsConfig(configDef.toString, userid, "configuration")
                }
              }
            }
          }
          case false => {
            //println("Message directory is invalid.")
            response = "Config directory is invalid."
          }
        }
      }
    } else {
      //input provided
      var message = new File(input.toString)
      val configDef = Source.fromFile(message).mkString
      response = MetadataAPIImpl.UploadModelsConfig(configDef.toString, userid, "configuration")
    }
    response
  }
  def dumpAllCfgObjects: String ={
    var response=""
    try{
      response= MetadataAPIImpl.GetAllCfgObjects("JSON", userid)
    }
    catch {
      case e: Exception => {
        response=e.getStackTrace.toString
      }
    }
    response

  }
  def removeEngineConfig: String ={
    var response="TO BE IMPLEMENTED"
    response
  }

  def IsValidDir(dirName: String): Boolean = {
    val iFile = new File(dirName)
    if (!iFile.exists) {
      println("The File Path (" + dirName + ") is not found: ")
      false
    } else if (!iFile.isDirectory) {
      println("The File Path (" + dirName + ") is not a directory: ")
      false
    } else
      true
  }

  def   getUserInputFromMainMenu(messages: Array[File]): Array[String] = {
    var listOfMsgDef: Array[String] = Array[String]()
    var srNo = 0
    println("\nPick a Config Definition file(s) from below choices\n")
    for (message <- messages) {
      srNo += 1
      println("[" + srNo + "]" + message)
    }
    print("\nEnter your choice(If more than 1 choice, please use commas to seperate them): \n")
    val userOptions: List[Int] = Console.readLine().filter(_ != '\n').split(',').filter(ch => (ch != null && ch != "")).map(_.trim.toInt).toList
    //check if user input valid. If not exit
    for (userOption <- userOptions) {
      userOption match {
        case userOption if (1 to srNo).contains(userOption) => {
          //find the file location corresponding to the message
          var message = messages(userOption - 1)
          //process message
          val messageDef = Source.fromFile(message).mkString
          listOfMsgDef = listOfMsgDef :+ messageDef
        }
        case _ => {
          println("Unknown option: ")
        }
      }
    }
    listOfMsgDef
  }
}

{'fraction_non_alphanumeric': 0.1060878962536023, 'fraction_numerical': 0.0023414985590778097, 'mean_word_length': 2.2518301610541727, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 27, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5016260', 'n_tokens_mistral': 3103, 'n_tokens_neox': 3042, 'n_words': 778}