<commit_before>package services

import models.Tables.GlobalSetting
import org.apache.pig.impl.logicalLayer.FrontendException
import org.apache.pig.impl.logicalLayer.schema.Schema
import org.apache.pig.{ExecType, PigServer}
import play.api.Logger
import qrygraph.shared.compilation.QueryCompiler
import qrygraph.shared.data._
import qrygraph.shared.pig._

import scala.collection.JavaConversions._

/**
  * here are functions that are used to determine the type of a query using the Pig toolSet
  */
object PigTypeDetection {

  /** this function returns the types of all aliases in the query */
  def evaluateTypes(settings: GlobalSetting, dataSources: List[DataSource], graph: PigQueryGraph): (Map[String, ResultType], Map[String, String]) = {
    Logger.info("Starting type evaluation")

    // On deployment, set the correct home dir
    if (!System.getProperty("user.dir").contains(":")) {
      Logger.info("Old user.dir: " + System.getProperty("user.dir"))
      System.setProperty("user.dir", "/app/")
      Logger.info("New user.dir: " + System.getProperty("user.dir"))
    }

    // create a new pigServer
    val pigServer = new PigServer(ExecType.LOCAL, PigProperties.generateProperties(settings))
    // create an empty map to store the resultTypes in
    var typesMap = Map[String, ResultType]()
    // here we store the errors
    var errorsMap = Map[String, String]()
    // remove components
    QueryCompiler
      .advancedCompile(graph)
      .foreach { case (node, pigCode) =>
        try {
          pigServer.registerQuery(pigCode)
        } catch {
          case f: FrontendException =>
            Logger.info(s"Error on node ${node.name} with msg: ${f.getCause.getMessage}")
            errorsMap += node.name -> f.getCause.getMessage
          case e: Throwable         =>
          //            e.printStackTrace()
        }
      }

    pigServer.getAliasKeySet.toList.foreach(alias => {
//      SystemOutHelper.disableSystemOut()
      val schema = pigServer.dumpSchema(alias)
      typesMap += alias -> schemaToScalaSchema(schema)
//      SystemOutHelper.enableSystemOut()
    })

    // shutdown pigServer
    pigServer.shutdown()<commit_msg>Allow invalidPlans to trigger an exception in the frontend<commit_after>package services

import models.Tables.GlobalSetting
import org.apache.pig.impl.logicalLayer.FrontendException
import org.apache.pig.impl.logicalLayer.schema.Schema
import org.apache.pig.impl.plan.PlanValidationException
import org.apache.pig.{ExecType, PigServer}
import play.api.Logger
import qrygraph.shared.compilation.QueryCompiler
import qrygraph.shared.data._
import qrygraph.shared.pig._

import scala.collection.JavaConversions._

/**
  * here are functions that are used to determine the type of a query using the Pig toolSet
  */
object PigTypeDetection {

  /** this function returns the types of all aliases in the query */
  def evaluateTypes(settings: GlobalSetting, dataSources: List[DataSource], graph: PigQueryGraph): (Map[String, ResultType], Map[String, String]) = {
    Logger.info("Starting type evaluation")

    // On deployment, set the correct home dir
    if (!System.getProperty("user.dir").contains(":")) {
      Logger.info("Old user.dir: " + System.getProperty("user.dir"))
      System.setProperty("user.dir", "/app/")
      Logger.info("New user.dir: " + System.getProperty("user.dir"))
    }

    // create a new pigServer
    val pigServer = new PigServer(ExecType.LOCAL, PigProperties.generateProperties(settings))
    // create an empty map to store the resultTypes in
    var typesMap = Map[String, ResultType]()
    // here we store the errors
    var errorsMap = Map[String, String]()
    // remove components
    QueryCompiler
      .advancedCompile(graph)
      .foreach { case (node, pigCode) =>
        try {
          pigServer.registerQuery(pigCode)
        } catch {
          case f: FrontendException =>
            Logger.info(s"Error on node ${node.name} with msg: ${f.getCause.getMessage}")
            errorsMap += node.name -> f.getCause.getMessage
          case e: Throwable         =>
          //            e.printStackTrace()
        }
      }

    pigServer.getAliasKeySet.toList.foreach(alias => {
      try {
        val schema = pigServer.dumpSchema(alias)
        typesMap += alias -> schemaToScalaSchema(schema)
      } catch {
        case f: PlanValidationException =>
          Logger.info(s"Error on node ${alias} with msg: ${f.getCause.getMessage}")
          errorsMap += alias -> f.getCause.getMessage
        case e: Throwable               =>
          errorsMap += alias -> e.getCause.getMessage
        //            e.printStackTrace()
      }

    })

    // shutdown pigServer
    pigServer.shutdown()
{'fraction_non_alphanumeric': 0.11165151194755762, 'fraction_numerical': 0.0, 'mean_word_length': 2.9318370739817126, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6712881', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1318, 'n_words': 335}