<commit_before>package shark.parse

import scala.collection.JavaConversions._

import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.exec.TaskFactory
import org.apache.hadoop.hive.ql.parse.{ASTNode, BaseSemanticAnalyzer, DDLSemanticAnalyzer, HiveParser}
import org.apache.hadoop.hive.ql.plan.DDLWork

import org.apache.spark.rdd.{UnionRDD, RDD}

import shark.execution.SparkDDLWork
import shark.{LogHelper, SharkEnv}
import shark.memstore2.MemoryMetadataManager


class SharkDDLSemanticAnalyzer(conf: HiveConf) extends DDLSemanticAnalyzer(conf) with LogHelper {

  override def analyzeInternal(astNode: ASTNode): Unit = {
    super.analyzeInternal(astNode)

    astNode.getToken.getType match {
      case HiveParser.TOK_ALTERTABLE_RENAME => {
        analyzeAlterTableRename(astNode)
      }
      case _ => Unit
    }
  }

  private def analyzeAlterTableRename(astNode: ASTNode) {
    val oldTableName = getTableName(astNode)
    val newTableName = BaseSemanticAnalyzer.getUnescapedName(
      astNode.getChild(1).asInstanceOf[ASTNode])

    // Hive's DDLSemanticAnalyzer#AnalyzeInternal() will only populate rootTasks with a DDLTask
    // and DDLWork that contains an AlterTableDesc.
    assert(rootTasks.size == 1)
    val ddlTask = rootTasks.head
    val ddlWork = ddlTask.getWork
    assert(ddlWork.isInstanceOf[DDLWork])

    val alterTableDesc = ddlWork.asInstanceOf[DDLWork].getAlterTblDesc
    val sparkDDLWork = new SparkDDLWork(alterTableDesc)
    ddlTask.addDependentTask(TaskFactory.get(sparkDDLWork, conf))
  }

  private def getTableName(node: ASTNode): String = {
    BaseSemanticAnalyzer.getUnescapedName(node.getChild(0).asInstanceOf[ASTNode])
  }
}
<commit_msg>Fix merge mistake in SharkDDLSemanticAnalyzer.<commit_after>package shark.parse

import scala.collection.JavaConversions._

import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.exec.TaskFactory
import org.apache.hadoop.hive.ql.parse.{ASTNode, BaseSemanticAnalyzer, DDLSemanticAnalyzer, HiveParser}
import org.apache.hadoop.hive.ql.plan.DDLWork

import org.apache.spark.rdd.{UnionRDD, RDD}

import shark.execution.SparkDDLWork
import shark.{LogHelper, SharkEnv}
import shark.memstore2.MemoryMetadataManager


class SharkDDLSemanticAnalyzer(conf: HiveConf) extends DDLSemanticAnalyzer(conf) with LogHelper {

  override def analyzeInternal(astNode: ASTNode): Unit = {
    super.analyzeInternal(astNode)

    astNode.getToken.getType match {
      case HiveParser.TOK_DROPTABLE => {
        SharkEnv.unpersist(getTableName(astNode))
      }
      case HiveParser.TOK_ALTERTABLE_RENAME => {
        analyzeAlterTableRename(astNode)
      }
      case _ => Unit
    }
  }

  private def analyzeAlterTableRename(astNode: ASTNode) {
    val oldTableName = getTableName(astNode)
    val newTableName = BaseSemanticAnalyzer.getUnescapedName(
      astNode.getChild(1).asInstanceOf[ASTNode])

    // Hive's DDLSemanticAnalyzer#AnalyzeInternal() will only populate rootTasks with a DDLTask
    // and DDLWork that contains an AlterTableDesc.
    assert(rootTasks.size == 1)
    val ddlTask = rootTasks.head
    val ddlWork = ddlTask.getWork
    assert(ddlWork.isInstanceOf[DDLWork])

    val alterTableDesc = ddlWork.asInstanceOf[DDLWork].getAlterTblDesc
    val sparkDDLWork = new SparkDDLWork(alterTableDesc)
    ddlTask.addDependentTask(TaskFactory.get(sparkDDLWork, conf))
  }

  private def getTableName(node: ASTNode): String = {
    BaseSemanticAnalyzer.getUnescapedName(node.getChild(0).asInstanceOf[ASTNode])
  }
}

{'fraction_non_alphanumeric': 0.08893280632411067, 'fraction_numerical': 0.002258610954263128, 'mean_word_length': 4.659744408945687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2889082', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1174, 'n_words': 197}