<commit_before>package org.deepdive.ddlog

// DeepDiveLog syntax
// See: https://docs.google.com/document/d/1SBIvvki3mnR28Mf0Pkin9w9mWNam5AA0SpIGj1ZN2c4

import scala.util.parsing.combinator._
import org.apache.commons.lang3.StringEscapeUtils
import scala.util.Try

// ***************************************
// * The union types for for the parser. *
// ***************************************
// case class Variable(varName : String, relName : String, index : Int )
// TODO make Atom a trait, and have multiple case classes, e.g., RelationAtom and CondExprAtom
// ddlog column variable type: constant or variable
sealed trait ColumnVariable
case class Variable(varName : String, relName : String, index : Int ) extends ColumnVariable
case class Constant(value : String, relName: String, index: Int) extends ColumnVariable
case class InlineFunction(functionName: String, args: List[ColumnVariable], isAggregation: Boolean) extends ColumnVariable
case class Expression(variables: List[ColumnVariable], ops: List[String], relName: String, index: Int)
case class Operator(operator: String, operand: ColumnVariable)

case class Atom(name : String, terms : List[Expression])
case class Attribute(name : String, terms : List[Variable], types : List[String])
case class ConjunctiveQuery(head: Atom, bodies: List[List[Atom]], conditions: List[Option[CompoundCondition]], isDistinct: Boolean)
case class Column(name : String, t : String)

// condition
case class BodyWithConditions(body: List[Atom], conditions: Option[CompoundCondition])

case class Condition(lhs: Expression, op: String, rhs: Expression)
case class CompoundCondition(conditions: List[List[Condition]])

// variable type
sealed trait VariableType {
  def cardinality: Long
}
case object BooleanType extends VariableType {
  def cardinality = 2
}
case class MultinomialType(numCategories: Int) extends VariableType {
  def cardinality = numCategories
}

sealed trait FactorWeight {
  def variables : List[String]
}

case class KnownFactorWeight(value: Double) extends FactorWeight {
  def variables = Nil
}
case class UnknownFactorWeight(variables: List[String]) extends FactorWeight

trait RelationType
case class RelationTypeDeclaration(names: List[String], types: List[String]) extends RelationType
case class RelationTypeAlias(likeRelationName: String) extends RelationType

trait FunctionImplementationDeclaration
case class RowWiseLineHandler(format: String, command: String) extends FunctionImplementationDeclaration

// Statements that will be parsed and compiled
trait Statement
case class SchemaDeclaration( a : Attribute , isQuery : Boolean, variableType : Option[VariableType]) extends Statement // atom and whether this is a query relation.
case class FunctionDeclaration( functionName: String, inputType: RelationType, outputType: RelationType, implementations: List[FunctionImplementationDeclaration], mode: String = null) extends Statement
case class ExtractionRule(q : ConjunctiveQuery, supervision: String = null) extends Statement // Extraction rule
case class FunctionCallRule(input : String, output : String, function : String) extends Statement // Extraction rule
case class InferenceRule(q : ConjunctiveQuery, weights : FactorWeight, semantics : String = "Imply", mode: String = null) extends Statement // Weighted rule


// Parser
class DeepDiveLogParser extends JavaTokenParsers {

  // JavaTokenParsers provides several useful number parsers:
  //   wholeNumber, decimalNumber, floatingPointNumber 
  def floatingPointNumberAsDouble = floatingPointNumber ^^ { _.toDouble }
  def stringLiteralAsString = stringLiteral ^^ {
    s => StringEscapeUtils.unescapeJava(
      s.stripPrefix("\"").stripSuffix("\""))
  }
  def stringLiteralAsSqlString = stringLiteral ^^ { s =>
    s"""'${s.stripPrefix("\"").stripSuffix("\"")}'"""
  }
  def constant = stringLiteralAsSqlString | wholeNumber | "TRUE" | "FALSE" | "NULL" | "TEXT" | "INT" | "BOOLEAN"

  // C/Java/Scala-style as well as shell script-style comments are supported
  // by treating them as whiteSpace
  protected override val whiteSpace = """(?:(?:^|\s+)#.*|//.*|(?m)/\*(\*(?!/)|[^*])*\*/|\s)+""".r

  // We just use Java identifiers to parse various names
  def relationName = ident
  def columnName   = ident
  def columnType   = ident ~ ("[]"?) ^^ {
      case ty ~ isArrayType => ty + isArrayType.getOrElse("")
    }
  def variableName = ident
  def functionName = ident
  def semanticType = ident
  def functionModeType = ident
  def inferenceModeType = ident

  def columnDeclaration: Parser[Column] =
    columnName ~ columnType ^^ {
      case(name ~ ty) => Column(name, ty)
    }

  def CategoricalParser = "Categorical" ~> "(" ~> """\d+""".r <~ ")" ^^ { n => MultinomialType(n.toInt) }
  def BooleanParser = "Boolean" ^^ { s => BooleanType }
  def dataType = CategoricalParser | BooleanParser

  def schemaDeclaration: Parser[SchemaDeclaration] =
    relationName ~ opt("?") ~ "(" ~ rep1sep(columnDeclaration, ",") ~ ")" ~ opt(dataType) ^^ {
      case (r ~ isQuery ~ "(" ~ attrs ~ ")" ~ vType) => {
        val vars = attrs.zipWithIndex map { case(x, i) => Variable(x.name, r, i) }
        var types = attrs map { case(x) => x.t }
        val variableType = vType match {
          case None => if (isQuery != None) Some(BooleanType) else None
          case Some(s) => Some(s)
        }
        SchemaDeclaration(Attribute(r, vars, types), (isQuery != None), variableType)
      }
    }

  def operator = "||" | "+" | "-" | "*" | "/"
  def castOp = "::"

  def variable = variableName ^^ { Variable(_, "", 0) }
  def columnConstant = constant ^^ { Constant(_, "", 0) }
  def variableOrConstant = columnConstant | variable
  val aggregationFunctions = Set("MAX", "SUM", "MIN")
  def inlineFunction = functionName ~ "(" ~ rep1sep(variableOrConstant, ",") ~ ")" ^^ {
    case (name ~ _ ~ args ~ _) => {
      if (aggregationFunctions contains name) InlineFunction(name, args, true)
      else InlineFunction(name, args, false)
    }
  }
  def columnVariable = columnConstant | inlineFunction | variable
  def operateOn = operator ~ columnVariable ^^ { case (v ~ o) => Operator(v,o) }
  def typecast = castOp ~ columnConstant ^^ { case (v ~ o) => Operator(v,o) }
  def operatorAndOperand = operateOn | typecast
  def expression = columnVariable ~ rep(operatorAndOperand) ^^ {
    case (v ~ opList) => {
      val variables = List(v) ++ (opList map (_.operand))
      val ops = opList map (_.operator)
      Expression(variables, ops, "", 0)
    }
  }

  // TODO support aggregate function syntax somehow
  def cqHead = relationName ~ "(" ~ repsep(expression, ",") ~ ")" ^^ {
      case (r ~ "(" ~ variableUses ~ ")") =>
        Atom(r, variableUses.zipWithIndex map {
          case (Expression(v,op,_,_),i) => {
            val vars = v map {
              case Variable(x,_,_) => Variable(x,r,i)
              case Constant(x,_,_) => Constant(x,r,i)
              case InlineFunction(x, args, a) => InlineFunction(x, args, a)
            }
            Expression(vars, op, r, i)
          }
        })
    }

  // TODO add conditional expressions for where clause
  def cqConditionalExpr = failure("No conditional expression supported yet")
  def cqBodyAtom: Parser[Atom] =
    ( relationName ~ "(" ~ repsep(expression, ",") ~ ")" ^^ {
        case (r ~ "(" ~ variableBindings ~ ")") =>
          Atom(r, variableBindings.zipWithIndex map {
          case (Expression(v,op,_,_),i) => {
            val vars = v map {
              case Variable(x,_,_) => Variable(x,r,i)
              case Constant(x,_,_) => Constant(x,r,i)
              case InlineFunction(x, args, _) => InlineFunction(x, args, false)
            }
            Expression(vars, op, r, i)
          }
        })
      }
    | cqConditionalExpr
    )
  def cqBody: Parser[List[Atom]] = rep1sep(cqBodyAtom, ",")

  // conditions
  def filterOperator = "LIKE" | ">" | "<" | ">=" | "<=" | "!=" | "="
  def condition = expression ~ filterOperator ~ expression ^^ {
    case (lhs ~ op ~ rhs) => {<commit_msg>Add bitwise and, is<commit_after>package org.deepdive.ddlog

// DeepDiveLog syntax
// See: https://docs.google.com/document/d/1SBIvvki3mnR28Mf0Pkin9w9mWNam5AA0SpIGj1ZN2c4

import scala.util.parsing.combinator._
import org.apache.commons.lang3.StringEscapeUtils
import scala.util.Try

// ***************************************
// * The union types for for the parser. *
// ***************************************
// case class Variable(varName : String, relName : String, index : Int )
// TODO make Atom a trait, and have multiple case classes, e.g., RelationAtom and CondExprAtom
// ddlog column variable type: constant or variable
sealed trait ColumnVariable
case class Variable(varName : String, relName : String, index : Int ) extends ColumnVariable
case class Constant(value : String, relName: String, index: Int) extends ColumnVariable
case class InlineFunction(functionName: String, args: List[ColumnVariable], isAggregation: Boolean) extends ColumnVariable
case class Expression(variables: List[ColumnVariable], ops: List[String], relName: String, index: Int)
case class Operator(operator: String, operand: ColumnVariable)

case class Atom(name : String, terms : List[Expression])
case class Attribute(name : String, terms : List[Variable], types : List[String])
case class ConjunctiveQuery(head: Atom, bodies: List[List[Atom]], conditions: List[Option[CompoundCondition]], isDistinct: Boolean)
case class Column(name : String, t : String)

// condition
case class BodyWithConditions(body: List[Atom], conditions: Option[CompoundCondition])

case class Condition(lhs: Expression, op: String, rhs: Expression)
case class CompoundCondition(conditions: List[List[Condition]])

// variable type
sealed trait VariableType {
  def cardinality: Long
}
case object BooleanType extends VariableType {
  def cardinality = 2
}
case class MultinomialType(numCategories: Int) extends VariableType {
  def cardinality = numCategories
}

sealed trait FactorWeight {
  def variables : List[String]
}

case class KnownFactorWeight(value: Double) extends FactorWeight {
  def variables = Nil
}
case class UnknownFactorWeight(variables: List[String]) extends FactorWeight

trait RelationType
case class RelationTypeDeclaration(names: List[String], types: List[String]) extends RelationType
case class RelationTypeAlias(likeRelationName: String) extends RelationType

trait FunctionImplementationDeclaration
case class RowWiseLineHandler(format: String, command: String) extends FunctionImplementationDeclaration

// Statements that will be parsed and compiled
trait Statement
case class SchemaDeclaration( a : Attribute , isQuery : Boolean, variableType : Option[VariableType]) extends Statement // atom and whether this is a query relation.
case class FunctionDeclaration( functionName: String, inputType: RelationType, outputType: RelationType, implementations: List[FunctionImplementationDeclaration], mode: String = null) extends Statement
case class ExtractionRule(q : ConjunctiveQuery, supervision: String = null) extends Statement // Extraction rule
case class FunctionCallRule(input : String, output : String, function : String) extends Statement // Extraction rule
case class InferenceRule(q : ConjunctiveQuery, weights : FactorWeight, semantics : String = "Imply", mode: String = null) extends Statement // Weighted rule


// Parser
class DeepDiveLogParser extends JavaTokenParsers {

  // JavaTokenParsers provides several useful number parsers:
  //   wholeNumber, decimalNumber, floatingPointNumber 
  def floatingPointNumberAsDouble = floatingPointNumber ^^ { _.toDouble }
  def stringLiteralAsString = stringLiteral ^^ {
    s => StringEscapeUtils.unescapeJava(
      s.stripPrefix("\"").stripSuffix("\""))
  }
  def stringLiteralAsSqlString = stringLiteral ^^ { s =>
    s"""'${s.stripPrefix("\"").stripSuffix("\"")}'"""
  }
  def constant = stringLiteralAsSqlString | wholeNumber | "TRUE" | "FALSE" | "NULL" | "TEXT" | "INT" | "BOOLEAN"

  // C/Java/Scala-style as well as shell script-style comments are supported
  // by treating them as whiteSpace
  protected override val whiteSpace = """(?:(?:^|\s+)#.*|//.*|(?m)/\*(\*(?!/)|[^*])*\*/|\s)+""".r

  // We just use Java identifiers to parse various names
  def relationName = ident
  def columnName   = ident
  def columnType   = ident ~ ("[]"?) ^^ {
      case ty ~ isArrayType => ty + isArrayType.getOrElse("")
    }
  def variableName = ident
  def functionName = ident
  def semanticType = ident
  def functionModeType = ident
  def inferenceModeType = ident

  def columnDeclaration: Parser[Column] =
    columnName ~ columnType ^^ {
      case(name ~ ty) => Column(name, ty)
    }

  def CategoricalParser = "Categorical" ~> "(" ~> """\d+""".r <~ ")" ^^ { n => MultinomialType(n.toInt) }
  def BooleanParser = "Boolean" ^^ { s => BooleanType }
  def dataType = CategoricalParser | BooleanParser

  def schemaDeclaration: Parser[SchemaDeclaration] =
    relationName ~ opt("?") ~ "(" ~ rep1sep(columnDeclaration, ",") ~ ")" ~ opt(dataType) ^^ {
      case (r ~ isQuery ~ "(" ~ attrs ~ ")" ~ vType) => {
        val vars = attrs.zipWithIndex map { case(x, i) => Variable(x.name, r, i) }
        var types = attrs map { case(x) => x.t }
        val variableType = vType match {
          case None => if (isQuery != None) Some(BooleanType) else None
          case Some(s) => Some(s)
        }
        SchemaDeclaration(Attribute(r, vars, types), (isQuery != None), variableType)
      }
    }

  def operator = "||" | "+" | "-" | "*" | "/" | "&"
  def castOp = "::"

  def variable = variableName ^^ { Variable(_, "", 0) }
  def columnConstant = constant ^^ { Constant(_, "", 0) }
  def variableOrConstant = columnConstant | variable
  val aggregationFunctions = Set("MAX", "SUM", "MIN")
  def inlineFunction = functionName ~ "(" ~ rep1sep(variableOrConstant, ",") ~ ")" ^^ {
    case (name ~ _ ~ args ~ _) => {
      if (aggregationFunctions contains name) InlineFunction(name, args, true)
      else InlineFunction(name, args, false)
    }
  }
  def columnVariable = columnConstant | inlineFunction | variable
  def operateOn = operator ~ columnVariable ^^ { case (v ~ o) => Operator(v,o) }
  def typecast = castOp ~ columnConstant ^^ { case (v ~ o) => Operator(v,o) }
  def operatorAndOperand = operateOn | typecast
  def expression = columnVariable ~ rep(operatorAndOperand) ^^ {
    case (v ~ opList) => {
      val variables = List(v) ++ (opList map (_.operand))
      val ops = opList map (_.operator)
      Expression(variables, ops, "", 0)
    }
  }

  // TODO support aggregate function syntax somehow
  def cqHead = relationName ~ "(" ~ repsep(expression, ",") ~ ")" ^^ {
      case (r ~ "(" ~ variableUses ~ ")") =>
        Atom(r, variableUses.zipWithIndex map {
          case (Expression(v,op,_,_),i) => {
            val vars = v map {
              case Variable(x,_,_) => Variable(x,r,i)
              case Constant(x,_,_) => Constant(x,r,i)
              case InlineFunction(x, args, a) => InlineFunction(x, args, a)
            }
            Expression(vars, op, r, i)
          }
        })
    }

  // TODO add conditional expressions for where clause
  def cqConditionalExpr = failure("No conditional expression supported yet")
  def cqBodyAtom: Parser[Atom] =
    ( relationName ~ "(" ~ repsep(expression, ",") ~ ")" ^^ {
        case (r ~ "(" ~ variableBindings ~ ")") =>
          Atom(r, variableBindings.zipWithIndex map {
          case (Expression(v,op,_,_),i) => {
            val vars = v map {
              case Variable(x,_,_) => Variable(x,r,i)
              case Constant(x,_,_) => Constant(x,r,i)
              case InlineFunction(x, args, _) => InlineFunction(x, args, false)
            }
            Expression(vars, op, r, i)
          }
        })
      }
    | cqConditionalExpr
    )
  def cqBody: Parser[List[Atom]] = rep1sep(cqBodyAtom, ",")

  // conditions
  def filterOperator = "LIKE" | ">" | "<" | ">=" | "<=" | "!=" | "=" | "IS" | "IS NOT"
  def condition = expression ~ filterOperator ~ expression ^^ {
    case (lhs ~ op ~ rhs) => {
{'fraction_non_alphanumeric': 0.13934528254916603, 'fraction_numerical': 0.0024894199651481204, 'mean_word_length': 3.7838642453111047, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 63, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3105924', 'n_tokens_mistral': 4797, 'n_tokens_neox': 4676, 'n_words': 1326}