<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package wvlet.airframe.stream.spi

import java.util.Locale

import wvlet.surface.Surface

/**
  *
  */
object SQL {

  trait Schema
  case class AnonSchema(columns: Seq[Column])                extends Schema
  case class TableSchema(name: String, columns: Seq[Column]) extends Schema

  case class Column(name: String, columnType: DataType)

  sealed trait DataType
  case object IntegerType                                    extends DataType
  case object FloatType                                      extends DataType
  case object BooleanType                                    extends DataType
  case object StringType                                     extends DataType
  case object JSONType                                       extends DataType
  case object BinaryType                                     extends DataType
  case object NullType                                       extends DataType
  case object TimestampType                                  extends DataType
  case class ObjectType(surface: Surface)                    extends DataType
  case class AraryType(elemType: DataType)                   extends DataType
  case class MapType(keyType: DataType, valueType: DataType) extends DataType

  trait Relation
  case class AliasedRelation(relation: Relation, alias: String, columnNames: Option[Seq[String]]) extends Relation
  case class Values(rows: Seq[Expression])                                                        extends Relation
  case class Table(schema: TableSchema)                                                           extends Relation
  case class SubQuery(query: Query)                                                               extends Relation
  case class RawSQL(sql: String, schema: Schema)                                                  extends Relation
  case class Query(body: Relation, orderBy: Seq[SortItem], limit: Option[Expression])             extends Relation
  case class Project(in: Relation, schema: Seq[Expression])                                       extends Relation

  // Joins
  case class Join(joinType: JoinType, left: Relation, right: Relation, cond: JoinCriteria) extends Relation
  sealed trait JoinType
  case object InnerJoin      extends JoinType
  case object LeftOuterJoin  extends JoinType
  case object RightOuterJoin extends JoinType
  case object FullOuterJoin  extends JoinType
  case object CrossJoin      extends JoinType

  sealed trait JoinCriteria
  case object NaturalJoin                    extends JoinCriteria
  case class JoinUsing(columns: Seq[String]) extends JoinCriteria
  case class JoinOn(expr: Expression)        extends JoinCriteria

  case class Aggregate(in: Relation, keys: Seq[Expression], aggregate: Seq[AggregateExpression]) extends Relation
  case class Select(item: Seq[SelectItem],
                    isDistinct: Boolean,
                    from: Option[Relation],
                    where: Option[Expression],
                    groupBy: Seq[Expression],
                    having: Option[Expression],
                    orderBy: Seq[SortItem],
                    limit: Option[String])
      extends Relation

  trait SelectItem
  case class AllColumns(prefix: Option[QName]) extends SelectItem {
    override def toString = s"${prefix.map(x => s"${x}.*").getOrElse("*")}"
  }
  case class SingleColumn(expr: Expression, alias: Option[String]) extends SelectItem {
    override def toString = alias.map(a => s"${expr} as ${a}").getOrElse(s"${expr}")
  }

  case class SortItem(sortKey: Expression,
                      ordering: SortOrdering = Ascending,
                      nullOrdering: NullOrdering = UndefinedOrder)

  sealed trait SetOperation                                               extends Relation
  case class Intersect(relations: Seq[Relation], isDistinct: Boolean)     extends SetOperation
  case class Except(left: Relation, right: Relation, isDistinct: Boolean) extends SetOperation
  case class Union(relations: Seq[Relation], isDistinct: Boolean) extends SetOperation {
    override def toString = {
      val name = if (isDistinct) "Union" else "UnionAll"
      s"${name}(${relations.mkString(",")})"
    }
  }

  sealed trait SortOrdering
  case object Ascending  extends SortOrdering
  case object Descending extends SortOrdering

  sealed trait NullOrdering
  case object NullIsFirst    extends NullOrdering
  case object NullIsLast     extends NullOrdering
  case object UndefinedOrder extends NullOrdering

  // Window functions
  case class Window(partitionBy: Seq[Expression], orderBy: Seq[SortItem], frame: Option[WindowFrame])

  sealed trait FrameType
  case object RangeFrame extends FrameType
  case object RowsFrame  extends FrameType

  sealed trait FrameBound
  case object UnboundedPreceding extends FrameBound
  case object Preceding          extends FrameBound
  case object CurrentRow         extends FrameBound
  case object Following          extends FrameBound
  case object UnboundedFollowing extends FrameBound

  case class WindowFrame(frameType: FrameType, start: FrameBound, end: Option[FrameBound])
  trait Expression
  case class QName(parts: Seq[String]) extends Expression {
    override def toString = parts.mkString(".")
  }

  // Function
  case class FunctionCall(name: QName, args: Seq[Expression], isDistinct: Boolean, window: Option[Window])
      extends Expression {
    def functionName: String = name.toString.toLowerCase(Locale.US)
    override def toString    = s"FunctionCall(${name}, ${args.mkString(", ")}, distinct:${isDistinct}, window:${window})"
  }
  case class LambdaExpr(body: Expression, args: Seq[String]) extends Expression

  sealed class Ref(name: QName) extends Expression

  sealed trait Cond                                        extends Expression
  case object NoOp                                         extends Cond
  case class Eq(a: Expression, b: Expression)              extends Cond
  case class And(a: Expression, b: Expression)             extends Cond
  case class Or(a: Expression, b: Expression)              extends Cond
  case class Not(expr: Expression)                         extends Cond
  case class LessThan(a: Expression, b: Expression)        extends Cond
  case class LessThanOrEq(a: Expression, b: Expression)    extends Cond
  case class GreaterThan(a: Expression, b: Expression)     extends Cond
  case class GreaterThanOrEq(a: Expression, b: Expression) extends Cond
  case class Between(a: Expression, b: Expression)         extends Cond
  case class IsNull(a: Expression)                         extends Cond
  case class IsNotNull(a: Expression)                      extends Cond
  case class In(a: Expression, list: Seq[Expression])      extends Cond

  case class IfExpr(cond: Cond, onTrue: Expression, onFalse: Expression) extends Expression
  case class CaseExpr(operand: Option[Expression], whenClauses: Seq[Expression], defaultValue: Option[Expression])
      extends Expression
  case class WhenClause(operand: Expression, result: Expression) extends Expression

  sealed trait ArithmeticExpression
  // Arithmetic expr
  abstract sealed class BinaryExprType(symbol: String)
  case object Add      extends BinaryExprType("+")
  case object Subtract extends BinaryExprType("-")
  case object Multiply extends BinaryExprType("*")<commit_msg>Split schema from SQL expressions<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package wvlet.airframe.stream.spi

import java.util.Locale

import wvlet.surface.Surface

object SQLSchema {
  sealed trait Schema
  case class AnonSchema(columns: Seq[Column])                extends Schema
  case class TableSchema(name: String, columns: Seq[Column]) extends Schema

  case class Column(name: String, columnType: DataType)

  sealed trait DataType
  case object IntegerType                                    extends DataType
  case object FloatType                                      extends DataType
  case object BooleanType                                    extends DataType
  case object StringType                                     extends DataType
  case object JSONType                                       extends DataType
  case object BinaryType                                     extends DataType
  case object NullType                                       extends DataType
  case object TimestampType                                  extends DataType
  case class ObjectType(surface: Surface)                    extends DataType
  case class AraryType(elemType: DataType)                   extends DataType
  case class MapType(keyType: DataType, valueType: DataType) extends DataType
}

/**
  *
  */
object SQL {
  sealed trait Expression
  case class QName(parts: Seq[String]) extends Expression {
    override def toString = parts.mkString(".")
  }

  sealed trait Relation
  case class AliasedRelation(relation: Relation, alias: String, columnNames: Option[Seq[String]]) extends Relation
  case class Values(rows: Seq[Expression])                                                        extends Relation
  case class Table(name: QName)                                                                   extends Relation
  case class SubQuery(query: Query)                                                               extends Relation
  case class RawSQL(sql: String)                                                                  extends Relation
  case class Query(body: Relation, orderBy: Seq[SortItem], limit: Option[Expression])             extends Relation
  case class Project(in: Relation, schema: Seq[Expression])                                       extends Relation

  // Joins
  case class Join(joinType: JoinType, left: Relation, right: Relation, cond: JoinCriteria) extends Relation
  sealed trait JoinType
  case object InnerJoin      extends JoinType
  case object LeftOuterJoin  extends JoinType
  case object RightOuterJoin extends JoinType
  case object FullOuterJoin  extends JoinType
  case object CrossJoin      extends JoinType

  sealed trait JoinCriteria
  case object NaturalJoin                    extends JoinCriteria
  case class JoinUsing(columns: Seq[String]) extends JoinCriteria
  case class JoinOn(expr: Expression)        extends JoinCriteria

  case class Aggregate(in: Relation, keys: Seq[Expression], aggregate: Seq[AggregateExpression]) extends Relation
  case class Select(item: Seq[SelectItem],
                    isDistinct: Boolean,
                    from: Option[Relation],
                    where: Option[Expression],
                    groupBy: Seq[Expression],
                    having: Option[Expression],
                    orderBy: Seq[SortItem],
                    limit: Option[String])
      extends Relation

  sealed trait SelectItem
  case class AllColumns(prefix: Option[QName]) extends SelectItem {
    override def toString = s"${prefix.map(x => s"${x}.*").getOrElse("*")}"
  }
  case class SingleColumn(expr: Expression, alias: Option[String]) extends SelectItem {
    override def toString = alias.map(a => s"${expr} as ${a}").getOrElse(s"${expr}")
  }

  case class SortItem(sortKey: Expression,
                      ordering: SortOrdering = Ascending,
                      nullOrdering: NullOrdering = UndefinedOrder)

  sealed trait SetOperation                                               extends Relation
  case class Intersect(relations: Seq[Relation], isDistinct: Boolean)     extends SetOperation
  case class Except(left: Relation, right: Relation, isDistinct: Boolean) extends SetOperation
  case class Union(relations: Seq[Relation], isDistinct: Boolean) extends SetOperation {
    override def toString = {
      val name = if (isDistinct) "Union" else "UnionAll"
      s"${name}(${relations.mkString(",")})"
    }
  }

  sealed trait SortOrdering
  case object Ascending  extends SortOrdering
  case object Descending extends SortOrdering

  sealed trait NullOrdering
  case object NullIsFirst    extends NullOrdering
  case object NullIsLast     extends NullOrdering
  case object UndefinedOrder extends NullOrdering

  // Window functions
  case class Window(partitionBy: Seq[Expression], orderBy: Seq[SortItem], frame: Option[WindowFrame])

  sealed trait FrameType
  case object RangeFrame extends FrameType
  case object RowsFrame  extends FrameType

  sealed trait FrameBound
  case object UnboundedPreceding extends FrameBound
  case object Preceding          extends FrameBound
  case object CurrentRow         extends FrameBound
  case object Following          extends FrameBound
  case object UnboundedFollowing extends FrameBound

  case class WindowFrame(frameType: FrameType, start: FrameBound, end: Option[FrameBound])

  // Function
  case class FunctionCall(name: QName, args: Seq[Expression], isDistinct: Boolean, window: Option[Window])
      extends Expression {
    def functionName: String = name.toString.toLowerCase(Locale.US)
    override def toString    = s"FunctionCall(${name}, ${args.mkString(", ")}, distinct:${isDistinct}, window:${window})"
  }
  case class LambdaExpr(body: Expression, args: Seq[String]) extends Expression

  class Ref(name: QName) extends Expression

  sealed trait Cond                                        extends Expression
  case object NoOp                                         extends Cond
  case class Eq(a: Expression, b: Expression)              extends Cond
  case class And(a: Expression, b: Expression)             extends Cond
  case class Or(a: Expression, b: Expression)              extends Cond
  case class Not(expr: Expression)                         extends Cond
  case class LessThan(a: Expression, b: Expression)        extends Cond
  case class LessThanOrEq(a: Expression, b: Expression)    extends Cond
  case class GreaterThan(a: Expression, b: Expression)     extends Cond
  case class GreaterThanOrEq(a: Expression, b: Expression) extends Cond
  case class Between(a: Expression, b: Expression)         extends Cond
  case class IsNull(a: Expression)                         extends Cond
  case class IsNotNull(a: Expression)                      extends Cond
  case class In(a: Expression, list: Seq[Expression])      extends Cond

  case class IfExpr(cond: Cond, onTrue: Expression, onFalse: Expression) extends Expression
  case class CaseExpr(operand: Option[Expression], whenClauses: Seq[Expression], defaultValue: Option[Expression])
      extends Expression
  case class WhenClause(operand: Expression, result: Expression) extends Expression

  sealed trait ArithmeticExpression
  // Arithmetic expr
  abstract sealed class BinaryExprType(symbol: String)
  case object Add      extends BinaryExprType("+")
  case object Subtract extends BinaryExprType("-")
  case object Multiply extends BinaryExprType("*")
{'fraction_non_alphanumeric': 0.06462628051093967, 'fraction_numerical': 0.0005058808650562792, 'mean_word_length': 2.5348681269557445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3872349', 'n_tokens_mistral': 3779, 'n_tokens_neox': 3778, 'n_words': 1390}