<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package optimizer

import (
	"github.com/juju/errors"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/ddl"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/parser/coldef"
	"github.com/pingcap/tidb/rset/rsets"
	"github.com/pingcap/tidb/stmt"
	"github.com/pingcap/tidb/stmt/stmts"
	"github.com/pingcap/tidb/table"
)

func convertAssignment(converter *expressionConverter, v *ast.Assignment) (*expression.Assignment, error) {
	oldAssign := &expression.Assignment{
		ColName: joinColumnName(v.Column),
	}
	oldExpr, err := convertExpr(converter, v.Expr)
	if err != nil {
		return nil, errors.Trace(err)
	}
	oldAssign.Expr = oldExpr
	return oldAssign, nil
}

func convertInsert(converter *expressionConverter, v *ast.InsertStmt) (stmt.Statement, error) {
	insertValues := stmts.InsertValues{}
	insertValues.Priority = v.Priority
	tableName := v.Table.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName)
	insertValues.TableIdent = table.Ident{Schema: tableName.Schema, Name: tableName.Name}
	for _, val := range v.Columns {
		insertValues.ColNames = append(insertValues.ColNames, joinColumnName(val))
	}
	for _, row := range v.Lists {
		var oldRow []expression.Expression
		for _, val := range row {
			oldExpr, err := convertExpr(converter, val)
			if err != nil {
				return nil, errors.Trace(err)
			}
			oldRow = append(oldRow, oldExpr)
		}
		insertValues.Lists = append(insertValues.Lists, oldRow)
	}
	for _, assign := range v.Setlist {
		oldAssign, err := convertAssignment(converter, assign)
		if err != nil {
			return nil, errors.Trace(err)
		}
		insertValues.Setlist = append(insertValues.Setlist, oldAssign)
	}

	if v.Select != nil {
		var err error
		switch x := v.Select.(type) {
		case *ast.SelectStmt:
			insertValues.Sel, err = convertSelect(converter, x)
		case *ast.UnionStmt:
			insertValues.Sel, err = convertUnion(converter, x)
		}
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	if v.Replace {
		return &stmts.ReplaceIntoStmt{
			InsertValues: insertValues,
			Text:         v.Text(),
		}, nil
	}
	oldInsert := &stmts.InsertIntoStmt{
		InsertValues: insertValues,
		Text:         v.Text(),
	}
	for _, onDup := range v.OnDuplicate {
		oldOnDup, err := convertAssignment(converter, onDup)
		if err != nil {
			return nil, errors.Trace(err)
		}
		oldInsert.OnDuplicate = append(oldInsert.OnDuplicate, oldOnDup)
	}
	return oldInsert, nil
}

func convertDelete(converter *expressionConverter, v *ast.DeleteStmt) (*stmts.DeleteStmt, error) {
	oldDelete := &stmts.DeleteStmt{
		BeforeFrom:  v.BeforeFrom,
		Ignore:      v.Ignore,
		LowPriority: v.LowPriority,
		MultiTable:  v.MultiTable,
		Quick:       v.Quick,
		Text:        v.Text(),
	}
	oldRefs, err := convertJoin(converter, v.TableRefs.TableRefs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	oldDelete.Refs = oldRefs
	for _, val := range v.Tables {
		tableIdent := table.Ident{Schema: val.Schema, Name: val.Name}
		oldDelete.TableIdents = append(oldDelete.TableIdents, tableIdent)
	}
	if v.Where != nil {
		oldDelete.Where, err = convertExpr(converter, v.Where)
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	if v.Order != nil {
		orderRset := &rsets.OrderByRset{}
		for _, val := range v.Order {
			oldExpr, err := convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			orderItem := rsets.OrderByItem{Expr: oldExpr, Asc: !val.Desc}
			orderRset.By = append(orderRset.By, orderItem)
		}
		oldDelete.Order = orderRset
	}
	if v.Limit != nil {
		oldDelete.Limit = &rsets.LimitRset{Count: v.Limit.Count}
	}
	return oldDelete, nil
}

func convertUpdate(converter *expressionConverter, v *ast.UpdateStmt) (*stmts.UpdateStmt, error) {
	oldUpdate := &stmts.UpdateStmt{
		Ignore:        v.Ignore,
		MultipleTable: v.MultipleTable,
		LowPriority:   v.LowPriority,
		Text:          v.Text(),
	}
	var err error
	oldUpdate.TableRefs, err = convertJoin(converter, v.TableRefs.TableRefs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if v.Where != nil {
		oldUpdate.Where, err = convertExpr(converter, v.Where)
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	for _, val := range v.List {
		var oldAssign *expression.Assignment
		oldAssign, err = convertAssignment(converter, val)
		if err != nil {
			return nil, errors.Trace(err)
		}
		oldUpdate.List = append(oldUpdate.List, oldAssign)
	}
	if v.Order != nil {
		orderRset := &rsets.OrderByRset{}
		for _, val := range v.Order {
			oldExpr, err := convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			orderItem := rsets.OrderByItem{Expr: oldExpr, Asc: !val.Desc}
			orderRset.By = append(orderRset.By, orderItem)
		}
		oldUpdate.Order = orderRset
	}
	if v.Limit != nil {
		oldUpdate.Limit = &rsets.LimitRset{Count: v.Limit.Count}
	}
	return oldUpdate, nil
}

func convertSelect(converter *expressionConverter, s *ast.SelectStmt) (*stmts.SelectStmt, error) {
	oldSelect := &stmts.SelectStmt{
		Distinct: s.Distinct,
		Text:     s.Text(),
	}
	oldSelect.Fields = make([]*field.Field, len(s.Fields.Fields))
	for i, val := range s.Fields.Fields {
		oldField := &field.Field{}
		oldField.AsName = val.AsName.O
		var err error
		if val.Expr != nil {
			oldField.Expr, err = convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			if _, ok := oldField.Expr.(*expression.Ident); !ok && oldField.AsName == "" {<commit_msg>optimizer: add comments for set field text.<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package optimizer

import (
	"github.com/juju/errors"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/ddl"
	"github.com/pingcap/tidb/expression"
	"github.com/pingcap/tidb/field"
	"github.com/pingcap/tidb/model"
	"github.com/pingcap/tidb/parser/coldef"
	"github.com/pingcap/tidb/rset/rsets"
	"github.com/pingcap/tidb/stmt"
	"github.com/pingcap/tidb/stmt/stmts"
	"github.com/pingcap/tidb/table"
)

func convertAssignment(converter *expressionConverter, v *ast.Assignment) (*expression.Assignment, error) {
	oldAssign := &expression.Assignment{
		ColName: joinColumnName(v.Column),
	}
	oldExpr, err := convertExpr(converter, v.Expr)
	if err != nil {
		return nil, errors.Trace(err)
	}
	oldAssign.Expr = oldExpr
	return oldAssign, nil
}

func convertInsert(converter *expressionConverter, v *ast.InsertStmt) (stmt.Statement, error) {
	insertValues := stmts.InsertValues{}
	insertValues.Priority = v.Priority
	tableName := v.Table.TableRefs.Left.(*ast.TableSource).Source.(*ast.TableName)
	insertValues.TableIdent = table.Ident{Schema: tableName.Schema, Name: tableName.Name}
	for _, val := range v.Columns {
		insertValues.ColNames = append(insertValues.ColNames, joinColumnName(val))
	}
	for _, row := range v.Lists {
		var oldRow []expression.Expression
		for _, val := range row {
			oldExpr, err := convertExpr(converter, val)
			if err != nil {
				return nil, errors.Trace(err)
			}
			oldRow = append(oldRow, oldExpr)
		}
		insertValues.Lists = append(insertValues.Lists, oldRow)
	}
	for _, assign := range v.Setlist {
		oldAssign, err := convertAssignment(converter, assign)
		if err != nil {
			return nil, errors.Trace(err)
		}
		insertValues.Setlist = append(insertValues.Setlist, oldAssign)
	}

	if v.Select != nil {
		var err error
		switch x := v.Select.(type) {
		case *ast.SelectStmt:
			insertValues.Sel, err = convertSelect(converter, x)
		case *ast.UnionStmt:
			insertValues.Sel, err = convertUnion(converter, x)
		}
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	if v.Replace {
		return &stmts.ReplaceIntoStmt{
			InsertValues: insertValues,
			Text:         v.Text(),
		}, nil
	}
	oldInsert := &stmts.InsertIntoStmt{
		InsertValues: insertValues,
		Text:         v.Text(),
	}
	for _, onDup := range v.OnDuplicate {
		oldOnDup, err := convertAssignment(converter, onDup)
		if err != nil {
			return nil, errors.Trace(err)
		}
		oldInsert.OnDuplicate = append(oldInsert.OnDuplicate, oldOnDup)
	}
	return oldInsert, nil
}

func convertDelete(converter *expressionConverter, v *ast.DeleteStmt) (*stmts.DeleteStmt, error) {
	oldDelete := &stmts.DeleteStmt{
		BeforeFrom:  v.BeforeFrom,
		Ignore:      v.Ignore,
		LowPriority: v.LowPriority,
		MultiTable:  v.MultiTable,
		Quick:       v.Quick,
		Text:        v.Text(),
	}
	oldRefs, err := convertJoin(converter, v.TableRefs.TableRefs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	oldDelete.Refs = oldRefs
	for _, val := range v.Tables {
		tableIdent := table.Ident{Schema: val.Schema, Name: val.Name}
		oldDelete.TableIdents = append(oldDelete.TableIdents, tableIdent)
	}
	if v.Where != nil {
		oldDelete.Where, err = convertExpr(converter, v.Where)
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	if v.Order != nil {
		orderRset := &rsets.OrderByRset{}
		for _, val := range v.Order {
			oldExpr, err := convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			orderItem := rsets.OrderByItem{Expr: oldExpr, Asc: !val.Desc}
			orderRset.By = append(orderRset.By, orderItem)
		}
		oldDelete.Order = orderRset
	}
	if v.Limit != nil {
		oldDelete.Limit = &rsets.LimitRset{Count: v.Limit.Count}
	}
	return oldDelete, nil
}

func convertUpdate(converter *expressionConverter, v *ast.UpdateStmt) (*stmts.UpdateStmt, error) {
	oldUpdate := &stmts.UpdateStmt{
		Ignore:        v.Ignore,
		MultipleTable: v.MultipleTable,
		LowPriority:   v.LowPriority,
		Text:          v.Text(),
	}
	var err error
	oldUpdate.TableRefs, err = convertJoin(converter, v.TableRefs.TableRefs)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if v.Where != nil {
		oldUpdate.Where, err = convertExpr(converter, v.Where)
		if err != nil {
			return nil, errors.Trace(err)
		}
	}
	for _, val := range v.List {
		var oldAssign *expression.Assignment
		oldAssign, err = convertAssignment(converter, val)
		if err != nil {
			return nil, errors.Trace(err)
		}
		oldUpdate.List = append(oldUpdate.List, oldAssign)
	}
	if v.Order != nil {
		orderRset := &rsets.OrderByRset{}
		for _, val := range v.Order {
			oldExpr, err := convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			orderItem := rsets.OrderByItem{Expr: oldExpr, Asc: !val.Desc}
			orderRset.By = append(orderRset.By, orderItem)
		}
		oldUpdate.Order = orderRset
	}
	if v.Limit != nil {
		oldUpdate.Limit = &rsets.LimitRset{Count: v.Limit.Count}
	}
	return oldUpdate, nil
}

func convertSelect(converter *expressionConverter, s *ast.SelectStmt) (*stmts.SelectStmt, error) {
	oldSelect := &stmts.SelectStmt{
		Distinct: s.Distinct,
		Text:     s.Text(),
	}
	oldSelect.Fields = make([]*field.Field, len(s.Fields.Fields))
	for i, val := range s.Fields.Fields {
		oldField := &field.Field{}
		oldField.AsName = val.AsName.O
		var err error
		if val.Expr != nil {
			oldField.Expr, err = convertExpr(converter, val.Expr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			// TODO: handle parenthesesed column name expression, which should not set AsName.
			if _, ok := oldField.Expr.(*expression.Ident); !ok && oldField.AsName == "" {
{'fraction_non_alphanumeric': 0.13233839050131926, 'fraction_numerical': 0.0013192612137203166, 'mean_word_length': 3.704809930178433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7627334', 'n_tokens_mistral': 4530, 'n_tokens_neox': 4275, 'n_words': 989}