<commit_before>package common_filters

import (
	"encoding/json"
	"fmt"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	"strings"
	"time"
)

// Extract the filters from the arguments of a Local->Get or Local->GetMeta query.
func ExtractFilters(args map[string]interface{}) (*LocalFilter, error) {
	where, wherePresent := args["where"]
	if !wherePresent {
		// No filters; all is fine!
		return nil, nil
	} else {
		whereMap := where.(map[string]interface{}) // guaranteed by GraphQL to be a map.
		rootClause, err := parseClause(whereMap)
		if err != nil {
			return nil, err
		} else {
			return &LocalFilter{Root: rootClause}, nil
		}
	}
}

// Parse a single clause
func parseClause(args map[string]interface{}) (*Clause, error) {
	operator, operatorOk := args["operator"]
	if !operatorOk {
		return nil, fmt.Errorf("operand is missing in clause %s", jsonify(args))
	}

	var clause *Clause
	var err error

	switch operator {
	case "And":
		clause, err = parseOperandsOp(args, OperatorAnd)
	case "Or":
		clause, err = parseOperandsOp(args, OperatorOr)
	case "Not":
		clause, err = parseOperandsOp(args, OperatorOr)
	case "Equal":
		clause, err = parseCompareOp(args, OperatorEqual)
	case "NotEqual":
		clause, err = parseCompareOp(args, OperatorNotEqual)
	case "GreaterThan":
		clause, err = parseCompareOp(args, OperatorGreaterThan)
	case "GreaterThanEqual":
		clause, err = parseCompareOp(args, OperatorGreaterThanEqual)
	case "LessThan":
		clause, err = parseCompareOp(args, OperatorLessThan)
	case "LessThanEqual":
		clause, err = parseCompareOp(args, OperatorLessThanEqual)
	default:
		err = fmt.Errorf("Unknown operator '%s' in clause %s", operator, jsonify(args))
	}

	return clause, err
}

// Parses a 'comperator' filter
// Each of those has:
// 1. The operator applied (e.g. Equal, LessThanEqual)
// 2. A value (valueString, valueDate)
// 3. The path ["SomeAction", "color"]
func parseCompareOp(args map[string]interface{}, operator Operator) (*Clause, error) {
	path, err := parsePath(args)
	if err != nil {
		return nil, err
	}

	value, err := parseValue(args)
	if err != nil {
		return nil, err
	}

	return &Clause{
		Operator: operator,
		On:       path,
		Value:    value,
	}, nil
}

// Parse an 'operand' filter.
// One of those has:
// 1. The operator appied (e.g. And, Or)
// 2. The operands (e.g. a list of operands)
//    Each operand will be parsed as a new clause.
func parseOperandsOp(args map[string]interface{}, operator Operator) (*Clause, error) {
	path, err := parsePath(args)
	if err != nil {
		return nil, err
	}

	rawOperands, ok := args["operands"]
	if !ok {
		return nil, fmt.Errorf("No operands given in clause '%s'", jsonify(args))
	}

	rawOperandsList, ok := rawOperands.([]interface{})
	if !ok {
		return nil, fmt.Errorf("The operands in clause '%s' are not a list", jsonify(args))
	}

	var operands []Clause

	for _, rawOperand := range rawOperandsList {
		rawOperandMap, ok := rawOperand.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("The operand '%s' is not valid", jsonify(rawOperand))
		}

		operand, err := parseClause(rawOperandMap)

		if err != nil {
			return nil, err
		}

		operands = append(operands, *operand)
	}

	if len(operands) == 0 {
		return nil, fmt.Errorf("Empty clause given")
	}

	return &Clause{
		Operator: operator,
		On:       path,
		Operands: operands,
	}, nil
}

// Parses the path
// It parses an array of strings in this format
// [0] ClassName -> The root class name we're drilling down from
// [1] propertyName -> The property name we're interested in.
func parsePath(args map[string]interface{}) (*Path, error) {
	rawPath, ok := args["path"]
	if !ok {
		return nil, fmt.Errorf("Missing the 'path' field for the filter '%s'", jsonify(args))
	}

	pathElements, ok := rawPath.([]interface{})
	if !ok {
		return nil, fmt.Errorf("The 'path' field for the filter '%s' is not a list of strings", jsonify(args))
	}

	if len(pathElements) < 2 {
		return nil, fmt.Errorf("The 'path' field for the filter '%s' is empty! You need to specify at least a class and a property name", jsonify(args))
	}

	// The sentinel is used to bootstrap the inlined recursion.
	// we return sentinal.Child at the end.
	var sentinel Path
<commit_msg>Hum, there is no path in operator filters<commit_after>package common_filters

import (
	"encoding/json"
	"fmt"
	"github.com/creativesoftwarefdn/weaviate/database/schema"
	"strings"
	"time"
)

// Extract the filters from the arguments of a Local->Get or Local->GetMeta query.
func ExtractFilters(args map[string]interface{}) (*LocalFilter, error) {
	where, wherePresent := args["where"]
	if !wherePresent {
		// No filters; all is fine!
		return nil, nil
	} else {
		whereMap := where.(map[string]interface{}) // guaranteed by GraphQL to be a map.
		rootClause, err := parseClause(whereMap)
		if err != nil {
			return nil, err
		} else {
			return &LocalFilter{Root: rootClause}, nil
		}
	}
}

// Parse a single clause
func parseClause(args map[string]interface{}) (*Clause, error) {
	operator, operatorOk := args["operator"]
	if !operatorOk {
		return nil, fmt.Errorf("operand is missing in clause %s", jsonify(args))
	}

	var clause *Clause
	var err error

	switch operator {
	case "And":
		clause, err = parseOperandsOp(args, OperatorAnd)
	case "Or":
		clause, err = parseOperandsOp(args, OperatorOr)
	case "Not":
		clause, err = parseOperandsOp(args, OperatorOr)
	case "Equal":
		clause, err = parseCompareOp(args, OperatorEqual)
	case "NotEqual":
		clause, err = parseCompareOp(args, OperatorNotEqual)
	case "GreaterThan":
		clause, err = parseCompareOp(args, OperatorGreaterThan)
	case "GreaterThanEqual":
		clause, err = parseCompareOp(args, OperatorGreaterThanEqual)
	case "LessThan":
		clause, err = parseCompareOp(args, OperatorLessThan)
	case "LessThanEqual":
		clause, err = parseCompareOp(args, OperatorLessThanEqual)
	default:
		err = fmt.Errorf("Unknown operator '%s' in clause %s", operator, jsonify(args))
	}

	return clause, err
}

// Parses a 'comperator' filter
// Each of those has:
// 1. The operator applied (e.g. Equal, LessThanEqual)
// 2. A value (valueString, valueDate)
// 3. The path ["SomeAction", "color"]
func parseCompareOp(args map[string]interface{}, operator Operator) (*Clause, error) {
	path, err := parsePath(args)
	if err != nil {
		return nil, err
	}

	value, err := parseValue(args)
	if err != nil {
		return nil, err
	}

	return &Clause{
		Operator: operator,
		On:       path,
		Value:    value,
	}, nil
}

// Parse an 'operand' filter.
// One of those has:
// 1. The operator appied (e.g. And, Or)
// 2. The operands (e.g. a list of operands)
//    Each operand will be parsed as a new clause.
func parseOperandsOp(args map[string]interface{}, operator Operator) (*Clause, error) {
	rawOperands, ok := args["operands"]
	if !ok {
		return nil, fmt.Errorf("No operands given in clause '%s'", jsonify(args))
	}

	rawOperandsList, ok := rawOperands.([]interface{})
	if !ok {
		return nil, fmt.Errorf("The operands in clause '%s' are not a list", jsonify(args))
	}

	var operands []Clause

	for _, rawOperand := range rawOperandsList {
		rawOperandMap, ok := rawOperand.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("The operand '%s' is not valid", jsonify(rawOperand))
		}

		operand, err := parseClause(rawOperandMap)

		if err != nil {
			return nil, err
		}

		operands = append(operands, *operand)
	}

	if len(operands) == 0 {
		return nil, fmt.Errorf("Empty clause given")
	}

	return &Clause{
		Operator: operator,
		Operands: operands,
	}, nil
}

// Parses the path
// It parses an array of strings in this format
// [0] ClassName -> The root class name we're drilling down from
// [1] propertyName -> The property name we're interested in.
func parsePath(args map[string]interface{}) (*Path, error) {
	rawPath, ok := args["path"]
	if !ok {
		return nil, fmt.Errorf("Missing the 'path' field for the filter '%s'", jsonify(args))
	}

	pathElements, ok := rawPath.([]interface{})
	if !ok {
		return nil, fmt.Errorf("The 'path' field for the filter '%s' is not a list of strings", jsonify(args))
	}

	if len(pathElements) < 2 {
		return nil, fmt.Errorf("The 'path' field for the filter '%s' is empty! You need to specify at least a class and a property name", jsonify(args))
	}

	// The sentinel is used to bootstrap the inlined recursion.
	// we return sentinal.Child at the end.
	var sentinel Path

{'fraction_non_alphanumeric': 0.1318143862579029, 'fraction_numerical': 0.0021472026720744364, 'mean_word_length': 4.0173548773189705, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2978355', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2780, 'n_words': 926}