<commit_before>package query

import (
	"encoding/json"
	"errors"

	"github.com/degdb/degdb/protocol"
	"github.com/spaolacci/murmur3"
)

var (
	ErrNotImplemented = errors.New("query protocol type is not implemented")
	ErrUnRooted       = errors.New("unrooted queries are not implemented")
	ErrArrayOp        = errors.New("triplestore QueryArrayOp is not implemented")
)

func Parse(query string) ([]*protocol.Triple, error) {
	var filters []*protocol.Triple
	if err := json.Unmarshal([]byte(query), &filters); err != nil {
		return nil, err
	}
	return filters, nil
}

func ShardQueryByHash(step *protocol.ArrayOp) map[uint64]*protocol.ArrayOp {
	m := make(map[uint64]*protocol.ArrayOp)
	// TODO(d4l3k): better query hash splitting
	if step.Mode == protocol.OR && len(step.Triples) > 0 {
		for _, triple := range step.Triples {
			if len(triple.Subj) == 0 {
				// TODO(d4l3k): Handle people sneaking in unrooted queries.
				continue
			}
			hash := murmur3.Sum64([]byte(triple.Subj))
			m[hash] = step
		}
	} else {
		m[0] = step
	}
	return m
}<commit_msg>Changed how unrooted queries are detected<commit_after>package query

import (
	"encoding/json"
	"errors"

	"github.com/degdb/degdb/protocol"
	"github.com/spaolacci/murmur3"
)

var (
	ErrNotImplemented = errors.New("query protocol type is not implemented")
	ErrUnRooted       = errors.New("unrooted queries are not implemented")
	ErrArrayOp        = errors.New("triplestore QueryArrayOp is not implemented")
)

func Parse(query string) ([]*protocol.Triple, error) {
	var filters []*protocol.Triple
	if err := json.Unmarshal([]byte(query), &filters); err != nil {
		return nil, err
	}
	return filters, nil
}

func ShardQueryByHash(step *protocol.ArrayOp) map[uint64]*protocol.ArrayOp {
	m := make(map[uint64]*protocol.ArrayOp)
	// TODO(d4l3k): better query hash splitting
	var bad bool
	if len(step.Triples) > 0 {
		for _, triple := range step.Triples {
			if len(triple.Subj) == 0 {
				bad = true
				break
			}
			hash := murmur3.Sum64([]byte(triple.Subj))
			m[hash] = step
		}
	} else {
		bad = true
	}
	if bad {
		return map[uint64]*protocol.ArrayOp{0: step}
	}
	return m
}
{'fraction_non_alphanumeric': 0.13489461358313817, 'fraction_numerical': 0.01405152224824356, 'mean_word_length': 3.921658986175115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4067627', 'n_tokens_mistral': 813, 'n_tokens_neox': 755, 'n_words': 180}