<commit_before>/*
 * Copyright (C) 2015 The Calrissian Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.calrissian.accumulorecipes.commons.support.qfd.planner.visitors;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import com.google.common.collect.Sets;
import org.calrissian.accumulorecipes.commons.support.qfd.planner.BaseTupleIndexKey;
import org.calrissian.accumulorecipes.commons.support.qfd.planner.TupleIndexKey;
import org.calrissian.mango.criteria.domain.AbstractKeyValueLeaf;
import org.calrissian.mango.criteria.domain.AndNode;
import org.calrissian.mango.criteria.domain.HasLeaf;
import org.calrissian.mango.criteria.domain.HasNotLeaf;
import org.calrissian.mango.criteria.domain.Leaf;
import org.calrissian.mango.criteria.domain.NegationLeaf;
import org.calrissian.mango.criteria.domain.Node;
import org.calrissian.mango.criteria.domain.NotEqualsLeaf;
import org.calrissian.mango.criteria.domain.OrNode;
import org.calrissian.mango.criteria.domain.ParentNode;
import org.calrissian.mango.criteria.support.NodeUtils;
import org.calrissian.mango.criteria.visitor.NodeVisitor;
import org.calrissian.mango.types.TypeRegistry;

public class CalculateShardsVisitor implements NodeVisitor {

    private final Map<TupleIndexKey,Set<String>> keysToShards;
    private Map<String, Set<TupleIndexKey>> accumuloKeyToTupleIndexKey = new HashMap<String, Set<TupleIndexKey>>();
    private TypeRegistry<String> registry;

    private Set<String> finalShards;

    public CalculateShardsVisitor(Map<TupleIndexKey,Set<String>> shards, TypeRegistry<String> registry) {
        this.keysToShards = shards;
        this.registry = registry;
        
        for (TupleIndexKey key : shards.keySet()) {
            Set<TupleIndexKey> tupleIndexKey = accumuloKeyToTupleIndexKey.get(key.getKey());
            if (tupleIndexKey == null) {
                tupleIndexKey = new HashSet<TupleIndexKey>();
                accumuloKeyToTupleIndexKey.put(key.getKey(), tupleIndexKey);
            }
            tupleIndexKey.add(key);
        }
    }

    @Override
    public void begin(ParentNode parentNode) {
        if(finalShards == null)
            finalShards = getShards(parentNode);
    }
<commit_msg>Adding a class-level comment for the CalculateShardsVisitor<commit_after>/*
 * Copyright (C) 2015 The Calrissian Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.calrissian.accumulorecipes.commons.support.qfd.planner.visitors;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import com.google.common.collect.Sets;
import org.calrissian.accumulorecipes.commons.support.qfd.planner.BaseTupleIndexKey;
import org.calrissian.accumulorecipes.commons.support.qfd.planner.TupleIndexKey;
import org.calrissian.mango.criteria.domain.AbstractKeyValueLeaf;
import org.calrissian.mango.criteria.domain.AndNode;
import org.calrissian.mango.criteria.domain.HasLeaf;
import org.calrissian.mango.criteria.domain.HasNotLeaf;
import org.calrissian.mango.criteria.domain.Leaf;
import org.calrissian.mango.criteria.domain.NegationLeaf;
import org.calrissian.mango.criteria.domain.Node;
import org.calrissian.mango.criteria.domain.NotEqualsLeaf;
import org.calrissian.mango.criteria.domain.OrNode;
import org.calrissian.mango.criteria.domain.ParentNode;
import org.calrissian.mango.criteria.support.NodeUtils;
import org.calrissian.mango.criteria.visitor.NodeVisitor;
import org.calrissian.mango.types.TypeRegistry;

/**
 * An optimization function that calculates the sets of shards for each section of a given query
 * tree and tries to eliminate as many shards as possible by determining shards that will
 * never match the given query. This is done by taking intersections of the shards in AND
 * trees and taking unions of shards in OR trees.
 */
public class CalculateShardsVisitor implements NodeVisitor {

    private final Map<TupleIndexKey,Set<String>> keysToShards;
    private Map<String, Set<TupleIndexKey>> accumuloKeyToTupleIndexKey = new HashMap<String, Set<TupleIndexKey>>();
    private TypeRegistry<String> registry;

    private Set<String> finalShards;

    public CalculateShardsVisitor(Map<TupleIndexKey,Set<String>> shards, TypeRegistry<String> registry) {
        this.keysToShards = shards;
        this.registry = registry;

        for (TupleIndexKey key : shards.keySet()) {
            Set<TupleIndexKey> tupleIndexKey = accumuloKeyToTupleIndexKey.get(key.getKey());
            if (tupleIndexKey == null) {
                tupleIndexKey = new HashSet<TupleIndexKey>();
                accumuloKeyToTupleIndexKey.put(key.getKey(), tupleIndexKey);
            }
            tupleIndexKey.add(key);
        }
    }

    @Override
    public void begin(ParentNode parentNode) {
        if(finalShards == null)
            finalShards = getShards(parentNode);
    }

{'fraction_non_alphanumeric': 0.09110848658680615, 'fraction_numerical': 0.0026995107136831447, 'mean_word_length': 4.222907488986784, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7397763', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1770, 'n_words': 428}