<commit_before>/*
 * Copyright 2014 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.netflix.spinnaker.cats.provider;

import com.netflix.spectator.api.Counter;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.cats.agent.CacheResult;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.cache.CacheFilter;
import com.netflix.spinnaker.cats.cache.DefaultCacheData;
import com.netflix.spinnaker.cats.cache.WriteableCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 * An implementation of ProviderCache that writes through to a provided backing
 * WriteableCache.
 *
 * This implementation will handle aggregating results from multiple sources, and
 * the view methods will merge relationships from all sources into a single relationship.
 */
public class DefaultProviderCache implements ProviderCache {

    private final Logger log = LoggerFactory.getLogger(DefaultProviderCache.class);

    private static final String ALL_ID = "_ALL_"; //dirty = true
    private static final Map<String, Object> ALL_ATTRIBUTE = Collections.unmodifiableMap(new HashMap<String, Object>(1) {{
        put("id", ALL_ID);
    }});

    private final WriteableCache backingStore;
    private final Registry registry;

    public DefaultProviderCache(WriteableCache backingStore, Registry registry) {
        this.backingStore = backingStore;
        this.registry = registry;
    }

    @Override
    public CacheData get(String type, String id) {
        return get(type, id, null);
    }

    @Override
    public CacheData get(String type, String id, CacheFilter cacheFilter) {
        validateTypes(type);
        if (ALL_ID.equals(id)) {
            return null;
        }
        CacheData item = backingStore.get(type, id, cacheFilter);
        if (item == null) {
            return null;
        }

        return mergeRelationships(item);
    }

    @Override
    public Collection<CacheData> getAll(String type) {
        return getAll(type, (CacheFilter) null);
    }

    @Override
    public Collection<CacheData> getAll(String type, CacheFilter cacheFilter) {
        validateTypes(type);
        Collection<CacheData> all = backingStore.getAll(type, cacheFilter);
        return buildResponse(all);
    }

    @Override
    public Collection<CacheData> getAll(String type, Collection<String> identifiers) {
        return getAll(type, identifiers, null);
    }

    @Override
    public Collection<CacheData> getAll(String type, Collection<String> identifiers, CacheFilter cacheFilter) {
        validateTypes(type);
        Collection<CacheData> byId = backingStore.getAll(type, identifiers, cacheFilter);
        return buildResponse(byId);
    }

    @Override
    public Collection<CacheData> getAll(String type, String... identifiers) {
        return getAll(type, Arrays.asList(identifiers));
    }

    @Override
    public Collection<String> existingIdentifiers(String type, Collection<String> identifiers) {
        Set<String> existing = new HashSet<>(backingStore.existingIdentifiers(type, identifiers));
        existing.remove(ALL_ID);
        return existing;
    }

    @Override
    public Collection<String> getIdentifiers(String type) {
        validateTypes(type);
        Set<String> identifiers = new HashSet<>(backingStore.getIdentifiers(type));
        identifiers.remove(ALL_ID);
        return identifiers;
    }

    @Override
    public Collection<String> filterIdentifiers(String type, String glob) {
        validateTypes(type);
        Set<String> identifiers = new HashSet<>(backingStore.filterIdentifiers(type, glob));
        identifiers.remove(ALL_ID);

        return identifiers;
    }

    @Override
    public void putCacheResult(String sourceAgentType, Collection<String> authoritativeTypes, CacheResult cacheResult) {
        Set<String> allTypes = new HashSet<>(cacheResult.getCacheResults().keySet());
        allTypes.addAll(authoritativeTypes);
        allTypes.addAll(cacheResult.getEvictions().keySet());
        validateTypes(allTypes);

        Map<String, Collection<String>> evictions = new HashMap<>();

        for (String type : allTypes) {
            final boolean authoritative = authoritativeTypes.contains(type);
            final Set<String> previousIdentifiers = getExistingSourceIdentifiers(type, sourceAgentType);
            final int originalSetSize = previousIdentifiers.size();
            int newItems = 0;
            final int cacheResultSize;
            if (cacheResult.getCacheResults().containsKey(type)) {
                final Collection<CacheData> cacheResultsForType = cacheResult.getCacheResults().get(type);
                cacheResultSize = cacheResultsForType.size();
                cacheDataType(type, sourceAgentType, cacheResultsForType);
                for (CacheData data : cacheResultsForType) {
                    if (!previousIdentifiers.remove(data.getId())) {
                      newItems++;
                    }
                }
            } else {
              cacheResultSize = 0;
            }
            final Set<String> evictionsForType = new HashSet<>();
            final int explicitEvictionSize;
            if (cacheResult.getEvictions().containsKey(type)) {
                evictionsForType.addAll(cacheResult.getEvictions().get(type));
                explicitEvictionSize = evictions.size();
            } else {
                explicitEvictionSize = 0;
            }

            final int noLongerPresentItemsCount;
            if (authoritative) {
                noLongerPresentItemsCount = previousIdentifiers.size();
                evictionsForType.addAll(previousIdentifiers);
            } else {
                noLongerPresentItemsCount = -1;
            }

            if (!evictionsForType.isEmpty()) {
                evictions.put(type, evictionsForType);
            }

            final int totalEvictions = evictionsForType.size();
            final int changedItems = totalEvictions + newItems;<commit_msg>fix(perf): fix performance regression due to default provider cache metrics causing extra redis reads. (#2816)<commit_after>/*
 * Copyright 2014 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.netflix.spinnaker.cats.provider;

import com.netflix.spectator.api.Counter;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.cats.agent.CacheResult;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.cache.CacheFilter;
import com.netflix.spinnaker.cats.cache.DefaultCacheData;
import com.netflix.spinnaker.cats.cache.WriteableCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 * An implementation of ProviderCache that writes through to a provided backing
 * WriteableCache.
 *
 * This implementation will handle aggregating results from multiple sources, and
 * the view methods will merge relationships from all sources into a single relationship.
 */
public class DefaultProviderCache implements ProviderCache {

    private final Logger log = LoggerFactory.getLogger(DefaultProviderCache.class);

    private static final String ALL_ID = "_ALL_"; //dirty = true
    private static final Map<String, Object> ALL_ATTRIBUTE = Collections.unmodifiableMap(new HashMap<String, Object>(1) {{
        put("id", ALL_ID);
    }});

    private final WriteableCache backingStore;
    private final Registry registry;

    public DefaultProviderCache(WriteableCache backingStore, Registry registry) {
        this.backingStore = backingStore;
        this.registry = registry;
    }

    @Override
    public CacheData get(String type, String id) {
        return get(type, id, null);
    }

    @Override
    public CacheData get(String type, String id, CacheFilter cacheFilter) {
        validateTypes(type);
        if (ALL_ID.equals(id)) {
            return null;
        }
        CacheData item = backingStore.get(type, id, cacheFilter);
        if (item == null) {
            return null;
        }

        return mergeRelationships(item);
    }

    @Override
    public Collection<CacheData> getAll(String type) {
        return getAll(type, (CacheFilter) null);
    }

    @Override
    public Collection<CacheData> getAll(String type, CacheFilter cacheFilter) {
        validateTypes(type);
        Collection<CacheData> all = backingStore.getAll(type, cacheFilter);
        return buildResponse(all);
    }

    @Override
    public Collection<CacheData> getAll(String type, Collection<String> identifiers) {
        return getAll(type, identifiers, null);
    }

    @Override
    public Collection<CacheData> getAll(String type, Collection<String> identifiers, CacheFilter cacheFilter) {
        validateTypes(type);
        Collection<CacheData> byId = backingStore.getAll(type, identifiers, cacheFilter);
        return buildResponse(byId);
    }

    @Override
    public Collection<CacheData> getAll(String type, String... identifiers) {
        return getAll(type, Arrays.asList(identifiers));
    }

    @Override
    public Collection<String> existingIdentifiers(String type, Collection<String> identifiers) {
        Set<String> existing = new HashSet<>(backingStore.existingIdentifiers(type, identifiers));
        existing.remove(ALL_ID);
        return existing;
    }

    @Override
    public Collection<String> getIdentifiers(String type) {
        validateTypes(type);
        Set<String> identifiers = new HashSet<>(backingStore.getIdentifiers(type));
        identifiers.remove(ALL_ID);
        return identifiers;
    }

    @Override
    public Collection<String> filterIdentifiers(String type, String glob) {
        validateTypes(type);
        Set<String> identifiers = new HashSet<>(backingStore.filterIdentifiers(type, glob));
        identifiers.remove(ALL_ID);

        return identifiers;
    }

    @Override
    public void putCacheResult(String sourceAgentType, Collection<String> authoritativeTypes, CacheResult cacheResult) {
        Set<String> allTypes = new HashSet<>(cacheResult.getCacheResults().keySet());
        allTypes.addAll(authoritativeTypes);
        allTypes.addAll(cacheResult.getEvictions().keySet());
        validateTypes(allTypes);

        Map<String, Collection<String>> evictions = new HashMap<>();

        for (String type : allTypes) {
            final boolean authoritative = authoritativeTypes.contains(type);
            final Set<String> previousIdentifiers = authoritative ? getExistingSourceIdentifiers(type, sourceAgentType) : Collections.emptySet();
            final int originalSetSize = authoritative ? previousIdentifiers.size() : -1;
            int newItems = 0;
            final int cacheResultSize;
            if (cacheResult.getCacheResults().containsKey(type)) {
                final Collection<CacheData> cacheResultsForType = cacheResult.getCacheResults().get(type);
                cacheResultSize = cacheResultsForType.size();
                cacheDataType(type, sourceAgentType, cacheResultsForType);
                if (authoritative) {
                    for (CacheData data : cacheResultsForType) {
                        if (!previousIdentifiers.remove(data.getId())) {
                            newItems++;
                        }
                    }
                }
            } else {
              cacheResultSize = 0;
            }
            final Set<String> evictionsForType = new HashSet<>();
            final int explicitEvictionSize;
            if (cacheResult.getEvictions().containsKey(type)) {
                evictionsForType.addAll(cacheResult.getEvictions().get(type));
                explicitEvictionSize = evictions.size();
            } else {
                explicitEvictionSize = 0;
            }

            final int noLongerPresentItemsCount;
            if (authoritative) {
                noLongerPresentItemsCount = previousIdentifiers.size();
                evictionsForType.addAll(previousIdentifiers);
            } else {
                noLongerPresentItemsCount = -1;
            }

            if (!evictionsForType.isEmpty()) {
                evictions.put(type, evictionsForType);
            }

            final int totalEvictions = evictionsForType.size();
            final int changedItems = totalEvictions + newItems;
{'fraction_non_alphanumeric': 0.09006978772899099, 'fraction_numerical': 0.002544344286129689, 'mean_word_length': 2.901588201928531, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2502236', 'n_tokens_mistral': 3598, 'n_tokens_neox': 3470, 'n_words': 913}