<commit_before>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.semantic;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.ExtensionNotApplicableException;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.RecursionGuard;
import com.intellij.openapi.util.RecursionManager;
import com.intellij.openapi.util.UserDataHolder;
import com.intellij.psi.PsiElement;
import com.intellij.psi.util.CachedValueProvider.Result;
import com.intellij.psi.util.CachedValuesManager;
import com.intellij.psi.util.PsiModificationTracker;
import com.intellij.util.ObjectUtils;
import com.intellij.util.ProcessingContext;
import com.intellij.util.SharedProcessingContext;
import com.intellij.util.SmartList;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.IntObjectMap;
import com.intellij.util.containers.MultiMap;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;
import java.util.function.BiFunction;

import static java.util.Collections.emptyList;

/**
 * @author peter
 */
public final class SemServiceImpl extends SemService {
  private static final Logger LOG = Logger.getInstance(SemServiceImpl.class);

  private final Object myLock = ObjectUtils.sentinel(getClass().getName());
  private volatile MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> myProducers;
  private final Project myProject;
  private final CachedValuesManager myCVManager;

  public SemServiceImpl(Project project) {
    myProject = project;
    myCVManager = CachedValuesManager.getManager(myProject);
    SemContributor.EP_NAME.addChangeListener(() -> myProducers = null, project);
  }

  private MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> collectProducers() {
    MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> map = new MultiMap<>();

    SemRegistrar registrar = new SemRegistrar() {
      @Override
      public <T extends SemElement> void registerSemProvider(
        SemKey<T> key,
        BiFunction<? super PsiElement, ? super ProcessingContext, ? extends Collection<T>> provider) {

        map.putValue(key, provider::apply);
      }
    };

    for (SemContributorEP contributor : SemContributor.EP_NAME.getExtensionList()) {
      SemContributor semContributor;
      try {
        semContributor =
          myProject.instantiateExtensionWithPicoContainerOnlyIfNeeded(contributor.implementation, contributor.getPluginDescriptor());
      }
      catch (ProcessCanceledException e) {
        throw e;
      }
      catch (ExtensionNotApplicableException e) {
        continue;
      }
      catch (Exception e) {
        LOG.error(e);
        continue;
      }
      semContributor.registerSemProviders(registrar, myProject);
    }

    return map;
  }

  @Override
  public <T extends SemElement> List<T> getSemElements(@NotNull SemKey<T> key, @NotNull PsiElement psi) {
    SemCacheChunk chunk = myCVManager.getCachedValue((UserDataHolder)psi, () ->
      Result.create(new SemCacheChunk(), PsiModificationTracker.MODIFICATION_COUNT));
    List<T> cached = findCached(key, chunk);
    return cached != null ? cached : createSemElements(key, psi, chunk);
  }

  @SuppressWarnings("unchecked")
  @NotNull
  private <T extends SemElement> List<T> createSemElements(@NotNull SemKey<T> key, @NotNull PsiElement psi, SemCacheChunk chunk) {
    ensureInitialized();

    RecursionGuard.StackStamp stamp = RecursionManager.markStack();

    LinkedHashSet<T> result = new LinkedHashSet<>();
    Map<SemKey<?>, List<SemElement>> map = new THashMap<>();

    ProcessingContext processingContext = new ProcessingContext(new SharedProcessingContext());
    for (SemKey<?> each : key.getInheritors()) {
      List<SemElement> list = createSemElements(each, psi, processingContext);
      map.put(each, list);
      result.addAll((List<T>)list);
    }

    if (stamp.mayCacheNow()) {
      for (SemKey<?> semKey : map.keySet()) {
        chunk.putSemElements(semKey, map.get(semKey));
      }
    }

    return new ArrayList<>(result);
  }

  private void ensureInitialized() {
    if (myProducers == null) {
      synchronized (myLock) {
        if (myProducers == null) {<commit_msg>IDEA-CR-66370 UAST: cache UElement for SEM patterns and providers<commit_after>// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.semantic;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.extensions.ExtensionNotApplicableException;
import com.intellij.openapi.progress.ProcessCanceledException;
import com.intellij.openapi.project.Project;
import com.intellij.openapi.util.RecursionGuard;
import com.intellij.openapi.util.RecursionManager;
import com.intellij.openapi.util.UserDataHolder;
import com.intellij.psi.PsiElement;
import com.intellij.psi.util.CachedValueProvider.Result;
import com.intellij.psi.util.CachedValuesManager;
import com.intellij.psi.util.PsiModificationTracker;
import com.intellij.util.ObjectUtils;
import com.intellij.util.ProcessingContext;
import com.intellij.util.SmartList;
import com.intellij.util.containers.ContainerUtil;
import com.intellij.util.containers.IntObjectMap;
import com.intellij.util.containers.MultiMap;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;
import java.util.function.BiFunction;

import static java.util.Collections.emptyList;

/**
 * @author peter
 */
public final class SemServiceImpl extends SemService {
  private static final Logger LOG = Logger.getInstance(SemServiceImpl.class);

  private final Object myLock = ObjectUtils.sentinel(getClass().getName());
  private volatile MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> myProducers;
  private final Project myProject;
  private final CachedValuesManager myCVManager;

  public SemServiceImpl(Project project) {
    myProject = project;
    myCVManager = CachedValuesManager.getManager(myProject);
    SemContributor.EP_NAME.addChangeListener(() -> myProducers = null, project);
  }

  private MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> collectProducers() {
    MultiMap<SemKey<?>, BiFunction<PsiElement, ProcessingContext, Collection<? extends SemElement>>> map = new MultiMap<>();

    SemRegistrar registrar = new SemRegistrar() {
      @Override
      public <T extends SemElement> void registerSemProvider(
        SemKey<T> key,
        BiFunction<? super PsiElement, ? super ProcessingContext, ? extends Collection<T>> provider) {

        map.putValue(key, provider::apply);
      }
    };

    for (SemContributorEP contributor : SemContributor.EP_NAME.getExtensionList()) {
      SemContributor semContributor;
      try {
        semContributor =
          myProject.instantiateExtensionWithPicoContainerOnlyIfNeeded(contributor.implementation, contributor.getPluginDescriptor());
      }
      catch (ProcessCanceledException e) {
        throw e;
      }
      catch (ExtensionNotApplicableException e) {
        continue;
      }
      catch (Exception e) {
        LOG.error(e);
        continue;
      }
      semContributor.registerSemProviders(registrar, myProject);
    }

    return map;
  }

  @Override
  public <T extends SemElement> List<T> getSemElements(@NotNull SemKey<T> key, @NotNull PsiElement psi) {
    SemCacheChunk chunk = myCVManager.getCachedValue((UserDataHolder)psi, () ->
      Result.create(new SemCacheChunk(), PsiModificationTracker.MODIFICATION_COUNT));
    List<T> cached = findCached(key, chunk);
    return cached != null ? cached : createSemElements(key, psi, chunk);
  }

  @SuppressWarnings("unchecked")
  @NotNull
  private <T extends SemElement> List<T> createSemElements(@NotNull SemKey<T> key, @NotNull PsiElement psi, SemCacheChunk chunk) {
    ensureInitialized();

    RecursionGuard.StackStamp stamp = RecursionManager.markStack();

    LinkedHashSet<T> result = new LinkedHashSet<>();
    Map<SemKey<?>, List<SemElement>> map = new THashMap<>();

    ProcessingContext processingContext = new ProcessingContext();
    for (SemKey<?> each : key.getInheritors()) {
      List<SemElement> list = createSemElements(each, psi, processingContext);
      map.put(each, list);
      result.addAll((List<T>)list);
    }

    if (stamp.mayCacheNow()) {
      for (SemKey<?> semKey : map.keySet()) {
        chunk.putSemElements(semKey, map.get(semKey));
      }
    }

    return new ArrayList<>(result);
  }

  private void ensureInitialized() {
    if (myProducers == null) {
      synchronized (myLock) {
        if (myProducers == null) {
{'fraction_non_alphanumeric': 0.1063455405256791, 'fraction_numerical': 0.0027493676454415483, 'mean_word_length': 4.44877171959257, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '137899', 'n_tokens_mistral': 2716, 'n_tokens_neox': 2638, 'n_words': 579}