<commit_before>package com.intellij.refactoring.util.duplicates;

import com.intellij.lang.ASTNode;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.psi.*;
import com.intellij.psi.controlFlow.*;
import com.intellij.psi.impl.source.PsiImmediateClassType;
import com.intellij.psi.search.LocalSearchScope;
import com.intellij.psi.search.searches.ReferencesSearch;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.psi.util.PsiUtil;
import com.intellij.psi.util.TypeConversionUtil;
import com.intellij.refactoring.util.RefactoringUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.containers.IntArrayList;
import org.jetbrains.annotations.Nullable;

import java.util.*;

/**
 * @author dsl
 */
public class DuplicatesFinder {
  private static final Logger LOG = Logger.getInstance("#com.intellij.refactoring.util.duplicates.DuplicatesFinder");
  private static final Key<Pair<PsiVariable, PsiType>> PARAMETER = Key.create("PARAMETER");
  private final PsiElement[] myPattern;
  private List<? extends PsiVariable> myParameters;
  private final List<? extends PsiVariable> myOutputParameters;
  private final List<PsiElement> myPatternAsList;
  private boolean myMultipleExitPoints = false;

  public DuplicatesFinder(PsiElement[] pattern,
                          List<? extends PsiVariable> parameters,
                          List<? extends PsiVariable> outputParameters
  ) {
    LOG.assertTrue(pattern.length > 0);
    myPattern = pattern;
    myPatternAsList = Arrays.asList(myPattern);
    myParameters = parameters;
    myOutputParameters = outputParameters;

    final PsiElement codeFragment = ControlFlowUtil.findCodeFragment(pattern[0]);
    try {
      final ControlFlow controlFlow = ControlFlowFactory.getInstance(codeFragment.getProject()).getControlFlow(codeFragment, new LocalsControlFlowPolicy(codeFragment), false);

      int startOffset;
      int i = 0;
      do {
        startOffset = controlFlow.getStartOffset(pattern[i++]);
      } while(startOffset < 0 && i < pattern.length);

      int endOffset;
      int j = pattern.length - 1;
      do {
        endOffset = controlFlow.getEndOffset(pattern[j--]);
      } while(endOffset < 0 && j >= 0);

      IntArrayList exitPoints = new IntArrayList();
      final Collection<PsiStatement> exitStatements = ControlFlowUtil
          .findExitPointsAndStatements(controlFlow, startOffset, endOffset, exitPoints, ControlFlowUtil.DEFAULT_EXIT_STATEMENTS_CLASSES);
      myMultipleExitPoints = exitPoints.size() > 1;

      if (myMultipleExitPoints) {
        myParameters = new ArrayList<PsiVariable>(myParameters); //to allow removal of items from the otherwise readonly list
        removeParametersUsedInExitsOnly(codeFragment, exitStatements, controlFlow, startOffset, endOffset);
      }
    }
    catch (AnalysisCanceledException e) {
    }
  }

  private void removeParametersUsedInExitsOnly(PsiElement codeFragment, Collection<PsiStatement> exitStatements, ControlFlow controlFlow, int startOffset, int endOffset) {
    LocalSearchScope scope = new LocalSearchScope(codeFragment);
    Variables:
    for (Iterator<? extends PsiVariable> iterator = myParameters.iterator(); iterator.hasNext();) {
      PsiVariable variable = iterator.next();
      Collection<PsiReference> refs = ReferencesSearch.search(variable, scope).findAll();
      for (PsiReference ref : refs) {
        PsiElement element = ref.getElement();
        int elementOffset = controlFlow.getStartOffset(element);
        if (elementOffset >= startOffset && elementOffset <= endOffset) {
          if (!isInExitStatements(element, exitStatements)) continue Variables;
        }
      }
      iterator.remove();
    }
  }

  private static boolean isInExitStatements(PsiElement element, Collection<PsiStatement> exitStatements) {
    for (PsiStatement exitStatement : exitStatements) {
      if (PsiTreeUtil.isAncestor(exitStatement, element, false)) return true;
    }
    return false;
  }


  public List<Match> findDuplicates(PsiElement scope) {
    annotatePattern();
    final ArrayList<Match> result = new ArrayList<Match>();
    findPatternOccurrences(result, scope);
    deannotatePattern();
    return result;
  }

  private void annotatePattern() {
    for (final PsiElement patternComponent : myPattern) {
      patternComponent.accept(new JavaRecursiveElementVisitor() {
        @Override public void visitReferenceElement(PsiJavaCodeReferenceElement reference) {
          final PsiElement element = reference.resolve();
          if (element instanceof PsiVariable && myParameters.contains(element)) {
            final PsiVariable variable = (PsiVariable)element;
            PsiType type = variable.getType();
            final PsiMethodCallExpression methodCallExpression = PsiTreeUtil.getParentOfType(reference, PsiMethodCallExpression.class);
            if (methodCallExpression != null) {
              int idx = ArrayUtil.find(methodCallExpression.getArgumentList().getExpressions(), reference);
              if (idx > -1) {
                final PsiMethod psiMethod = methodCallExpression.resolveMethod();
                if (psiMethod != null) {
                  final PsiParameter[] parameters = psiMethod.getParameterList().getParameters();
                  if (idx >= parameters.length) { //vararg parameter
                    idx = parameters.length - 1;
                  }
                  if (idx > 0) { //incomplete code
                    type = parameters[idx].getType();
                  }
                  if (type instanceof PsiEllipsisType) {
                    type = ((PsiEllipsisType)type).getComponentType();
                  }
                }
              }
            }
            reference.putUserData(PARAMETER, Pair.create(variable, type));
          }
          if (element instanceof PsiVariable && myOutputParameters.contains(element)) {
            final PsiVariable psiVariable = (PsiVariable)element;
            reference.putUserData(PARAMETER, Pair.create(psiVariable, psiVariable.getType()));
          }
          PsiElement qualifier = reference.getQualifier();
          if (qualifier != null) {
            qualifier.accept(this);
          }
        }
      });
    }
  }
<commit_msg>IDEADEV-26386<commit_after>package com.intellij.refactoring.util.duplicates;

import com.intellij.lang.ASTNode;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.Comparing;
import com.intellij.openapi.util.Key;
import com.intellij.openapi.util.Pair;
import com.intellij.psi.*;
import com.intellij.psi.controlFlow.*;
import com.intellij.psi.impl.source.PsiImmediateClassType;
import com.intellij.psi.search.LocalSearchScope;
import com.intellij.psi.search.searches.ReferencesSearch;
import com.intellij.psi.util.PsiTreeUtil;
import com.intellij.psi.util.PsiUtil;
import com.intellij.psi.util.TypeConversionUtil;
import com.intellij.refactoring.util.RefactoringUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.containers.IntArrayList;
import org.jetbrains.annotations.Nullable;

import java.util.*;

/**
 * @author dsl
 */
public class DuplicatesFinder {
  private static final Logger LOG = Logger.getInstance("#com.intellij.refactoring.util.duplicates.DuplicatesFinder");
  private static final Key<Pair<PsiVariable, PsiType>> PARAMETER = Key.create("PARAMETER");
  private final PsiElement[] myPattern;
  private List<? extends PsiVariable> myParameters;
  private final List<? extends PsiVariable> myOutputParameters;
  private final List<PsiElement> myPatternAsList;
  private boolean myMultipleExitPoints = false;

  public DuplicatesFinder(PsiElement[] pattern,
                          List<? extends PsiVariable> parameters,
                          List<? extends PsiVariable> outputParameters
  ) {
    LOG.assertTrue(pattern.length > 0);
    myPattern = pattern;
    myPatternAsList = Arrays.asList(myPattern);
    myParameters = parameters;
    myOutputParameters = outputParameters;

    final PsiElement codeFragment = ControlFlowUtil.findCodeFragment(pattern[0]);
    try {
      final ControlFlow controlFlow = ControlFlowFactory.getInstance(codeFragment.getProject()).getControlFlow(codeFragment, new LocalsControlFlowPolicy(codeFragment), false);

      int startOffset;
      int i = 0;
      do {
        startOffset = controlFlow.getStartOffset(pattern[i++]);
      } while(startOffset < 0 && i < pattern.length);

      int endOffset;
      int j = pattern.length - 1;
      do {
        endOffset = controlFlow.getEndOffset(pattern[j--]);
      } while(endOffset < 0 && j >= 0);

      IntArrayList exitPoints = new IntArrayList();
      final Collection<PsiStatement> exitStatements = ControlFlowUtil
          .findExitPointsAndStatements(controlFlow, startOffset, endOffset, exitPoints, ControlFlowUtil.DEFAULT_EXIT_STATEMENTS_CLASSES);
      myMultipleExitPoints = exitPoints.size() > 1;

      if (myMultipleExitPoints) {
        myParameters = new ArrayList<PsiVariable>(myParameters); //to allow removal of items from the otherwise readonly list
        removeParametersUsedInExitsOnly(codeFragment, exitStatements, controlFlow, startOffset, endOffset);
      }
    }
    catch (AnalysisCanceledException e) {
    }
  }

  private void removeParametersUsedInExitsOnly(PsiElement codeFragment, Collection<PsiStatement> exitStatements, ControlFlow controlFlow, int startOffset, int endOffset) {
    LocalSearchScope scope = new LocalSearchScope(codeFragment);
    Variables:
    for (Iterator<? extends PsiVariable> iterator = myParameters.iterator(); iterator.hasNext();) {
      PsiVariable variable = iterator.next();
      Collection<PsiReference> refs = ReferencesSearch.search(variable, scope).findAll();
      for (PsiReference ref : refs) {
        PsiElement element = ref.getElement();
        int elementOffset = controlFlow.getStartOffset(element);
        if (elementOffset >= startOffset && elementOffset <= endOffset) {
          if (!isInExitStatements(element, exitStatements)) continue Variables;
        }
      }
      iterator.remove();
    }
  }

  private static boolean isInExitStatements(PsiElement element, Collection<PsiStatement> exitStatements) {
    for (PsiStatement exitStatement : exitStatements) {
      if (PsiTreeUtil.isAncestor(exitStatement, element, false)) return true;
    }
    return false;
  }


  public List<Match> findDuplicates(PsiElement scope) {
    annotatePattern();
    final ArrayList<Match> result = new ArrayList<Match>();
    findPatternOccurrences(result, scope);
    deannotatePattern();
    return result;
  }

  private void annotatePattern() {
    for (final PsiElement patternComponent : myPattern) {
      patternComponent.accept(new JavaRecursiveElementVisitor() {
        @Override public void visitReferenceElement(PsiJavaCodeReferenceElement reference) {
          final PsiElement element = reference.resolve();
          if (element instanceof PsiVariable && myParameters.contains(element)) {
            final PsiVariable variable = (PsiVariable)element;
            PsiType type = variable.getType();
            final PsiMethodCallExpression methodCallExpression = PsiTreeUtil.getParentOfType(reference, PsiMethodCallExpression.class);
            if (methodCallExpression != null) {
              int idx = ArrayUtil.find(methodCallExpression.getArgumentList().getExpressions(), reference);
              if (idx > -1) {
                final PsiMethod psiMethod = methodCallExpression.resolveMethod();
                if (psiMethod != null) {
                  final PsiParameter[] parameters = psiMethod.getParameterList().getParameters();
                  if (idx >= parameters.length) { //vararg parameter
                    idx = parameters.length - 1;
                    if (idx >= 0) { //incomplete code
                      type = parameters[idx].getType();
                    }
                  }
                  if (type instanceof PsiEllipsisType) {
                    type = ((PsiEllipsisType)type).getComponentType();
                  }
                }
              }
            }
            reference.putUserData(PARAMETER, Pair.create(variable, type));
          }
          if (element instanceof PsiVariable && myOutputParameters.contains(element)) {
            final PsiVariable psiVariable = (PsiVariable)element;
            reference.putUserData(PARAMETER, Pair.create(psiVariable, psiVariable.getType()));
          }
          PsiElement qualifier = reference.getQualifier();
          if (qualifier != null) {
            qualifier.accept(this);
          }
        }
      });
    }
  }

{'fraction_non_alphanumeric': 0.09520105778953099, 'fraction_numerical': 0.0021000233335925953, 'mean_word_length': 3.2212738017071567, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2896076', 'n_tokens_mistral': 3465, 'n_tokens_neox': 3393, 'n_words': 728}