<commit_before>package com.jetbrains.python.validation;

import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiWhiteSpace;
import com.intellij.util.containers.HashSet;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.codeInsight.intentions.*;
import com.jetbrains.python.psi.*;
import org.jetbrains.annotations.NotNull;

import java.util.List;
import java.util.Set;

/**
 * Created by IntelliJ IDEA.
 * User: Alexey.Ivanov
 * Date: 16.02.2010
 * Time: 16:16:45
 */
public class UnsupportedFeatures extends PyAnnotator {
  private static final Set<String> REMOVED_METHODS = new HashSet<String>();

  static {
    REMOVED_METHODS.add("cmp");
    REMOVED_METHODS.add("apply");
    REMOVED_METHODS.add("callable");
    REMOVED_METHODS.add("coerce");
    REMOVED_METHODS.add("execfile");
    REMOVED_METHODS.add("reduce");
    REMOVED_METHODS.add("reload");
  }

  @NotNull
  private static LanguageLevel getLanguageLevel(PyElement node) {
    VirtualFile virtualFile = node.getContainingFile().getVirtualFile();
    if (virtualFile != null) {
      return LanguageLevel.forFile(virtualFile);
    }
    return LanguageLevel.getDefault();
  }

  private static boolean isPy2(PyElement node) {
    return !getLanguageLevel(node).isPy3K();
  }

  private static boolean isPy3K(PyElement node) {
    return getLanguageLevel(node).isPy3K();
  }

  @Override
  public void visitPyDictCompExpression(PyDictCompExpression node) {
    if (isPy2(node)) {
      getHolder().createWarningAnnotation(node, "Dictionary comprehension is not supported in Python 2").registerFix(new ConvertDictCompIntention());
    }
  }

  @Override
  public void visitPySetLiteralExpression(PySetLiteralExpression node) {
    LanguageLevel languageLevel = getLanguageLevel(node);
    if (languageLevel == LanguageLevel.PYTHON24
        || languageLevel == LanguageLevel.PYTHON25
        || languageLevel == LanguageLevel.PYTHON26) {
      getHolder().createWarningAnnotation(node, "This Python version does not support set literal expressions").registerFix(new ConvertSetLiteralIntention());
    }
  }

  @Override
  public void visitPyExceptBlock(PyExceptPart node) {
    PyExpression exceptClass = node.getExceptClass();
    if (exceptClass != null) {
      LanguageLevel languageLevel = getLanguageLevel(node);
      if (languageLevel == LanguageLevel.PYTHON24 || languageLevel == LanguageLevel.PYTHON25) {
        PsiElement element = exceptClass.getNextSibling();
        while (element instanceof PsiWhiteSpace) {
          element = element.getNextSibling();
        }
        if (element != null && "as".equals(element.getText())) {
          getHolder().createWarningAnnotation(node, "This Python version does not support this syntax");
        }
      }
      else if (isPy3K(node)) {
        PsiElement element = exceptClass.getNextSibling();
        while (element instanceof PsiWhiteSpace) {
          element = element.getNextSibling();
        }
        if (element != null && ",".equals(element.getText())) {
          getHolder().createWarningAnnotation(node, "Python 3 does not support this syntax").registerFix(new ReplaceExceptPartIntention());
        }
      }
    }
  }

  @Override
  public void visitPyImportStatement(PyImportStatement node) {
    PyImportElement[] importElements = node.getImportElements();
    for (PyImportElement importElement : importElements) {
      PyReferenceExpression importReference = importElement.getImportReference();
      if (importReference != null) {
        String name = importReference.getName();
        if (isPy2(node)) {
          if ("builtins".equals(name)) {
            getHolder().createWarningAnnotation(node, "There is no module builtins in Python 2")
              .registerFix(new ReplaceBuiltinsIntention());
          }
        }
        else {
          if ("__builtin__".equals(name)) {
            getHolder().createWarningAnnotation(node, "Module __builtin__ renamed to builtins").registerFix(new ReplaceBuiltinsIntention());
          }
        }
      }
    }
  }

  @Override
  public void visitPyCallExpression(PyCallExpression node) {
    if (isPy2(node)) {
      final PsiElement firstChild = node.getFirstChild();
      if (firstChild != null) {
        final String name = firstChild.getText();
        if ("super".equals(name)) {
          final PyArgumentList argumentList = node.getArgumentList();
          if (argumentList != null && argumentList.getArguments().length == 0) {
            getHolder().createWarningAnnotation(node, "super() should have arguments in Python 2");
          }
        }
      }
    } else {
      final String name = node.getCallee().getName();
      if ("raw_input".equals(name)) {
        getHolder().createWarningAnnotation(node.getCallee(), PyBundle.message("ANN.method.$0.removed.use.$1", name, "input")).
                        registerFix(new ReplaceMethodIntention("input"));
      }
      else if (REMOVED_METHODS.contains(name)) {
        getHolder().createWarningAnnotation(node.getCallee(), PyBundle.message("ANN.method.$0.removed", name));
      }<commit_msg>more correct checking for builtins module (PY-1145)<commit_after>package com.jetbrains.python.validation;

import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiWhiteSpace;
import com.intellij.util.containers.HashSet;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.codeInsight.intentions.*;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.impl.PyQualifiedName;
import org.jetbrains.annotations.NotNull;

import java.util.List;
import java.util.Set;

/**
 * Created by IntelliJ IDEA.
 * User: Alexey.Ivanov
 * Date: 16.02.2010
 * Time: 16:16:45
 */
public class UnsupportedFeatures extends PyAnnotator {
  private static final Set<String> REMOVED_METHODS = new HashSet<String>();

  static {
    REMOVED_METHODS.add("cmp");
    REMOVED_METHODS.add("apply");
    REMOVED_METHODS.add("callable");
    REMOVED_METHODS.add("coerce");
    REMOVED_METHODS.add("execfile");
    REMOVED_METHODS.add("reduce");
    REMOVED_METHODS.add("reload");
  }

  @NotNull
  private static LanguageLevel getLanguageLevel(PyElement node) {
    VirtualFile virtualFile = node.getContainingFile().getVirtualFile();
    if (virtualFile != null) {
      return LanguageLevel.forFile(virtualFile);
    }
    return LanguageLevel.getDefault();
  }

  private static boolean isPy2(PyElement node) {
    return !getLanguageLevel(node).isPy3K();
  }

  private static boolean isPy3K(PyElement node) {
    return getLanguageLevel(node).isPy3K();
  }

  @Override
  public void visitPyDictCompExpression(PyDictCompExpression node) {
    if (isPy2(node)) {
      getHolder().createWarningAnnotation(node, "Dictionary comprehension is not supported in Python 2").registerFix(new ConvertDictCompIntention());
    }
  }

  @Override
  public void visitPySetLiteralExpression(PySetLiteralExpression node) {
    LanguageLevel languageLevel = getLanguageLevel(node);
    if (languageLevel == LanguageLevel.PYTHON24
        || languageLevel == LanguageLevel.PYTHON25
        || languageLevel == LanguageLevel.PYTHON26) {
      getHolder().createWarningAnnotation(node, "This Python version does not support set literal expressions").registerFix(new ConvertSetLiteralIntention());
    }
  }

  @Override
  public void visitPyExceptBlock(PyExceptPart node) {
    PyExpression exceptClass = node.getExceptClass();
    if (exceptClass != null) {
      LanguageLevel languageLevel = getLanguageLevel(node);
      if (languageLevel == LanguageLevel.PYTHON24 || languageLevel == LanguageLevel.PYTHON25) {
        PsiElement element = exceptClass.getNextSibling();
        while (element instanceof PsiWhiteSpace) {
          element = element.getNextSibling();
        }
        if (element != null && "as".equals(element.getText())) {
          getHolder().createWarningAnnotation(node, "This Python version does not support this syntax");
        }
      }
      else if (isPy3K(node)) {
        PsiElement element = exceptClass.getNextSibling();
        while (element instanceof PsiWhiteSpace) {
          element = element.getNextSibling();
        }
        if (element != null && ",".equals(element.getText())) {
          getHolder().createWarningAnnotation(node, "Python 3 does not support this syntax").registerFix(new ReplaceExceptPartIntention());
        }
      }
    }
  }

  @Override
  public void visitPyImportStatement(PyImportStatement node) {
    PyImportElement[] importElements = node.getImportElements();
    for (PyImportElement importElement : importElements) {
      final PyQualifiedName qName = importElement.getImportedQName();
      if (qName != null) {
        if (isPy2(node)) {
          if (qName.matches("builtins")) {
            getHolder().createWarningAnnotation(node, "There is no module builtins in Python 2")
              .registerFix(new ReplaceBuiltinsIntention());
          }
        }
        else {
          if (qName.matches("__builtin__")) {
            getHolder().createWarningAnnotation(node, "Module __builtin__ renamed to builtins").registerFix(new ReplaceBuiltinsIntention());
          }
        }
      }
    }
  }

  @Override
  public void visitPyCallExpression(PyCallExpression node) {
    if (isPy2(node)) {
      final PsiElement firstChild = node.getFirstChild();
      if (firstChild != null) {
        final String name = firstChild.getText();
        if ("super".equals(name)) {
          final PyArgumentList argumentList = node.getArgumentList();
          if (argumentList != null && argumentList.getArguments().length == 0) {
            getHolder().createWarningAnnotation(node, "super() should have arguments in Python 2");
          }
        }
      }
    } else {
      final String name = node.getCallee().getName();
      if ("raw_input".equals(name)) {
        getHolder().createWarningAnnotation(node.getCallee(), PyBundle.message("ANN.method.$0.removed.use.$1", name, "input")).
                        registerFix(new ReplaceMethodIntention("input"));
      }
      else if (REMOVED_METHODS.contains(name)) {
        getHolder().createWarningAnnotation(node.getCallee(), PyBundle.message("ANN.method.$0.removed", name));
      }
{'fraction_non_alphanumeric': 0.11468558654874138, 'fraction_numerical': 0.008164058703469725, 'mean_word_length': 3.4642082429501087, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6647896', 'n_tokens_mistral': 2954, 'n_tokens_neox': 2864, 'n_words': 592}