<commit_before>package com.jetbrains.python.sdk;

import com.intellij.execution.process.ProcessOutput;
import com.intellij.notification.Notification;
import com.intellij.notification.NotificationType;
import com.intellij.notification.Notifications;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.application.ex.ApplicationManagerEx;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.JarFileSystem;
import com.intellij.openapi.vfs.LocalFileSystem;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ArrayUtil;
import com.intellij.util.SmartList;
import com.intellij.util.io.ZipUtil;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PythonHelpersLocator;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.jetbrains.python.sdk.SkeletonVersionChecker.fromVersionString;

/**
 * Handles a refresh of SDK's skeletons.
 * Does all the heavy lifting calling skeleton generator, managing blacklists, etc.
 * One-time, non-reusable instances.
 * <br/>
 * User: dcheryasov
 * Date: 4/15/11 5:38 PM
 */
public class PySkeletonRefresher {
  @Nullable final ProgressIndicator myIndicator;
  final Sdk mySdk;
  String mySkeletonsPath;
  private static final int MINUTE = 60 * 1000;
  final static String GENERATOR3 = "generator3.py";

  @NonNls public static final String BLACKLIST_FILE_NAME = ".blacklist";
  final static Pattern BLACKLIST_LINE = Pattern.compile("^([^=]+) = (\\d+\\.\\d+) (\\d+)\\s*$");
  // we use the equals sign after filename so that we can freely include space in the filename

  private static final Logger LOG = Logger.getInstance("#" + PySkeletonRefresher.class.getName());
  private String myExtraSyspath;
  private VirtualFile myPregeneratedSkeletons;
  private int myGeneratorVersion;
  private Map<String,Pair<Integer,Long>> myBlacklist;
  private SkeletonVersionChecker myVersionChecker;

  private static class ListBinariesResult {
    public final int generatorVersion;
    public final Map<String, File> modules;

    ListBinariesResult(int generatorVersion, Map<String, File> modules) {
      this.generatorVersion = generatorVersion;
      this.modules = modules;
    }
  }

  /**
   * Creates a new object that refreshes skeletons of given SDK.
   * @param sdk a Python SDK
   * @param skeletonsPath if known; null means 'determine and create as needed'.
   * @param indicator to report progress of long operations
   */
  public PySkeletonRefresher(@NotNull Sdk sdk, @Nullable String skeletonsPath, @Nullable ProgressIndicator indicator) {
    myIndicator = indicator;
    mySdk = sdk;
    mySkeletonsPath = skeletonsPath;
  }

  private void indicate(String msg) {
    if (myIndicator != null) {
      myIndicator.checkCanceled();
      myIndicator.setText(msg);
      myIndicator.setText2("");
    }
  }

  private void indicateMinor(String msg) {
    if (myIndicator != null) {
      myIndicator.setText2(msg);
    }
  }

  private void checkCanceled() {
    if (myIndicator != null) {
      myIndicator.checkCanceled();
    }
  }

  private static String getExtraSysPath(Sdk sdk, String skeletonsPath) {
    final VirtualFile[] classDirs = sdk.getRootProvider().getFiles(OrderRootType.CLASSES);
    final StringBuilder builder = new StringBuilder("\"");
    int i = 0;
    while (i < classDirs.length) {
      if (i > 0) {
        builder.append(File.pathSeparator);
      }
      if (classDirs[i].isInLocalFileSystem()) {
        final String pathname = classDirs[i].getPath();
        if (!skeletonsPath.equals(pathname)) {
          builder.append(pathname);
        }
      }
      i += 1;
    }
    builder.append("\"");
    return builder.toString();
  }

  /**
   * Creates if needed all path(s) used to store skeletons of its SDK.
   * @return path name of skeleton dir for the SDK, guaranteed to be already created.
   */
  @NotNull
  public String getSkeletonsPath() {
    if (mySkeletonsPath == null) {
      mySkeletonsPath = PythonSdkType.getSkeletonsPath(mySdk.getHomePath());
      final File skeletonsDir = new File(mySkeletonsPath);
      if (!skeletonsDir.exists() && !skeletonsDir.mkdirs()) {<commit_msg>Fixed NPE in PySkeletonRefresher (EA-32558)<commit_after>package com.jetbrains.python.sdk;

import com.intellij.execution.process.ProcessOutput;
import com.intellij.notification.Notification;
import com.intellij.notification.NotificationType;
import com.intellij.notification.Notifications;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.application.ex.ApplicationManagerEx;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.projectRoots.Sdk;
import com.intellij.openapi.roots.OrderRootType;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.Ref;
import com.intellij.openapi.util.SystemInfo;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.vfs.JarFileSystem;
import com.intellij.openapi.vfs.LocalFileSystem;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.util.ArrayUtil;
import com.intellij.util.SmartList;
import com.intellij.util.io.ZipUtil;
import com.jetbrains.python.PyBundle;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.PythonHelpersLocator;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.jetbrains.python.sdk.SkeletonVersionChecker.fromVersionString;

/**
 * Handles a refresh of SDK's skeletons.
 * Does all the heavy lifting calling skeleton generator, managing blacklists, etc.
 * One-time, non-reusable instances.
 * <br/>
 * User: dcheryasov
 * Date: 4/15/11 5:38 PM
 */
public class PySkeletonRefresher {
  @Nullable final ProgressIndicator myIndicator;
  final Sdk mySdk;
  String mySkeletonsPath;
  private static final int MINUTE = 60 * 1000;
  final static String GENERATOR3 = "generator3.py";

  @NonNls public static final String BLACKLIST_FILE_NAME = ".blacklist";
  final static Pattern BLACKLIST_LINE = Pattern.compile("^([^=]+) = (\\d+\\.\\d+) (\\d+)\\s*$");
  // we use the equals sign after filename so that we can freely include space in the filename

  private static final Logger LOG = Logger.getInstance("#" + PySkeletonRefresher.class.getName());
  private String myExtraSyspath;
  private VirtualFile myPregeneratedSkeletons;
  private int myGeneratorVersion;
  private Map<String,Pair<Integer,Long>> myBlacklist;
  private SkeletonVersionChecker myVersionChecker;

  private static class ListBinariesResult {
    public final int generatorVersion;
    public final Map<String, File> modules;

    ListBinariesResult(int generatorVersion, Map<String, File> modules) {
      this.generatorVersion = generatorVersion;
      this.modules = modules;
    }
  }

  /**
   * Creates a new object that refreshes skeletons of given SDK.
   * @param sdk a Python SDK
   * @param skeletonsPath if known; null means 'determine and create as needed'.
   * @param indicator to report progress of long operations
   */
  public PySkeletonRefresher(@NotNull Sdk sdk, @Nullable String skeletonsPath, @Nullable ProgressIndicator indicator) {
    myIndicator = indicator;
    mySdk = sdk;
    mySkeletonsPath = skeletonsPath;
  }

  private void indicate(String msg) {
    if (myIndicator != null) {
      myIndicator.checkCanceled();
      myIndicator.setText(msg);
      myIndicator.setText2("");
    }
  }

  private void indicateMinor(String msg) {
    if (myIndicator != null) {
      myIndicator.setText2(msg);
    }
  }

  private void checkCanceled() {
    if (myIndicator != null) {
      myIndicator.checkCanceled();
    }
  }

  private static String getExtraSysPath(@NotNull Sdk sdk, @Nullable String skeletonsPath) {
    final VirtualFile[] classDirs = sdk.getRootProvider().getFiles(OrderRootType.CLASSES);
    final StringBuilder builder = new StringBuilder("\"");
    int i = 0;
    while (i < classDirs.length) {
      if (i > 0) {
        builder.append(File.pathSeparator);
      }
      if (classDirs[i].isInLocalFileSystem()) {
        final String pathname = classDirs[i].getPath();
        if (pathname != null && !pathname.equals(skeletonsPath)) {
          builder.append(pathname);
        }
      }
      i += 1;
    }
    builder.append("\"");
    return builder.toString();
  }

  /**
   * Creates if needed all path(s) used to store skeletons of its SDK.
   * @return path name of skeleton dir for the SDK, guaranteed to be already created.
   */
  @NotNull
  public String getSkeletonsPath() {
    if (mySkeletonsPath == null) {
      mySkeletonsPath = PythonSdkType.getSkeletonsPath(mySdk.getHomePath());
      final File skeletonsDir = new File(mySkeletonsPath);
      if (!skeletonsDir.exists() && !skeletonsDir.mkdirs()) {
{'fraction_non_alphanumeric': 0.10579634464751958, 'fraction_numerical': 0.004908616187989556, 'mean_word_length': 4.120855614973262, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2890345', 'n_tokens_mistral': 3053, 'n_tokens_neox': 2933, 'n_words': 710}