<commit_before>/*
 * Copyright 2000-2013 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.ide.plugins;

import com.google.common.hash.Hashing;
import com.google.common.io.Files;
import com.intellij.ide.IdeBundle;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.application.ex.ApplicationInfoEx;
import com.intellij.openapi.application.impl.ApplicationInfoImpl;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.util.BuildNumber;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.util.PathUtil;
import com.intellij.util.net.HttpConfigurable;
import com.intellij.util.net.NetUtils;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.zip.GZIPInputStream;

/**
 * @author stathik
 * @since Mar 28, 2003
 */
public class RepositoryHelper {
  @NonNls public static final String PLUGIN_LIST_FILE = "availables.xml";

  public static List<IdeaPluginDescriptor> loadPluginsFromRepository(@Nullable ProgressIndicator indicator) throws Exception {
    return loadPluginsFromRepository(indicator, null);
  }

  @NotNull
  public static Pair<URLConnection, String> openConnection(@NotNull String initialUrl, boolean supportGzip) throws IOException {
    int i = 0;
    String url = initialUrl;
    while (i++ < 99) {
      URLConnection connection;
      if (ApplicationManager.getApplication() != null) {
        connection = HttpConfigurable.getInstance().openConnection(url);
      }
      else {
        connection = new URL(url).openConnection();
        connection.setConnectTimeout(HttpConfigurable.CONNECTION_TIMEOUT);
        connection.setReadTimeout(HttpConfigurable.CONNECTION_TIMEOUT);
      }

      if (supportGzip) {
        connection.setRequestProperty("Accept-Encoding", "gzip");
      }
      connection.setUseCaches(false);

      if (connection instanceof HttpURLConnection) {
        int responseCode = ((HttpURLConnection)connection).getResponseCode();
        if (responseCode != HttpURLConnection.HTTP_OK && responseCode != HttpURLConnection.HTTP_NOT_MODIFIED) {
          if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) {
            url = connection.getHeaderField("Location");
          }
          else {
            url = null;
          }

          if (url == null) {
            throw new IOException(IdeBundle.message("error.connection.failed.with.http.code.N", responseCode));
          }
          else {
            ((HttpURLConnection)connection).disconnect();
            continue;
          }
        }
      }
      return Pair.create(connection, url == initialUrl ? null : url);
    }
    throw new IOException("Infinite redirection");
  }

  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
  @NotNull
  public static InputStream getConnectionInputStream(@NotNull URLConnection connection) throws IOException {
    InputStream inputStream = connection.getInputStream();
    if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
      try {
        return new GZIPInputStream(inputStream);
      }
      catch (IOException e) {
        inputStream.close();
        throw e;
      }
    }
    else {
      return inputStream;
    }
  }

  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
  public static List<IdeaPluginDescriptor> loadPluginsFromRepository(@Nullable ProgressIndicator indicator, @Nullable BuildNumber buildnumber) throws Exception {
    ApplicationInfoEx appInfo = ApplicationInfoImpl.getShadowInstance();

    String url = appInfo.getPluginsListUrl() + "?build=" + (buildnumber != null ? buildnumber.asString() : appInfo.getApiVersion());

    if (indicator != null) {
      indicator.setText2(IdeBundle.message("progress.connecting.to.plugin.manager", appInfo.getPluginManagerUrl()));
    }

    File pluginListFile = new File(PathManager.getPluginsPath(), PLUGIN_LIST_FILE);
    if (pluginListFile.length() > 0) {
      try {
        url = url + "&crc32=" + Files.hash(pluginListFile, Hashing.crc32()).toString();
      }
      catch (NoSuchMethodError e) {
        String guavaPath = PathUtil.getJarPathForClass(Hashing.class);
        throw new RuntimeException(guavaPath, e);
      }
    }

    HttpURLConnection connection = (HttpURLConnection)openConnection(url, true).first;
    if (indicator != null) {
      indicator.setText2(IdeBundle.message("progress.waiting.for.reply.from.plugin.manager", appInfo.getPluginManagerUrl()));
    }
    connection.connect();
    try {
      if (indicator != null) {
        indicator.checkCanceled();
      }

      if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
        return loadPluginList(pluginListFile);
      }

      if (indicator != null) {
        indicator.setText2(IdeBundle.message("progress.downloading.list.of.plugins"));
      }
      return readPluginsStream(connection, indicator, PLUGIN_LIST_FILE);
    }
    finally {
      connection.disconnect();
    }
  }

  private synchronized static List<IdeaPluginDescriptor> readPluginsStream(@NotNull URLConnection connection,
                                                                           ProgressIndicator indicator,
                                                                           @NotNull String file) throws Exception {
    File localFile;
    InputStream input = getConnectionInputStream(connection);
    try {
      localFile = createLocalPluginsDescriptions(file);<commit_msg>CR-IC-6500#CFR-95450<commit_after>/*
 * Copyright 2000-2013 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.ide.plugins;

import com.google.common.hash.Hashing;
import com.google.common.io.Files;
import com.intellij.ide.IdeBundle;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.application.PathManager;
import com.intellij.openapi.application.ex.ApplicationInfoEx;
import com.intellij.openapi.application.impl.ApplicationInfoImpl;
import com.intellij.openapi.progress.ProgressIndicator;
import com.intellij.openapi.util.BuildNumber;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.openapi.util.io.FileUtilRt;
import com.intellij.util.PathUtil;
import com.intellij.util.net.HttpConfigurable;
import com.intellij.util.net.NetUtils;
import org.jetbrains.annotations.NonNls;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.zip.GZIPInputStream;

/**
 * @author stathik
 * @since Mar 28, 2003
 */
public class RepositoryHelper {
  @NonNls public static final String PLUGIN_LIST_FILE = "availables.xml";

  public static List<IdeaPluginDescriptor> loadPluginsFromRepository(@Nullable ProgressIndicator indicator) throws Exception {
    return loadPluginsFromRepository(indicator, null);
  }

  @NotNull
  public static Pair<URLConnection, String> openConnection(@NotNull String initialUrl, boolean supportGzip) throws IOException {
    int i = 0;
    String url = initialUrl;
    while (i++ < 99) {
      URLConnection connection;
      if (ApplicationManager.getApplication() != null) {
        connection = HttpConfigurable.getInstance().openConnection(url);
      }
      else {
        connection = new URL(url).openConnection();
        connection.setConnectTimeout(HttpConfigurable.CONNECTION_TIMEOUT);
        connection.setReadTimeout(HttpConfigurable.CONNECTION_TIMEOUT);
      }

      if (supportGzip) {
        connection.setRequestProperty("Accept-Encoding", "gzip");
      }
      connection.setUseCaches(false);

      if (connection instanceof HttpURLConnection) {
        int responseCode = ((HttpURLConnection)connection).getResponseCode();
        if (responseCode != HttpURLConnection.HTTP_OK && responseCode != HttpURLConnection.HTTP_NOT_MODIFIED) {
          if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) {
            url = connection.getHeaderField("Location");
          }
          else {
            url = null;
          }

          if (url == null) {
            throw new IOException(IdeBundle.message("error.connection.failed.with.http.code.N", responseCode));
          }
          else {
            ((HttpURLConnection)connection).disconnect();
            continue;
          }
        }
      }
      return Pair.create(connection, url == initialUrl ? null : url);
    }
    throw new IOException("Infinite redirection");
  }

  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
  @NotNull
  public static InputStream getConnectionInputStream(@NotNull URLConnection connection) throws IOException {
    InputStream inputStream = connection.getInputStream();
    if ("gzip".equalsIgnoreCase(connection.getContentEncoding())) {
      try {
        return new GZIPInputStream(inputStream);
      }
      catch (IOException e) {
        inputStream.close();
        throw e;
      }
    }
    else {
      return inputStream;
    }
  }

  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
  public static List<IdeaPluginDescriptor> loadPluginsFromRepository(@Nullable ProgressIndicator indicator, @Nullable BuildNumber buildnumber) throws Exception {
    ApplicationInfoEx appInfo = ApplicationInfoImpl.getShadowInstance();

    String url = appInfo.getPluginsListUrl() + "?build=" + (buildnumber != null ? buildnumber.asString() : appInfo.getApiVersion());

    if (indicator != null) {
      indicator.setText2(IdeBundle.message("progress.connecting.to.plugin.manager", appInfo.getPluginManagerUrl()));
    }

    File pluginListFile = new File(PathManager.getPluginsPath(), PLUGIN_LIST_FILE);
    if (pluginListFile.length() > 0) {
      try {
        url = url + "&crc32=" + Files.hash(pluginListFile, Hashing.crc32()).toString();
      }
      catch (NoSuchMethodError e) {
        String guavaPath = PathUtil.getJarPathForClass(Hashing.class);
        throw new RuntimeException(guavaPath, e);
      }
    }

    URLConnection connection = openConnection(url, true).first;
    if (indicator != null) {
      indicator.setText2(IdeBundle.message("progress.waiting.for.reply.from.plugin.manager", appInfo.getPluginManagerUrl()));
    }
    connection.connect();
    try {
      if (indicator != null) {
        indicator.checkCanceled();
      }

      if (connection instanceof HttpURLConnection && ((HttpURLConnection)connection).getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
        return loadPluginList(pluginListFile);
      }

      if (indicator != null) {
        indicator.setText2(IdeBundle.message("progress.downloading.list.of.plugins"));
      }
      return readPluginsStream(connection, indicator, PLUGIN_LIST_FILE);
    }
    finally {
      if (connection instanceof HttpURLConnection) {
        ((HttpURLConnection)connection).disconnect();
      }
    }
  }

  private synchronized static List<IdeaPluginDescriptor> readPluginsStream(@NotNull URLConnection connection,
                                                                           ProgressIndicator indicator,
                                                                           @NotNull String file) throws Exception {
    File localFile;
    InputStream input = getConnectionInputStream(connection);
    try {
      localFile = createLocalPluginsDescriptions(file);
{'fraction_non_alphanumeric': 0.09914057704112952, 'fraction_numerical': 0.0051411909146715775, 'mean_word_length': 3.397098515519568, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4557362', 'n_tokens_mistral': 3591, 'n_tokens_neox': 3513, 'n_words': 760}