<commit_before>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.intellij.build.impl.compilation

import com.google.gson.Gson
import com.intellij.openapi.util.io.StreamUtil
import com.intellij.openapi.util.text.StringUtil
import groovy.transform.CompileStatic
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.client.methods.HttpPut
import org.apache.http.entity.ContentType
import org.apache.http.entity.FileEntity
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.impl.client.LaxRedirectStrategy
import org.apache.http.util.EntityUtils
import org.jetbrains.annotations.NotNull
import org.jetbrains.intellij.build.BuildMessages

@CompileStatic
class CompilationPartsUploader implements Closeable {
  private final BuildMessages myMessages
  protected final String myServerUrl
  protected final CloseableHttpClient myHttpClient

  CompilationPartsUploader(@NotNull String serverUrl, @NotNull BuildMessages messages) {
    myServerUrl = fixServerUrl(serverUrl)
    myMessages = messages
    CompilationPartsUtil.initLog4J(messages)
    myHttpClient = HttpClientBuilder.create()
      .setUserAgent('Parts Uploader')
      .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
      .setMaxConnTotal(20)
      .setMaxConnPerRoute(10)
      .build()
  }

  @Override
  void close() throws IOException {
    StreamUtil.closeStream(myHttpClient)
    CompilationPartsUtil.deinitLog4J()
  }

  @SuppressWarnings("unused")
  protected void debug(String message) {
    myMessages.debug(message)
  }

  @SuppressWarnings("unused")
  protected void log(String message) {
    myMessages.info(message)
  }

  @SuppressWarnings("unused")
  protected void error(String message) {
    myMessages.error(message)
  }

  boolean upload(@NotNull final String path, @NotNull final File file, boolean sendHead) throws UploadException {
    debug("Preparing to upload " + file + " to " + myServerUrl)

    if (!file.exists()) {
      throw new UploadException("The file " + file.getPath() + " does not exist")
    }
    if (sendHead) {
      int code = doHead(path)
      if (code == 200) {
        log("File '$path' already exist on server, nothing to upload")
        return false
      }
      if (code != 404) {
        error("HEAD $path responded with unexpected $code")
      }
    }
    final String response = doPut(path, file)
    if (StringUtil.isEmptyOrSpaces(response)) {
      log("Performed '$path' upload.")
    }
    else {
      debug("Performed '$path' upload. Server answered: " + response)
    }
    return true
  }

  @CompileStatic
  static class CheckFilesResponse {
    List<String> found
    List<String> missing

    CheckFilesResponse() {
    }
  }

  CheckFilesResponse getFoundAndMissingFiles(String metadataJson) {
    String path = '/check-files'

    CloseableHttpResponse response = null
    String responseString = null
    try {
      String url = myServerUrl + StringUtil.trimStart(path, '/')
      debug("POST " + url)

      def request = new HttpPost(url)
      request.setEntity(new StringEntity(metadataJson, ContentType.APPLICATION_JSON))

      response = myHttpClient.execute(request)

      debug("POST code: ${response.getStatusLine().getStatusCode()}")

      responseString = EntityUtils.toString(response.getEntity(), ContentType.APPLICATION_JSON.charset)
      def parsedResponse = new Gson().fromJson(responseString, CheckFilesResponse.class)
      return parsedResponse
    }
    catch (Exception e) {
      def additionalMessage = responseString == null ? "" : "\nResponse: $responseString"
      // FIXME temporary fail here
      myMessages.error("Failed to check for found and mising files ('$path'): ${e.message}" + additionalMessage)
      return null
    }
    finally {
      StreamUtil.closeStream(response)
    }
  }

  @NotNull
  protected int doHead(String path) throws UploadException {
    CloseableHttpResponse response = null
    try {
      String url = myServerUrl + StringUtil.trimStart(path, '/')
      debug("HEAD " + url)<commit_msg>Compilation parts: Revert error to warning<commit_after>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package org.jetbrains.intellij.build.impl.compilation

import com.google.gson.Gson
import com.intellij.openapi.util.io.StreamUtil
import com.intellij.openapi.util.text.StringUtil
import groovy.transform.CompileStatic
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.client.methods.HttpPut
import org.apache.http.entity.ContentType
import org.apache.http.entity.FileEntity
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.impl.client.LaxRedirectStrategy
import org.apache.http.util.EntityUtils
import org.jetbrains.annotations.NotNull
import org.jetbrains.intellij.build.BuildMessages

@CompileStatic
class CompilationPartsUploader implements Closeable {
  private final BuildMessages myMessages
  protected final String myServerUrl
  protected final CloseableHttpClient myHttpClient

  CompilationPartsUploader(@NotNull String serverUrl, @NotNull BuildMessages messages) {
    myServerUrl = fixServerUrl(serverUrl)
    myMessages = messages
    CompilationPartsUtil.initLog4J(messages)
    myHttpClient = HttpClientBuilder.create()
      .setUserAgent('Parts Uploader')
      .setRedirectStrategy(LaxRedirectStrategy.INSTANCE)
      .setMaxConnTotal(20)
      .setMaxConnPerRoute(10)
      .build()
  }

  @Override
  void close() throws IOException {
    StreamUtil.closeStream(myHttpClient)
    CompilationPartsUtil.deinitLog4J()
  }

  @SuppressWarnings("unused")
  protected void debug(String message) {
    myMessages.debug(message)
  }

  @SuppressWarnings("unused")
  protected void log(String message) {
    myMessages.info(message)
  }

  @SuppressWarnings("unused")
  protected void error(String message) {
    myMessages.error(message)
  }

  boolean upload(@NotNull final String path, @NotNull final File file, boolean sendHead) throws UploadException {
    debug("Preparing to upload " + file + " to " + myServerUrl)

    if (!file.exists()) {
      throw new UploadException("The file " + file.getPath() + " does not exist")
    }
    if (sendHead) {
      int code = doHead(path)
      if (code == 200) {
        log("File '$path' already exist on server, nothing to upload")
        return false
      }
      if (code != 404) {
        error("HEAD $path responded with unexpected $code")
      }
    }
    final String response = doPut(path, file)
    if (StringUtil.isEmptyOrSpaces(response)) {
      log("Performed '$path' upload.")
    }
    else {
      debug("Performed '$path' upload. Server answered: " + response)
    }
    return true
  }

  @CompileStatic
  static class CheckFilesResponse {
    List<String> found
    List<String> missing

    CheckFilesResponse() {
    }
  }

  CheckFilesResponse getFoundAndMissingFiles(String metadataJson) {
    String path = '/check-files'

    CloseableHttpResponse response = null
    String responseString = null
    try {
      String url = myServerUrl + StringUtil.trimStart(path, '/')
      debug("POST " + url)

      def request = new HttpPost(url)
      request.setEntity(new StringEntity(metadataJson, ContentType.APPLICATION_JSON))

      response = myHttpClient.execute(request)

      debug("POST code: ${response.getStatusLine().getStatusCode()}")

      responseString = EntityUtils.toString(response.getEntity(), ContentType.APPLICATION_JSON.charset)
      def parsedResponse = new Gson().fromJson(responseString, CheckFilesResponse.class)
      return parsedResponse
    }
    catch (Exception e) {
      def additionalMessage = responseString == null ? "" : "\nResponse: $responseString"
      myMessages.warning("Failed to check for found and mising files ('$path'): ${e.message}" + additionalMessage)
      return null
    }
    finally {
      StreamUtil.closeStream(response)
    }
  }

  @NotNull
  protected int doHead(String path) throws UploadException {
    CloseableHttpResponse response = null
    try {
      String url = myServerUrl + StringUtil.trimStart(path, '/')
      debug("HEAD " + url)
{'fraction_non_alphanumeric': 0.09592353754030401, 'fraction_numerical': 0.005066789497927222, 'mean_word_length': 3.7904026475455046, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4447329', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2438, 'n_words': 586}