<commit_before>// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util

import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.util.SystemInfoRt
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.openapi.util.text.StringUtil
import com.intellij.openapi.vfs.StandardFileSystems
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.util.io.URLUtil
import gnu.trove.TObjectHashingStrategy
import java.net.URI
import java.net.URISyntaxException
import java.util.regex.Pattern

private val LOG = Logger.getInstance(Urls::class.java)

// about ";" see WEB-100359
private val URI_PATTERN = Pattern.compile("^([^:/?#]+):(//)?([^/?#]*)([^?#;]*)(.*)")

object Urls {
  val caseInsensitiveUrlHashingStrategy: TObjectHashingStrategy<Url> by lazy { CaseInsensitiveUrlHashingStrategy() }

  @JvmStatic
  fun newUri(scheme: String?, path: String): UrlImpl = UrlImpl(scheme, null, path)

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String, rawParameters: String?): UrlImpl {
    return UrlImpl(scheme, authority, path, rawParameters)
  }

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String, parameters: Map<String, String?>): UrlImpl {
    var parametersString: String? = null
    if (parameters.isNotEmpty()) {
      val result = StringBuilder()
      result.append("?")
      encodeParameters(parameters, result)
      parametersString = result.toString()
    }
    return UrlImpl(scheme, authority, path, parametersString)
  }

  @JvmStatic
  fun encodeParameters(parameters: Map<String, String?>, result: StringBuilder) {
    val initialSize = result.length
    for ((name, value) in parameters) {
      if (result.length != initialSize) {
        result.append('&')
      }
      // https://stackoverflow.com/questions/5330104/encoding-url-query-parameters-in-java
      result.append(URLUtil.encodeURIComponent(name))
      if (value != null && value.isNotEmpty()) {
        result.append('=')
        result.append(URLUtil.encodeURIComponent(value))
      }
    }
  }

  @JvmStatic
  fun newLocalFileUrl(path: String): Url = LocalFileUrl(FileUtilRt.toSystemIndependentName(path))

  @JvmStatic
  fun newLocalFileUrl(file: VirtualFile): Url = LocalFileUrl(file.path)

  @JvmStatic
  fun newFromEncoded(url: String): Url {
    val result = parseEncoded(url)
    LOG.assertTrue(result != null, url)
    return result!!
  }

  @JvmStatic
  fun parseEncoded(url: String): Url? = parse(url, false)

  @JvmStatic
  fun newHttpUrl(authority: String, path: String?): Url {
    return newUrl("http", authority, path)
  }

  @JvmStatic
  fun newHttpUrl(authority: String, path: String?, parameters: String?): Url {
    return UrlImpl("http", authority, path, parameters)
  }

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String?): Url {
    return UrlImpl(scheme, authority, path)
  }

  /**
   * Url will not be normalized (see [VfsUtilCore.toIdeaUrl]), parsed as is
   */
  @JvmStatic
  fun newFromIdea(url: CharSequence): Url {<commit_msg>Cleanup (replaces double assertion with an exception)<commit_after>// Copyright 2000-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.intellij.util

import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.util.SystemInfoRt
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.io.FileUtilRt
import com.intellij.openapi.util.text.StringUtil
import com.intellij.openapi.vfs.StandardFileSystems
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.util.io.URLUtil
import gnu.trove.TObjectHashingStrategy
import java.net.MalformedURLException
import java.net.URI
import java.net.URISyntaxException
import java.util.regex.Pattern

private val LOG = Logger.getInstance(Urls::class.java)

// about ";" see WEB-100359
private val URI_PATTERN = Pattern.compile("^([^:/?#]+):(//)?([^/?#]*)([^?#;]*)(.*)")

object Urls {
  val caseInsensitiveUrlHashingStrategy: TObjectHashingStrategy<Url> by lazy { CaseInsensitiveUrlHashingStrategy() }

  @JvmStatic
  fun newUri(scheme: String?, path: String): UrlImpl = UrlImpl(scheme, null, path)

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String, rawParameters: String?): UrlImpl {
    return UrlImpl(scheme, authority, path, rawParameters)
  }

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String, parameters: Map<String, String?>): UrlImpl {
    var parametersString: String? = null
    if (parameters.isNotEmpty()) {
      val result = StringBuilder()
      result.append("?")
      encodeParameters(parameters, result)
      parametersString = result.toString()
    }
    return UrlImpl(scheme, authority, path, parametersString)
  }

  @JvmStatic
  fun encodeParameters(parameters: Map<String, String?>, result: StringBuilder) {
    val initialSize = result.length
    for ((name, value) in parameters) {
      if (result.length != initialSize) {
        result.append('&')
      }
      // https://stackoverflow.com/questions/5330104/encoding-url-query-parameters-in-java
      result.append(URLUtil.encodeURIComponent(name))
      if (value != null && value.isNotEmpty()) {
        result.append('=')
        result.append(URLUtil.encodeURIComponent(value))
      }
    }
  }

  @JvmStatic
  fun newLocalFileUrl(path: String): Url = LocalFileUrl(FileUtilRt.toSystemIndependentName(path))

  @JvmStatic
  fun newLocalFileUrl(file: VirtualFile): Url = LocalFileUrl(file.path)

  @JvmStatic
  fun newFromEncoded(url: String): Url = parseEncoded(url) ?: throw MalformedURLException("Malformed URL: ${url}")

  @JvmStatic
  fun parseEncoded(url: String): Url? = parse(url, false)

  @JvmStatic
  fun newHttpUrl(authority: String, path: String?): Url {
    return newUrl("http", authority, path)
  }

  @JvmStatic
  fun newHttpUrl(authority: String, path: String?, parameters: String?): Url {
    return UrlImpl("http", authority, path, parameters)
  }

  @JvmStatic
  fun newUrl(scheme: String, authority: String, path: String?): Url {
    return UrlImpl(scheme, authority, path)
  }

  /**
   * Url will not be normalized (see [VfsUtilCore.toIdeaUrl]), parsed as is
   */
  @JvmStatic
  fun newFromIdea(url: CharSequence): Url {
{'fraction_non_alphanumeric': 0.12383141762452107, 'fraction_numerical': 0.007049808429118774, 'mean_word_length': 4.344799344799345, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4664758', 'n_tokens_mistral': 2175, 'n_tokens_neox': 2101, 'n_words': 487}