<commit_before>/**
 * Copyright 2011-2015 eBusiness Information, Groupe Excilys (www.ebusinessinformation.fr)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.gatling.recorder.scenario

import java.nio.charset.Charset

import scala.collection.breakOut
import scala.collection.JavaConversions.asScalaBuffer
import scala.concurrent.duration.FiniteDuration
import scala.io.Codec.UTF8

import io.gatling.http.HeaderNames._
import io.gatling.http.HeaderValues._
import io.gatling.http.fetch.{ EmbeddedResource, HtmlParser }
import io.gatling.http.util.HttpHelper.parseFormBody
import io.gatling.recorder.config.RecorderConfiguration

import org.asynchttpclient.util.Base64
import org.asynchttpclient.uri.Uri
import org.jboss.netty.handler.codec.http.{ HttpMessage, HttpRequest, HttpResponse }

private[recorder] case class TimedScenarioElement[+T <: ScenarioElement](sendTime: Long, arrivalTime: Long, element: T)

private[recorder] sealed trait RequestBody
private[recorder] case class RequestBodyParams(params: List[(String, String)]) extends RequestBody
private[recorder] case class RequestBodyBytes(bytes: Array[Byte]) extends RequestBody

private[recorder] sealed trait ResponseBody
private[recorder] case class ResponseBodyBytes(bytes: Array[Byte]) extends ResponseBody

private[recorder] sealed trait ScenarioElement

private[recorder] case class PauseElement(duration: FiniteDuration) extends ScenarioElement
private[recorder] case class TagElement(text: String) extends ScenarioElement

private[recorder] object RequestElement {

  val HtmlContentType = """(?i)text/html\s*(;\s+charset=(.+))?""".r

  private def extractContent(message: HttpMessage) =
    if (message.getContent.readableBytes > 0) {
      val bufferBytes = new Array[Byte](message.getContent.readableBytes)
      message.getContent.getBytes(message.getContent.readerIndex, bufferBytes)
      Some(bufferBytes)
    } else None

  val CacheHeaders = Set(CacheControl, IfMatch, IfModifiedSince, IfNoneMatch, IfRange, IfUnmodifiedSince)

  def apply(request: HttpRequest, response: HttpResponse)(implicit configuration: RecorderConfiguration): RequestElement = {
    val requestHeaders: Map[String, String] = request.headers.entries.map { entry => (entry.getKey, entry.getValue) }(breakOut)
    val requestContentType = requestHeaders.get(ContentType)
    val requestUserAgent = requestHeaders.get(UserAgent)
    val responseContentType = Option(response.headers().get(ContentType))

    val embeddedResources = responseContentType.collect {
      case HtmlContentType(_, headerCharset) =>
        val charsetName = Option(headerCharset).filter(Charset.isSupported).getOrElse(UTF8.name)
        val charset = Charset.forName(charsetName)
        extractContent(response).map(bytes => {
          val htmlBuff = new String(bytes, charset)
          val userAgent = requestHeaders.get(UserAgent).flatMap(io.gatling.http.fetch.UserAgent.parseFromHeader)
          new HtmlParser().getEmbeddedResources(Uri.create(request.getUri), htmlBuff, userAgent)
        })
    }.flatten.getOrElse(Nil)

    val containsFormParams = requestContentType.exists(_.contains(ApplicationFormUrlEncoded))

    val requestBody = extractContent(request).map(content =>
      if (containsFormParams)
        // The payload consists of a Unicode string using only characters in the range U+0000 to U+007F
        // cf: http://www.w3.org/TR/html5/forms.html#application/x-www-form-urlencoded-decoding-algorithm
        RequestBodyParams(parseFormBody(new String(content, UTF8.name)))
      else
        RequestBodyBytes(content))

    val responseBody = extractContent(request) map ResponseBodyBytes

    val filteredRequestHeaders =
      if (configuration.http.removeCacheHeaders)
        requestHeaders.filterKeys(name => !CacheHeaders.contains(name))
      else
        requestHeaders

    RequestElement(new String(request.getUri), request.getMethod.toString, filteredRequestHeaders, requestBody, responseBody, response.getStatus.getCode, embeddedResources)
  }
}

private[recorder] case class RequestElement(uri: String,
                                            method: String,
                                            headers: Map[String, String],
                                            body: Option[RequestBody],
                                            responseBody: Option[ResponseBody],
                                            statusCode: Int,
                                            embeddedResources: List[EmbeddedResource],
                                            nonEmbeddedResources: List[RequestElement] = Nil) extends ScenarioElement {

  val (baseUrl, pathQuery) = {
    val uriComponents = Uri.create(uri)

    val base = new StringBuilder().append(uriComponents.getScheme).append("://").append(uriComponents.getHost)
    val port = uriComponents.getScheme match {
      case "http" if !Set(-1, 80).contains(uriComponents.getPort) => ":" + uriComponents.getPort
      case "https" if !Set(-1, 443).contains(uriComponents.getPort) => ":" + uriComponents.getPort
      case _ => ""
    }
    base.append(port)<commit_msg>Save response body, close #2757<commit_after>/**
 * Copyright 2011-2015 eBusiness Information, Groupe Excilys (www.ebusinessinformation.fr)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.gatling.recorder.scenario

import java.nio.charset.Charset

import scala.collection.breakOut
import scala.collection.JavaConversions.asScalaBuffer
import scala.concurrent.duration.FiniteDuration
import scala.io.Codec.UTF8

import io.gatling.http.HeaderNames._
import io.gatling.http.HeaderValues._
import io.gatling.http.fetch.{ EmbeddedResource, HtmlParser }
import io.gatling.http.util.HttpHelper.parseFormBody
import io.gatling.recorder.config.RecorderConfiguration

import org.asynchttpclient.util.Base64
import org.asynchttpclient.uri.Uri
import org.jboss.netty.handler.codec.http.{ HttpMessage, HttpRequest, HttpResponse }

private[recorder] case class TimedScenarioElement[+T <: ScenarioElement](sendTime: Long, arrivalTime: Long, element: T)

private[recorder] sealed trait RequestBody
private[recorder] case class RequestBodyParams(params: List[(String, String)]) extends RequestBody
private[recorder] case class RequestBodyBytes(bytes: Array[Byte]) extends RequestBody

private[recorder] sealed trait ResponseBody
private[recorder] case class ResponseBodyBytes(bytes: Array[Byte]) extends ResponseBody

private[recorder] sealed trait ScenarioElement

private[recorder] case class PauseElement(duration: FiniteDuration) extends ScenarioElement
private[recorder] case class TagElement(text: String) extends ScenarioElement

private[recorder] object RequestElement {

  val HtmlContentType = """(?i)text/html\s*(;\s+charset=(.+))?""".r

  private def extractContent(message: HttpMessage) =
    if (message.getContent.readableBytes > 0) {
      val bufferBytes = new Array[Byte](message.getContent.readableBytes)
      message.getContent.getBytes(message.getContent.readerIndex, bufferBytes)
      Some(bufferBytes)
    } else None

  val CacheHeaders = Set(CacheControl, IfMatch, IfModifiedSince, IfNoneMatch, IfRange, IfUnmodifiedSince)

  def apply(request: HttpRequest, response: HttpResponse)(implicit configuration: RecorderConfiguration): RequestElement = {
    val requestHeaders: Map[String, String] = request.headers.entries.map { entry => (entry.getKey, entry.getValue) }(breakOut)
    val requestContentType = requestHeaders.get(ContentType)
    val requestUserAgent = requestHeaders.get(UserAgent)
    val responseContentType = Option(response.headers().get(ContentType))

    val embeddedResources = responseContentType.collect {
      case HtmlContentType(_, headerCharset) =>
        val charsetName = Option(headerCharset).filter(Charset.isSupported).getOrElse(UTF8.name)
        val charset = Charset.forName(charsetName)
        extractContent(response).map(bytes => {
          val htmlBuff = new String(bytes, charset)
          val userAgent = requestHeaders.get(UserAgent).flatMap(io.gatling.http.fetch.UserAgent.parseFromHeader)
          new HtmlParser().getEmbeddedResources(Uri.create(request.getUri), htmlBuff, userAgent)
        })
    }.flatten.getOrElse(Nil)

    val containsFormParams = requestContentType.exists(_.contains(ApplicationFormUrlEncoded))

    val requestBody = extractContent(request).map(content =>
      if (containsFormParams)
        // The payload consists of a Unicode string using only characters in the range U+0000 to U+007F
        // cf: http://www.w3.org/TR/html5/forms.html#application/x-www-form-urlencoded-decoding-algorithm
        RequestBodyParams(parseFormBody(new String(content, UTF8.name)))
      else
        RequestBodyBytes(content))

    val responseBody = extractContent(response) map ResponseBodyBytes

    val filteredRequestHeaders =
      if (configuration.http.removeCacheHeaders)
        requestHeaders.filterKeys(name => !CacheHeaders.contains(name))
      else
        requestHeaders

    RequestElement(new String(request.getUri), request.getMethod.toString, filteredRequestHeaders, requestBody, responseBody, response.getStatus.getCode, embeddedResources)
  }
}

private[recorder] case class RequestElement(uri: String,
                                            method: String,
                                            headers: Map[String, String],
                                            body: Option[RequestBody],
                                            responseBody: Option[ResponseBody],
                                            statusCode: Int,
                                            embeddedResources: List[EmbeddedResource],
                                            nonEmbeddedResources: List[RequestElement] = Nil) extends ScenarioElement {

  val (baseUrl, pathQuery) = {
    val uriComponents = Uri.create(uri)

    val base = new StringBuilder().append(uriComponents.getScheme).append("://").append(uriComponents.getHost)
    val port = uriComponents.getScheme match {
      case "http" if !Set(-1, 80).contains(uriComponents.getPort) => ":" + uriComponents.getPort
      case "https" if !Set(-1, 443).contains(uriComponents.getPort) => ":" + uriComponents.getPort
      case _ => ""
    }
    base.append(port)
{'fraction_non_alphanumeric': 0.09367515302049144, 'fraction_numerical': 0.006386942251397144, 'mean_word_length': 3.7073068893528185, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5329037', 'n_tokens_mistral': 3028, 'n_tokens_neox': 2918, 'n_words': 727}