<commit_before>package org.labdb.labdb

import java.time.{ZoneId, ZonedDateTime}
import javax.servlet.http.{HttpServletRequest, HttpServletResponse}

import org.scalatra._
import org.slf4j.{Logger, LoggerFactory}

import scalaj.http.Http

trait LabdbStack extends ScalatraServlet {

  val logger: Logger = LoggerFactory.getLogger(getClass)
  val PROXY_SUFFIX = "-backend.labdb.io"

  def nowUTC: ZonedDateTime = ZonedDateTime.now(ZoneId.of("Z"))

  def proxyHost(currHost: String): String =
    if (Env.isDev) "localhost" else currHost.replaceAllLiterally(".labdb.io", "") + PROXY_SUFFIX
  def proxyPort: String = if (Env.isDev) ":3001" else ""
  def proxyProtocol: String = if (Env.isDev) "http://" else "https://"

  def proxyRequest(implicit request: HttpServletRequest, response: HttpServletResponse): ActionResult = {
    var proxiedUrl = proxyProtocol + proxyHost(request.getServerName) + proxyPort + request.getRequestURI
    val queryString = request.getQueryString
    if (queryString != null) {
      proxiedUrl += "?" + queryString
    }
    var proxiedRequest = Http(proxiedUrl)

    if (request.body != null && request.body.length > 0) {
      proxiedRequest = proxiedRequest.postData(request.body)
    }
    val headersForProxyRequest = request.headers
      .filter {
        case (key: String, _) => !key.toLowerCase.startsWith("cf-")
      } - "X-Forwarded-For"

    proxiedRequest = proxiedRequest
      .headers(headersForProxyRequest)
      .method(request.getMethod)

    val result = proxiedRequest.asString
    val headers = result.headers
      .map {
        // TODO(colin): is the last header what we want if there are repeated headers?
        case (key: String, values: IndexedSeq[String]) => (key, values.last)
      }

    if (List(301, 302, 303, 307) contains result.code) {
      // For redirect, we want to grab the relative url and then return our own
      // redirect.
      val newUrl = headers("Location").replaceAll("https?://[^/]+", "");
      val newHeaders = headers - "Location"
      TemporaryRedirect(
        newUrl,
        newHeaders)
    } else {
      ActionResult(
        ResponseStatus(result.code, ""),
        result.body,
        headers
      )
    }
  }

  notFound {
    // remove content type in case it was set through an action
    contentType = null
    resourceNotFound()
  }
}<commit_msg>Remove content encoding header<commit_after>package org.labdb.labdb

import java.time.{ZoneId, ZonedDateTime}
import javax.servlet.http.{HttpServletRequest, HttpServletResponse}

import org.scalatra._
import org.slf4j.{Logger, LoggerFactory}

import scalaj.http.Http

trait LabdbStack extends ScalatraServlet {

  val logger: Logger = LoggerFactory.getLogger(getClass)
  val PROXY_SUFFIX = "-backend.labdb.io"

  def nowUTC: ZonedDateTime = ZonedDateTime.now(ZoneId.of("Z"))

  def proxyHost(currHost: String): String =
    if (Env.isDev) "localhost" else currHost.replaceAllLiterally(".labdb.io", "") + PROXY_SUFFIX
  def proxyPort: String = if (Env.isDev) ":3001" else ""
  def proxyProtocol: String = if (Env.isDev) "http://" else "https://"

  def proxyRequest(implicit request: HttpServletRequest, response: HttpServletResponse): ActionResult = {
    var proxiedUrl = proxyProtocol + proxyHost(request.getServerName) + proxyPort + request.getRequestURI
    val queryString = request.getQueryString
    if (queryString != null) {
      proxiedUrl += "?" + queryString
    }
    var proxiedRequest = Http(proxiedUrl)

    if (request.body != null && request.body.length > 0) {
      proxiedRequest = proxiedRequest.postData(request.body)
    }
    val headersForProxyRequest = request.headers
      .filter {
        case (key: String, _) => !key.toLowerCase.startsWith("cf-")
      } - "X-Forwarded-For"

    proxiedRequest = proxiedRequest
      .headers(headersForProxyRequest)
      .method(request.getMethod)

    val result = proxiedRequest.asString
    val headers = result.headers
      .map {
        // TODO(colin): is the last header what we want if there are repeated headers?
        case (key: String, values: IndexedSeq[String]) => (key, values.last)
      } - "Content-Encoding"

    if (List(301, 302, 303, 307) contains result.code) {
      // For redirect, we want to grab the relative url and then return our own
      // redirect.
      val newUrl = headers("Location").replaceAll("https?://[^/]+", "");
      val newHeaders = headers - "Location"
      TemporaryRedirect(
        newUrl,
        newHeaders)
    } else {
      ActionResult(
        ResponseStatus(result.code, ""),
        result.body,
        headers
      )
    }
  }

  notFound {
    // remove content type in case it was set through an action
    contentType = null
    resourceNotFound()
  }
}
{'fraction_non_alphanumeric': 0.11424957841483979, 'fraction_numerical': 0.007588532883642495, 'mean_word_length': 3.162280701754386, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3623988', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1422, 'n_words': 360}