<commit_before>package com.socrata.tileserver
package services

import java.net.URLDecoder
import java.nio.charset.StandardCharsets.UTF_8
import javax.servlet.http.HttpServletResponse
import javax.servlet.http.HttpServletResponse.{SC_NOT_MODIFIED => ScNotModified}
import javax.servlet.http.HttpServletResponse.{SC_OK => ScOk}
import scala.util.{Try, Success, Failure}

import com.rojoma.json.v3.ast.JValue
import com.rojoma.json.v3.codec.JsonEncode.toJValue
import com.rojoma.json.v3.interpolation._
import com.rojoma.json.v3.io.JsonReader
import com.rojoma.json.v3.io.JsonReaderException
import com.vividsolutions.jts.geom.GeometryFactory
import org.apache.commons.io.IOUtils
import org.slf4j.{Logger, LoggerFactory, MDC}

import com.socrata.thirdparty.curator.CuratedServiceClient
import com.socrata.http.client.{RequestBuilder, Response}
import com.socrata.http.server.implicits._
import com.socrata.http.server.responses._
import com.socrata.http.server.routing.{SimpleResource, TypedPathComponent}
import com.socrata.http.server.util.RequestId.{RequestId, ReqIdHeader, getFromRequest}
import com.socrata.http.server.{HttpRequest, HttpResponse, HttpService}
import com.socrata.thirdparty.geojson.{GeoJson, FeatureCollectionJson, FeatureJson}

import TileService._
import exceptions.InvalidGeoJsonException
import util.TileEncoder.Feature
import util.{HeaderFilter, QuadTile, TileEncoder}

/** Service that provides the actual tiles.
  *
  * @constructor This should only be called once, by the main application.
  * @param client The client to talk to the upstream geo-json service.
  */
case class TileService(client: CuratedServiceClient) extends SimpleResource {
  /** Type of callback we will be passing to `client`. */
  type Callback = Response => HttpResponse

  /** The types (file extensions) supported by this endpoint. */
  val types: Set[String] = Set("pbf", "bpbf", "json", "txt")

  // Call to the underlying service.
  private[services] def geoJsonQuery(requestId: RequestId,
                                     req: HttpRequest,
                                     id: String,
                                     params: Map[String, String],
                                     callback: Response => HttpResponse): HttpResponse = {
    val headers = HeaderFilter.headers(req)

    val jsonReq = { base: RequestBuilder =>
      val req = base.path(Seq("id", s"$id.geojson")).
        addHeaders(headers).
        addHeader(ReqIdHeader -> requestId).
        query(params).get
      logger.info(URLDecoder.decode(req.toString, UTF_8.name))
      req
    }

    client.execute(jsonReq, callback)
  }

  private val handleErrors: PartialFunction[Throwable, HttpResponse] = {
    case readerEx: JsonReaderException =>
      fatal("Invalid JSON returned from underlying service", readerEx)
    case geoJsonEx: InvalidGeoJsonException =>
      fatal("Invalid Geo-JSON returned from underlying service", geoJsonEx)
    case unknown =>
      fatal("Unknown error", unknown)
  }

  private[services] def processResponse(tile: QuadTile,
                                        ext: String): Callback =
  { resp: Response =>
    def createResponse(parsed: (JValue, Seq[FeatureJson])): HttpResponse = {
      val (jValue, features) = parsed
      if (logger.isDebugEnabled) {
        logger.debug(s"Underlying json: ${jValue.toString}")
      }

      val enc = TileEncoder(rollup(tile, features))
      val payload = ext match {
        case "pbf" => ContentType("application/octet-stream") ~> ContentBytes(enc.bytes)
        case "bpbf" => Content("text/plain", enc.base64)
        case "txt" => Content("text/plain", enc.toString)
        case "json" => Json(jValue)
      }

      OK ~> HeaderFilter.extract(resp) ~> payload
    }

    lazy val result = resp.resultCode match {
      case ScOk => features(resp).map(createResponse).recover(handleErrors).get
      case ScNotModified => NotModified
      case _ => echoResponse(resp)
    }

    Header("Access-Control-Allow-Origin", "*") ~> result
  }

  // Do the actual heavy lifting for the request handling.
  private[services] def handleRequest(req: HttpRequest,
                                      identifier: String,
                                      pointColumn: String,
                                      tile: QuadTile,
                                      ext: String) : HttpResponse = {
    val withinBox = tile.withinBox(pointColumn)

    Try {<commit_msg>Exclude debug enabled check<commit_after>package com.socrata.tileserver
package services

import java.net.URLDecoder
import java.nio.charset.StandardCharsets.UTF_8
import javax.servlet.http.HttpServletResponse
import javax.servlet.http.HttpServletResponse.{SC_NOT_MODIFIED => ScNotModified}
import javax.servlet.http.HttpServletResponse.{SC_OK => ScOk}
import scala.util.{Try, Success, Failure}

import com.rojoma.json.v3.ast.JValue
import com.rojoma.json.v3.codec.JsonEncode.toJValue
import com.rojoma.json.v3.interpolation._
import com.rojoma.json.v3.io.JsonReader
import com.rojoma.json.v3.io.JsonReaderException
import com.vividsolutions.jts.geom.GeometryFactory
import org.apache.commons.io.IOUtils
import org.slf4j.{Logger, LoggerFactory, MDC}

import com.socrata.thirdparty.curator.CuratedServiceClient
import com.socrata.http.client.{RequestBuilder, Response}
import com.socrata.http.server.implicits._
import com.socrata.http.server.responses._
import com.socrata.http.server.routing.{SimpleResource, TypedPathComponent}
import com.socrata.http.server.util.RequestId.{RequestId, ReqIdHeader, getFromRequest}
import com.socrata.http.server.{HttpRequest, HttpResponse, HttpService}
import com.socrata.thirdparty.geojson.{GeoJson, FeatureCollectionJson, FeatureJson}

import TileService._
import exceptions.InvalidGeoJsonException
import util.TileEncoder.Feature
import util.{HeaderFilter, QuadTile, TileEncoder}

/** Service that provides the actual tiles.
  *
  * @constructor This should only be called once, by the main application.
  * @param client The client to talk to the upstream geo-json service.
  */
case class TileService(client: CuratedServiceClient) extends SimpleResource {
  /** Type of callback we will be passing to `client`. */
  type Callback = Response => HttpResponse

  /** The types (file extensions) supported by this endpoint. */
  val types: Set[String] = Set("pbf", "bpbf", "json", "txt")

  // Call to the underlying service.
  private[services] def geoJsonQuery(requestId: RequestId,
                                     req: HttpRequest,
                                     id: String,
                                     params: Map[String, String],
                                     callback: Response => HttpResponse): HttpResponse = {
    val headers = HeaderFilter.headers(req)

    val jsonReq = { base: RequestBuilder =>
      val req = base.path(Seq("id", s"$id.geojson")).
        addHeaders(headers).
        addHeader(ReqIdHeader -> requestId).
        query(params).get
      logger.info(URLDecoder.decode(req.toString, UTF_8.name))
      req
    }

    client.execute(jsonReq, callback)
  }

  private val handleErrors: PartialFunction[Throwable, HttpResponse] = {
    case readerEx: JsonReaderException =>
      fatal("Invalid JSON returned from underlying service", readerEx)
    case geoJsonEx: InvalidGeoJsonException =>
      fatal("Invalid Geo-JSON returned from underlying service", geoJsonEx)
    case unknown =>
      fatal("Unknown error", unknown)
  }

  private[services] def processResponse(tile: QuadTile,
                                        ext: String): Callback =
  { resp: Response =>
    def createResponse(parsed: (JValue, Seq[FeatureJson])): HttpResponse = {
      val (jValue, features) = parsed

      // $COVERAGE-OFF$
      if (logger.isDebugEnabled) {
        logger.debug(s"Underlying json: ${jValue.toString}")
      }
      // $COVERAGE-ON$

      val enc = TileEncoder(rollup(tile, features))
      val payload = ext match {
        case "pbf" => ContentType("application/octet-stream") ~> ContentBytes(enc.bytes)
        case "bpbf" => Content("text/plain", enc.base64)
        case "txt" => Content("text/plain", enc.toString)
        case "json" => Json(jValue)
      }

      OK ~> HeaderFilter.extract(resp) ~> payload
    }

    lazy val result = resp.resultCode match {
      case ScOk => features(resp).map(createResponse).recover(handleErrors).get
      case ScNotModified => NotModified
      case _ => echoResponse(resp)
    }

    Header("Access-Control-Allow-Origin", "*") ~> result
  }

  // Do the actual heavy lifting for the request handling.
  private[services] def handleRequest(req: HttpRequest,
                                      identifier: String,
                                      pointColumn: String,
                                      tile: QuadTile,
                                      ext: String) : HttpResponse = {
    val withinBox = tile.withinBox(pointColumn)

    Try {
{'fraction_non_alphanumeric': 0.10413407821229051, 'fraction_numerical': 0.0022346368715083797, 'mean_word_length': 2.9449096518289997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '937298', 'n_tokens_mistral': 2558, 'n_tokens_neox': 2508, 'n_words': 593}