<commit_before>package slamdata.engine.api

import scala.collection.immutable.{TreeSet}

import slamdata.engine._
import slamdata.engine.sql.Query
import slamdata.engine.config._
import slamdata.engine.fs._
import slamdata.engine.fp._

import argonaut._
import Argonaut._

import org.http4s.{Query => HQuery, _}
import org.http4s.dsl.{Path => HPath, _}
import org.http4s.argonaut._
import org.http4s.server._
import org.http4s.util.{CaseInsensitiveString, Renderable}

import scalaz._
import Scalaz._

import scalaz.concurrent._
import scalaz.stream._

class FileSystemApi(fs: FSTable[Backend]) {
  import java.io.{FileSystem => _, _}
  import Method.{MOVE, OPTIONS}

  private def jsonStream(v: Process[Task, Data])(implicit EE: EncodeJson[DataEncodingError]): Task[Response] = {
    val codec = DataCodec.Readable
    Ok(v.map { data =>
      DataCodec.render(data)(codec).fold(err => EE.encode(err).toString, identity) + "\r\n"
    })
  }

  private def lookupBackend(path: Path): Task[Response] \/ (Backend, Path, Path) =
    fs.lookup(path) \/> (NotFound("No data source is mounted to the path " + path))

  private def backendFor(path: Path): Task[Response] \/ (Backend, Path) =
    lookupBackend(path).map { case (backend, mountPath, _) => (backend, mountPath) }

  private def dataSourceFor(path: Path): Task[Response] \/ (FileSystem, Path) =
    lookupBackend(path).map { case (backend, _, relPath) => (backend.dataSource, relPath) }

  private def errorResponse(status: org.http4s.dsl.impl.EntityResponseGenerator, e: Throwable) = {
    e match {
      case PhaseError(phases, causedBy) => status(Json.obj(
          "error"  := causedBy.getMessage,
          "phases" := phases))

      case _ => status(e.getMessage)
    }
  }

  private def vars(req: Request) = Variables(req.params.map { case (k, v) => (VarName(k), VarValue(v)) })

  private val QueryParameterMustContainQuery = BadRequest("The request must contain a query")
  private val POSTContentMustContainQuery    = BadRequest("The body of the POST must contain a query")
  private val DestinationHeaderMustExist     = BadRequest("The 'Destination' header must be specified")

  private def upload[A](body: String, path: Path, f: (FileSystem, Path, Process[Task, Data]) => List[Throwable] \/ Unit) = {
    val codec = DataCodec.Precise
    def parseJsonLines(str: String): (List[WriteError], List[Data]) =
      unzipDisj(str.split("\n").map(line => DataCodec.parse(line)(codec).leftMap(
        e => WriteError(Data.Str("parse error: " + line), Some(e.message)))).toList)

    def errorBody(status: org.http4s.dsl.impl.EntityResponseGenerator, errs: List[Throwable])(implicit EJ: EncodeJson[WriteError]) =
      status(Json(
        "errors" := errs.map {
          case e @ WriteError(_, _) => EJ.encode(e)
          case e: slamdata.engine.Error => Json("detail" := e.message)
          case e => Json("detail" := e.toString)
        }))

    (for {
      t1   <- dataSourceFor(path)
      (dataSource, relPath) = t1

      (errs, json) = parseJsonLines(body)
      _    <- if (!errs.isEmpty) -\/ (errorBody(BadRequest, errs)) else \/- (())

      _    <- f(dataSource, relPath, Process.emitAll(json)).leftMap {
        case (pe @ PathError(_)) :: Nil => errorBody(BadRequest, pe :: Nil)
        case es                         => errorBody(InternalServerError, es)
      }
    } yield Ok("")).fold(identity, identity)
  }

  object AsPath {
    def unapply(p: HPath): Option[Path] = {
      Some(Path("/" + p.toList.map(java.net.URLDecoder.decode(_, "UTF-8")).mkString("/")))
    }
  }

  object AsDirPath {
    def unapply(p: HPath): Option[Path] = AsPath.unapply(p).map(_.asDir)
  }

  implicit val QueryDecoder = new QueryParamDecoder[Query] {
    def decode(value: QueryParameterValue): ValidationNel[ParseFailure, Query] =
      Query(value.value).successNel[ParseFailure]
  }
  object Q extends QueryParamDecoderMatcher[Query]("q")

  object Offset extends OptionalQueryParamDecoderMatcher[Long]("offset")
  object Limit extends OptionalQueryParamDecoderMatcher[Long]("limit")

  // Note: CORS middleware is coming in http4s post-0.6.5
  val corsResponse = Ok().putHeaders(
    AccessControlAllowOriginAll,
    `Access-Control-Allow-Methods`(List("GET", "POST", "OPTIONS")),
    `Access-Control-Max-Age`(20*24*60*60),
    `Access-Control-Allow-Headers`(List(Destination)))  // NB: actually needed for POST only

  def queryService = {<commit_msg>Add all methods used by SlamData to CORS list<commit_after>package slamdata.engine.api

import scala.collection.immutable.{TreeSet}

import slamdata.engine._
import slamdata.engine.sql.Query
import slamdata.engine.config._
import slamdata.engine.fs._
import slamdata.engine.fp._

import argonaut._
import Argonaut._

import org.http4s.{Query => HQuery, _}
import org.http4s.dsl.{Path => HPath, _}
import org.http4s.argonaut._
import org.http4s.server._
import org.http4s.util.{CaseInsensitiveString, Renderable}

import scalaz._
import Scalaz._

import scalaz.concurrent._
import scalaz.stream._

class FileSystemApi(fs: FSTable[Backend]) {
  import java.io.{FileSystem => _, _}
  import Method.{MOVE, OPTIONS}

  private def jsonStream(v: Process[Task, Data])(implicit EE: EncodeJson[DataEncodingError]): Task[Response] = {
    val codec = DataCodec.Readable
    Ok(v.map { data =>
      DataCodec.render(data)(codec).fold(err => EE.encode(err).toString, identity) + "\r\n"
    })
  }

  private def lookupBackend(path: Path): Task[Response] \/ (Backend, Path, Path) =
    fs.lookup(path) \/> (NotFound("No data source is mounted to the path " + path))

  private def backendFor(path: Path): Task[Response] \/ (Backend, Path) =
    lookupBackend(path).map { case (backend, mountPath, _) => (backend, mountPath) }

  private def dataSourceFor(path: Path): Task[Response] \/ (FileSystem, Path) =
    lookupBackend(path).map { case (backend, _, relPath) => (backend.dataSource, relPath) }

  private def errorResponse(status: org.http4s.dsl.impl.EntityResponseGenerator, e: Throwable) = {
    e match {
      case PhaseError(phases, causedBy) => status(Json.obj(
          "error"  := causedBy.getMessage,
          "phases" := phases))

      case _ => status(e.getMessage)
    }
  }

  private def vars(req: Request) = Variables(req.params.map { case (k, v) => (VarName(k), VarValue(v)) })

  private val QueryParameterMustContainQuery = BadRequest("The request must contain a query")
  private val POSTContentMustContainQuery    = BadRequest("The body of the POST must contain a query")
  private val DestinationHeaderMustExist     = BadRequest("The 'Destination' header must be specified")

  private def upload[A](body: String, path: Path, f: (FileSystem, Path, Process[Task, Data]) => List[Throwable] \/ Unit) = {
    val codec = DataCodec.Precise
    def parseJsonLines(str: String): (List[WriteError], List[Data]) =
      unzipDisj(str.split("\n").map(line => DataCodec.parse(line)(codec).leftMap(
        e => WriteError(Data.Str("parse error: " + line), Some(e.message)))).toList)

    def errorBody(status: org.http4s.dsl.impl.EntityResponseGenerator, errs: List[Throwable])(implicit EJ: EncodeJson[WriteError]) =
      status(Json(
        "errors" := errs.map {
          case e @ WriteError(_, _) => EJ.encode(e)
          case e: slamdata.engine.Error => Json("detail" := e.message)
          case e => Json("detail" := e.toString)
        }))

    (for {
      t1   <- dataSourceFor(path)
      (dataSource, relPath) = t1

      (errs, json) = parseJsonLines(body)
      _    <- if (!errs.isEmpty) -\/ (errorBody(BadRequest, errs)) else \/- (())

      _    <- f(dataSource, relPath, Process.emitAll(json)).leftMap {
        case (pe @ PathError(_)) :: Nil => errorBody(BadRequest, pe :: Nil)
        case es                         => errorBody(InternalServerError, es)
      }
    } yield Ok("")).fold(identity, identity)
  }

  object AsPath {
    def unapply(p: HPath): Option[Path] = {
      Some(Path("/" + p.toList.map(java.net.URLDecoder.decode(_, "UTF-8")).mkString("/")))
    }
  }

  object AsDirPath {
    def unapply(p: HPath): Option[Path] = AsPath.unapply(p).map(_.asDir)
  }

  implicit val QueryDecoder = new QueryParamDecoder[Query] {
    def decode(value: QueryParameterValue): ValidationNel[ParseFailure, Query] =
      Query(value.value).successNel[ParseFailure]
  }
  object Q extends QueryParamDecoderMatcher[Query]("q")

  object Offset extends OptionalQueryParamDecoderMatcher[Long]("offset")
  object Limit extends OptionalQueryParamDecoderMatcher[Long]("limit")

  // Note: CORS middleware is coming in http4s post-0.6.5
  val corsResponse = Ok().putHeaders(
    AccessControlAllowOriginAll,
    `Access-Control-Allow-Methods`(List("GET", "PUT", "POST", "DELETE", "MOVE", "OPTIONS")),
    `Access-Control-Max-Age`(20*24*60*60),
    `Access-Control-Allow-Headers`(List(Destination)))  // NB: actually needed for POST only

  def queryService = {
{'fraction_non_alphanumeric': 0.14417143498260968, 'fraction_numerical': 0.004936609446875351, 'mean_word_length': 3.751599147121535, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '246742', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2859, 'n_words': 635}