<commit_before>import 
  asynchttpserver,
  asyncdispatch, 
  times, 
  strutils, 
  pegs, 
  strtabs, 
  logger,
  cgi,
  os,
  json,
  tables,
  jwt
import 
  types, 
  utils, 
  api_v1,
  api_v2,
  api_v3,
  api_v4

export 
  api_v4

proc getReqInfo(req: LSRequest): string =
  var url = req.url.path
  if req.url.anchor != "":
    url = url & "#" & req.url.anchor
  if req.url.query != "":
    url = url & "?" & req.url.query
  return req.hostname & " " & $req.reqMethod & " " & url

proc handleCtrlC() {.noconv.} =
  echo ""
  LOG.info("Exiting...")
  quit()

proc processApiUrl(req: LSRequest, LS: LiteStore, info: ResourceInfo): LSResponse = 
  let reqUri = info.resource & "/" & info.id
  let uriParts = reqUri.split("/")
  let uri = "/" & uriParts[0..uriParts.len-2].join("/") & "/"
  let reqMethod = $req.reqMethod
  # Authentication/Authorization
  if LS.auth != newJNull():
    if LS.auth["access"].hasKey(uri):
      let access = LS.auth["access"][uri]
      if access.hasKey(reqMethod):
        if not req.headers.hasKey("Authorization"): 
          return resError(Http401, "Unauthorized - No token")
        let token = req.headers["Authorization"].replace(peg"^ 'Bearer '", "")
        # Validate token
        try:
          let jwt = token.toJwt()
          var sig = LS.auth["signature"].getStr 
          discard verify(jwt, sig) 
          verifyTimeClaims(jwt)
          # Validate scope
          let scopes = $jwt.claims["scope"].node.str.split(peg"\s+")
          if not scopes.contains access[reqMethod].getStr:
            return resError(Http403, "Forbidden - You are not permitted to access this resource")
        except:
          return resError(Http401, "Unauthorized - Invalid token")
  if info.version == "v4":
    if info.resource.match(peg"^docs / info / tags$"):
      return api_v4.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v4.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v3":
    if info.resource.match(peg"^docs / info$"):
      return api_v3.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v3.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v2":
    if info.resource.match(peg"^docs / info$"):
      return api_v2.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v2.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v1": 
    if info.resource.match(peg"^docs / info$"):
      return api_v1.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v1.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  else:
    if info.version == "v1" or info.version == "v2" or info.version == "v3" or info.version == "v4":
      return resError(Http400, "Bad Request - Invalid API version: $1" % info.version)
    else:
      if info.resource.decodeURL.strip == "":
        return resError(Http400, "Bad Request - No resource specified." % info.resource)
      else:
        return resError(Http404, "Resource Not Found: $1" % info.resource)

proc process*(req: LSRequest, LS: LiteStore): LSResponse {.gcsafe.}=
  var matches = @["", "", ""]
  template route(req: LSRequest, peg: Peg, op: untyped): untyped =
    if req.url.path.find(peg, matches) != -1:
      op
  try: 
    var info: ResourceInfo
    req.route peg"^\/?$":
      info.version = "v4"
      info.resource = "info"
      return req.processApiUrl(LS, info)
    req.route peg"^\/favicon.ico$":
      result.code = Http200
      result.content = LS.favicon
      result.headers = ctHeader("image/x-icon")
      return result
    req.route PEG_DEFAULT_URL:
      info.version = "v4"
      info.resource = matches[0]
      info.id = matches[1]
      return req.processApiUrl(LS, info)
    req.route PEG_URL:
      info.version = matches[0]
      info.resource = matches[1]
      info.id = matches[2]
      return req.processApiUrl(LS, info)
    raise newException(EInvalidRequest, req.getReqInfo())
  except EInvalidRequest:
    let e = (ref EInvalidRequest)(getCurrentException())
    let trace = e.getStackTrace()
    return resError(Http404, "Resource Not Found: $1" % getCurrentExceptionMsg().split(" ")[2], trace)
  except:
    let e = getCurrentException()
    let trace = e.getStackTrace()
    return resError(Http500, "Internal Server Error: $1" % getCurrentExceptionMsg(), trace)

setControlCHook(handleCtrlC)

proc serve*(LS: LiteStore) =
  var server = newAsyncHttpServer()
  proc handleHttpRequest(req: LSRequest): Future[void] {.async, gcsafe, closure.} =
    LOG.info(getReqInfo(req).replace("$", "$$"))
    let res = req.process(LS)
    let areq = asynchttpserver.Request(req)
    await areq.respond(res.code, res.content, res.headers)
  echo(LS.appname & " v" & LS.appversion & " started on " & LS.address & ":" & $LS.port & ".")
  if LS.mount:
    echo("Mirroring datastore changes to: " & LS.directory)
  asyncCheck server.serve(LS.port.Port, handleHttpRequest, LS.address)

<commit_msg>Supporting wildcard resources and multiple resource scopes.<commit_after>import 
  asynchttpserver,
  asyncdispatch, 
  times, 
  strutils, 
  pegs, 
  strtabs, 
  logger,
  cgi,
  os,
  json,
  tables,
  jwt
import 
  types, 
  utils, 
  api_v1,
  api_v2,
  api_v3,
  api_v4

export 
  api_v4

proc getReqInfo(req: LSRequest): string =
  var url = req.url.path
  if req.url.anchor != "":
    url = url & "#" & req.url.anchor
  if req.url.query != "":
    url = url & "?" & req.url.query
  return req.hostname & " " & $req.reqMethod & " " & url

proc handleCtrlC() {.noconv.} =
  echo ""
  LOG.info("Exiting...")
  quit()

proc processApiUrl(req: LSRequest, LS: LiteStore, info: ResourceInfo): LSResponse = 
  let uriSingle = "/" & info.resource & "/" & info.id
  let uriParts = uriSingle.split("/")
  let uriAny = uriParts[0..uriParts.len-2].join("/") & "/*"
  let reqMethod = $req.reqMethod
  # Authentication/Authorization
  if LS.auth != newJNull():
    var uri = ""
    if LS.auth["access"].hasKey(uriSingle):
      uri = uriSingle
    elif LS.auth["access"].hasKey(uriAny):
      uri = uriAny
    if uri != "":
      let access = LS.auth["access"][uri]
      if access.hasKey(reqMethod):
        LOG.debug("Authenticating: " & reqMethod & " " & uri)
        if not req.headers.hasKey("Authorization"): 
          return resError(Http401, "Unauthorized - No token")
        let token = req.headers["Authorization"].replace(peg"^ 'Bearer '", "")
        # Validate token
        try:
          let jwt = token.toJwt()
          var sig = LS.auth["signature"].getStr 
          discard verify(jwt, sig) 
          verifyTimeClaims(jwt)
          let scopes = access[reqMethod]
          # Validate scope
          var authorized = ""
          let reqScopes = ($jwt.claims["scope"].node.str).split(peg"\s+")
          LOG.debug("Resource scopes: " & $scopes)
          LOG.debug("Request scopes: " & $reqScopes)
          for scope in scopes:
            for reqScope in reqScopes:
              if reqScope == scope.getStr:
                authorized = scope.getStr
                break
          if authorized == "":
            return resError(Http403, "Forbidden - You are not permitted to access this resource")
          LOG.debug("Authorization successful: " & authorized)
        except:
          return resError(Http401, "Unauthorized - Invalid token")
  if info.version == "v4":
    if info.resource.match(peg"^docs / info / tags$"):
      return api_v4.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v4.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v3":
    if info.resource.match(peg"^docs / info$"):
      return api_v3.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v3.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v2":
    if info.resource.match(peg"^docs / info$"):
      return api_v2.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v2.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  elif info.version == "v1": 
    if info.resource.match(peg"^docs / info$"):
      return api_v1.route(req, LS, info.resource, info.id)
    elif info.resource.match(peg"^dir$"):
      if LS.directory.len > 0:
        return api_v1.serveFile(req, LS, info.id)
      else:
        return resError(Http400, "Bad Request - Not serving any directory." % info.version)
    else:
      return resError(Http404, "Resource Not Found: $1" % info.resource)
  else:
    if info.version == "v1" or info.version == "v2" or info.version == "v3" or info.version == "v4":
      return resError(Http400, "Bad Request - Invalid API version: $1" % info.version)
    else:
      if info.resource.decodeURL.strip == "":
        return resError(Http400, "Bad Request - No resource specified." % info.resource)
      else:
        return resError(Http404, "Resource Not Found: $1" % info.resource)

proc process*(req: LSRequest, LS: LiteStore): LSResponse {.gcsafe.}=
  var matches = @["", "", ""]
  template route(req: LSRequest, peg: Peg, op: untyped): untyped =
    if req.url.path.find(peg, matches) != -1:
      op
  try: 
    var info: ResourceInfo
    req.route peg"^\/?$":
      info.version = "v4"
      info.resource = "info"
      return req.processApiUrl(LS, info)
    req.route peg"^\/favicon.ico$":
      result.code = Http200
      result.content = LS.favicon
      result.headers = ctHeader("image/x-icon")
      return result
    req.route PEG_DEFAULT_URL:
      info.version = "v4"
      info.resource = matches[0]
      info.id = matches[1]
      return req.processApiUrl(LS, info)
    req.route PEG_URL:
      info.version = matches[0]
      info.resource = matches[1]
      info.id = matches[2]
      return req.processApiUrl(LS, info)
    raise newException(EInvalidRequest, req.getReqInfo())
  except EInvalidRequest:
    let e = (ref EInvalidRequest)(getCurrentException())
    let trace = e.getStackTrace()
    return resError(Http404, "Resource Not Found: $1" % getCurrentExceptionMsg().split(" ")[2], trace)
  except:
    let e = getCurrentException()
    let trace = e.getStackTrace()
    return resError(Http500, "Internal Server Error: $1" % getCurrentExceptionMsg(), trace)

setControlCHook(handleCtrlC)

proc serve*(LS: LiteStore) =
  var server = newAsyncHttpServer()
  proc handleHttpRequest(req: LSRequest): Future[void] {.async, gcsafe, closure.} =
    LOG.info(getReqInfo(req).replace("$", "$$"))
    let res = req.process(LS)
    let areq = asynchttpserver.Request(req)
    await areq.respond(res.code, res.content, res.headers)
  echo(LS.appname & " v" & LS.appversion & " started on " & LS.address & ":" & $LS.port & ".")
  if LS.mount:
    echo("Mirroring datastore changes to: " & LS.directory)
  asyncCheck server.serve(LS.port.Port, handleHttpRequest, LS.address)


{'fraction_non_alphanumeric': 0.14089180221736666, 'fraction_numerical': 0.015375900299425427, 'mean_word_length': 2.835505896958411, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '340539', 'n_tokens_mistral': 4241, 'n_tokens_neox': 4090, 'n_words': 989}