<commit_before>module MemoryWayback.App

open System
open System.Diagnostics
open Suave
open Suave.Web
open Suave.Json
open Suave.Http
open Suave.Http
open Suave.Files
open Suave.Successful
open Suave.RequestErrors
open Suave.Operators
open Suave.EventSource
open Suave.Filters
open Suave.Writers
open Suave.Utils
open System.IO
open System.Text
open ExtCore.Control
open Nessos.FsPickler
open Nessos.FsPickler.Json
open MemoryWayback.DbTypes
open MemoryWayback.Types
open MemoryWayback.MediaQuery
open MemoryWayback.Persistence
open MemoryWayback.OrmlitePersistence
open MemoryWayback.MemoryPersistence
open MemoryWayback.FileHelper

//let logger = Loggers.sane_defaults_for Debug

let pickler = FsPickler.CreateJsonSerializer(indent = true, omitHeader = true)

let getDbPersistence () = OrmlitePersistence() :> IPersistence

let getMemoryPersistence dir = 
  let p = MemoryPersistence("mem",[],mediasId) :> IPersistence
  match dir with
  | Some d -> MediaLibraryUpdate.updateMedia realFileHelper d p
  | None -> p

let mutable Persistence = getMemoryPersistence None

let persist stateFunc =
  let (res,p) = stateFunc Persistence
  Persistence <- p
  res

let greetings q =
  defaultArg (Option.ofChoice(q ^^ "name")) "World" |> sprintf "Hello %s"

let okJson o =
  OK (pickler.PickleToString o)
    >=> setMimeType "application/json"

let parseDateParam ctx parm deflt =
  let date = choice {
    let! stringDate = ctx.request.queryParam parm
    return DateTime.Parse stringDate
  }
  Choice.orDefault deflt date

let parseTypes ctx parm deflt =
  Choice.orDefault deflt (choice {
    let! types = ctx.request.queryParam parm
    return (types.Split ',')
      |> Array.map (fun v -> Enum.Parse(typeof<MediaType>, v) :?> MediaType)
      |> Array.toList
    })

let handleQuery ctx =
  printfn "%A" ctx.request.query
  let fromDt = parseDateParam ctx "from" (DateTime(0L))
  let toDt = parseDateParam ctx "to" DateTime.UtcNow
  let typeEnums = parseTypes ctx "types" [ MediaType.Photo ; MediaType.Video ]
  //printfn "%A - %A : %A" fromDt toDt typeEnums
  let o = {
    Query.From = fromDt
    To = toDt
    Types = typeEnums
  }
  printfn "%A" o
  (okJson <| persist (getResults o)) ctx


let app =
  choose
    [ GET >=> choose
        [ 
          path "/media-query" >=> handleQuery 
          pathScan "/media/%s" Files.browseFileHome
        ]
    ]

let defaultArgs = [| "db" ; "." |]

let startApp (MediaDirectory dir) =
  let cfg = { defaultConfig with homeFolder = Some dir }
  startWebServer cfg app
  0

let start (args : string[]) =
  let realArgs = if (args.Length = 0) then defaultArgs else args

  let dir = MediaDirectory realArgs.[1]
  match realArgs.[0] with
  | "db" -> startApp dir<commit_msg>use /api in path<commit_after>module MemoryWayback.App

open System
open System.Diagnostics
open Suave
open Suave.Web
open Suave.Json
open Suave.Http
open Suave.Http
open Suave.Files
open Suave.Successful
open Suave.RequestErrors
open Suave.Operators
open Suave.EventSource
open Suave.Filters
open Suave.Writers
open Suave.Utils
open System.IO
open System.Text
open ExtCore.Control
open Nessos.FsPickler
open Nessos.FsPickler.Json
open MemoryWayback.DbTypes
open MemoryWayback.Types
open MemoryWayback.MediaQuery
open MemoryWayback.Persistence
open MemoryWayback.OrmlitePersistence
open MemoryWayback.MemoryPersistence
open MemoryWayback.FileHelper

//let logger = Loggers.sane_defaults_for Debug

let pickler = FsPickler.CreateJsonSerializer(indent = true, omitHeader = true)

let getDbPersistence () = OrmlitePersistence() :> IPersistence

let getMemoryPersistence dir = 
  let p = MemoryPersistence("mem",[],mediasId) :> IPersistence
  match dir with
  | Some d -> MediaLibraryUpdate.updateMedia realFileHelper d p
  | None -> p

let mutable Persistence = getMemoryPersistence None

let persist stateFunc =
  let (res,p) = stateFunc Persistence
  Persistence <- p
  res

let greetings q =
  defaultArg (Option.ofChoice(q ^^ "name")) "World" |> sprintf "Hello %s"

let okJson o =
  OK (pickler.PickleToString o)
    >=> setMimeType "application/json"

let parseDateParam ctx parm deflt =
  let date = choice {
    let! stringDate = ctx.request.queryParam parm
    return DateTime.Parse stringDate
  }
  Choice.orDefault deflt date

let parseTypes ctx parm deflt =
  Choice.orDefault deflt (choice {
    let! types = ctx.request.queryParam parm
    return (types.Split ',')
      |> Array.map (fun v -> Enum.Parse(typeof<MediaType>, v) :?> MediaType)
      |> Array.toList
    })

let handleQuery ctx =
  printfn "%A" ctx.request.query
  let fromDt = parseDateParam ctx "from" (DateTime(0L))
  let toDt = parseDateParam ctx "to" DateTime.UtcNow
  let typeEnums = parseTypes ctx "types" [ MediaType.Photo ; MediaType.Video ]
  //printfn "%A - %A : %A" fromDt toDt typeEnums
  let o = {
    Query.From = fromDt
    To = toDt
    Types = typeEnums
  }
  printfn "%A" o
  (okJson <| persist (getResults o)) ctx


let app =
  choose
    [ GET >=> choose
        [ 
          path "/api/media-query" >=> handleQuery 
          pathScan "/api/media/%s" Files.browseFileHome
        ]
    ]

let defaultArgs = [| "db" ; "." |]

let startApp (MediaDirectory dir) =
  let cfg = { defaultConfig with homeFolder = Some dir }
  startWebServer cfg app
  0

let start (args : string[]) =
  let realArgs = if (args.Length = 0) then defaultArgs else args

  let dir = MediaDirectory realArgs.[1]
  match realArgs.[0] with
  | "db" -> startApp dir
{'fraction_non_alphanumeric': 0.09561387410534043, 'fraction_numerical': 0.0018351991191044228, 'mean_word_length': 3.68615649183147, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7453414', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1778, 'n_words': 522}