<commit_before>#if INTERACTIVE
#r "../packages/Suave/lib/net40/Suave.dll"
#r "../packages/FSharp.Data/lib/net40/FSharp.Data.dll"
#r "../packages/Newtonsoft.Json/lib/net40/Newtonsoft.Json.dll"
#load "../packages/FSharp.Azure.StorageTypeProvider/StorageTypeProvider.fsx"
#load "config.fs" "common/serializer.fs" "pivot.fs" "storage.fs" "listing.fs" 
#else
module Gallery.App
#endif
open System
open System.IO

open Suave
open Suave.Filters
open Suave.Writers
open Suave.Operators
open FSharp.Data
open Gallery.CsvService

// --------------------------------------------------------------------------------------
// Server with /upload endpoint and /csv for Pivot type provider
// --------------------------------------------------------------------------------------

#if INTERACTIVE
let root = "https://localhost:8897"
#else
let root = "https://gallery-csv-service.azurewebsites.net"
#endif

let app =
  choose [
    GET >=> path "/" >=> Successful.OK "Service is running..." 
    POST >=> path "/update" >=> Storage.updateRecord
    POST >=> path "/upload" >=> Storage.uploadFile
    GET >=> path "/tags" >=> request (fun _ ctx -> async {
        let! files = Storage.getRecords ()
        let tags = 
          files 
          |> Seq.collect (fun f -> f.tags) 
          |> Seq.map (fun t -> JsonValue.Array [| JsonValue.String (Listing.getTagId t); JsonValue.String t |])
          |> Array.ofSeq
          |> JsonValue.Array
        return! Successful.OK (tags.ToString()) ctx })
    setHeader  "Access-Control-Allow-Origin" "*"
    >=> setHeader "Access-Control-Allow-Headers" "content-type"
    >=> choose [
      OPTIONS >=> Successful.OK "CORS approved"
      GET >=> pathScan "/providers/listing%s" (fun _ ctx -> async { 
        let! files = Storage.getRecords ()
        return! Listing.handleRequest root files ctx })
      GET >=> pathScan "/providers/csv/%s" (fun source ctx -> async {
        let! file = Storage.fetchFile source
        match file with 
        | None -> return! RequestErrors.BAD_REQUEST (sprintf "File with id '%s' does not exist!" source) ctx
        | Some (meta, data) -> return! Pivot.handleRequest meta data (List.map fst ctx.request.query) ctx }) ]
  ]

// When port was specified, we start the app (in Azure), 
// otherwise we do nothing (it is hosted by 'build.fsx')
match System.Environment.GetCommandLineArgs() |> Seq.tryPick (fun s ->
    if s.StartsWith("port=") then Some(int(s.Substring("port=".Length)))
    else None ) with
| Some port ->
    let serverConfig =
      { Web.defaultConfig with
          maxContentLength = 1024 * 1024 * 50
          bindings = [ HttpBinding.createSimple HTTP "127.0.0.1" port ] }
    Web.startWebServer serverConfig app
| _ -> ()



<commit_msg>Return distinct tags<commit_after>#if INTERACTIVE
#r "../packages/Suave/lib/net40/Suave.dll"
#r "../packages/FSharp.Data/lib/net40/FSharp.Data.dll"
#r "../packages/Newtonsoft.Json/lib/net40/Newtonsoft.Json.dll"
#load "../packages/FSharp.Azure.StorageTypeProvider/StorageTypeProvider.fsx"
#load "config.fs" "common/serializer.fs" "pivot.fs" "storage.fs" "listing.fs" 
#else
module Gallery.App
#endif
open System
open System.IO

open Suave
open Suave.Filters
open Suave.Writers
open Suave.Operators
open FSharp.Data
open Gallery.CsvService

// --------------------------------------------------------------------------------------
// Server with /upload endpoint and /csv for Pivot type provider
// --------------------------------------------------------------------------------------

#if INTERACTIVE
let root = "https://localhost:8897"
#else
let root = "https://gallery-csv-service.azurewebsites.net"
#endif

let app =
  choose [
    GET >=> path "/" >=> Successful.OK "Service is running..." 
    POST >=> path "/update" >=> Storage.updateRecord
    POST >=> path "/upload" >=> Storage.uploadFile
    GET >=> path "/tags" >=> request (fun _ ctx -> async {
        let! files = Storage.getRecords ()
        let tags = 
          files 
          |> Seq.collect (fun f -> f.tags) 
          |> Seq.distinct
          |> Seq.map (fun t -> JsonValue.Array [| JsonValue.String (Listing.getTagId t); JsonValue.String t |])
          |> Array.ofSeq
          |> JsonValue.Array
        return! Successful.OK (tags.ToString()) ctx })
    setHeader  "Access-Control-Allow-Origin" "*"
    >=> setHeader "Access-Control-Allow-Headers" "content-type"
    >=> choose [
      OPTIONS >=> Successful.OK "CORS approved"
      GET >=> pathScan "/providers/listing%s" (fun _ ctx -> async { 
        let! files = Storage.getRecords ()
        return! Listing.handleRequest root files ctx })
      GET >=> pathScan "/providers/csv/%s" (fun source ctx -> async {
        let! file = Storage.fetchFile source
        match file with 
        | None -> return! RequestErrors.BAD_REQUEST (sprintf "File with id '%s' does not exist!" source) ctx
        | Some (meta, data) -> return! Pivot.handleRequest meta data (List.map fst ctx.request.query) ctx }) ]
  ]

// When port was specified, we start the app (in Azure), 
// otherwise we do nothing (it is hosted by 'build.fsx')
match System.Environment.GetCommandLineArgs() |> Seq.tryPick (fun s ->
    if s.StartsWith("port=") then Some(int(s.Substring("port=".Length)))
    else None ) with
| Some port ->
    let serverConfig =
      { Web.defaultConfig with
          maxContentLength = 1024 * 1024 * 50
          bindings = [ HttpBinding.createSimple HTTP "127.0.0.1" port ] }
    Web.startWebServer serverConfig app
| _ -> ()




{'fraction_non_alphanumeric': 0.1961391367692588, 'fraction_numerical': 0.00947004188672373, 'mean_word_length': 2.2768496420047732, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 86, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1940085', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1611, 'n_words': 435}