<commit_before>open Lwt.Infix

let server_src = Logs.Src.create "server" ~doc:"HTTP server"
module Server_log = (val Logs.src_log server_src : Logs.LOG)

module Main (FS:V1_LWT.KV_RO) (S:Cohttp_lwt.Server) = struct

  let read_fs fs name =
    FS.size fs name >>= function
    | `Error (FS.Unknown_key _) -> Lwt.fail (Failure ("read " ^ name))
    | `Ok size ->
      FS.read fs name 0 (Int64.to_int size) >>= function
      | `Error (FS.Unknown_key _) -> Lwt.fail (Failure ("read " ^ name))
      | `Ok bufs -> Lwt.return (Cstruct.copyv bufs)

  let rec dispatcher fs uri =
    match Uri.path uri with
    | ("" | "/") as path ->
      Server_log.info (fun f -> f "request for '%s'" path);
      dispatcher fs (Uri.with_path uri "index.html")
    | path ->
      Server_log.info (fun f -> f "request for '%s'" path);
      Lwt.catch (fun () ->
          read_fs fs path >>= fun body ->
          let mime_type = Magic_mime.lookup path in
          let headers = Cohttp.Header.init () in
          let headers = Cohttp.Header.add headers "content-type" mime_type in
          S.respond_string ~status:`OK ~body ~headers ()
        ) (fun _exn -> S.respond_not_found ())

  let start fs http =
    Logs.(set_level (Some Info));

    let callback (_, cid) request _body =
      let uri = Cohttp.Request.uri request in
      let cid = Cohttp.Connection.to_string cid in
      Server_log.info (fun f -> f "[%s] serving %s" cid (Uri.to_string uri));
      dispatcher fs uri
    in
    let conn_closed (_, cid) =
      let cid = Cohttp.Connection.to_string cid in
      Server_log.info (fun f -> f "[%s] closing" cid);
    in
    let port = Key_gen.port () in
    Server_log.info (fun f -> f "listening on %d/TCP" port);
    http (`TCP port) (S.make ~conn_closed ~callback ())

end
<commit_msg>don't override log level set at build or config time<commit_after>open Lwt.Infix

let server_src = Logs.Src.create "server" ~doc:"HTTP server"
module Server_log = (val Logs.src_log server_src : Logs.LOG)

module Main (FS:V1_LWT.KV_RO) (S:Cohttp_lwt.Server) = struct

  let read_fs fs name =
    FS.size fs name >>= function
    | `Error (FS.Unknown_key _) -> Lwt.fail (Failure ("read " ^ name))
    | `Ok size ->
      FS.read fs name 0 (Int64.to_int size) >>= function
      | `Error (FS.Unknown_key _) -> Lwt.fail (Failure ("read " ^ name))
      | `Ok bufs -> Lwt.return (Cstruct.copyv bufs)

  let rec dispatcher fs uri =
    match Uri.path uri with
    | ("" | "/") as path ->
      Server_log.info (fun f -> f "request for '%s'" path);
      dispatcher fs (Uri.with_path uri "index.html")
    | path ->
      Server_log.info (fun f -> f "request for '%s'" path);
      Lwt.catch (fun () ->
          read_fs fs path >>= fun body ->
          let mime_type = Magic_mime.lookup path in
          let headers = Cohttp.Header.init () in
          let headers = Cohttp.Header.add headers "content-type" mime_type in
          S.respond_string ~status:`OK ~body ~headers ()
        ) (fun _exn -> S.respond_not_found ())

  let start fs http =
    let callback (_, cid) request _body =
      let uri = Cohttp.Request.uri request in
      let cid = Cohttp.Connection.to_string cid in
      Server_log.info (fun f -> f "[%s] serving %s" cid (Uri.to_string uri));
      dispatcher fs uri
    in
    let conn_closed (_, cid) =
      let cid = Cohttp.Connection.to_string cid in
      Server_log.info (fun f -> f "[%s] closing" cid);
    in
    let port = Key_gen.port () in
    Server_log.info (fun f -> f "listening on %d/TCP" port);
    http (`TCP port) (S.make ~conn_closed ~callback ())

end

{'fraction_non_alphanumeric': 0.13268156424581007, 'fraction_numerical': 0.0022346368715083797, 'mean_word_length': 2.521140609636185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595854', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1232, 'n_words': 380}