<commit_before>let () = Printexc.record_backtrace true

open Lwt

type result =
  | Ok
  | SyndicError of (Xmlm.pos * string)
  | W3CError of (Xmlm.pos * string) list
  | AnotherError of exn

exception Is_not_a_file

type src = [`Data of string | `Uri of Uri.t]
type fmt = [`Atom | `Rss1 | `Rss2 | `Opml1]

let curl_setup_simple h =
  let open Curl in
  set_useragent h "Syndic" ;
  set_nosignal h true ;
  set_connecttimeout h 5 ;
  set_timeout h 10 ;
  set_followlocation h true ;
  set_maxredirs h 10 ;
  set_ipresolve h IPRESOLVE_V4 ;
  set_encoding h CURL_ENCODING_ANY

let download h =
  let b = Buffer.create 16 in
  Curl.set_writefunction h (fun s -> Buffer.add_string b s ; String.length s) ;
  Lwt.bind (Curl_lwt.perform h) (fun code ->
      Lwt.return (code, Buffer.contents b) )

let get url =
  let open Lwt.Infix in
  let h = Curl.init () in
  Curl.set_url h (Uri.to_string url) ;
  curl_setup_simple h ;
  Lwt.try_bind
    (fun () -> download h)
    (fun (_code, contents) -> Lwt.return (`String (0, contents)))
    (fun exn -> Lwt.fail exn)
  >>= fun ret -> Curl.cleanup h ; Lwt.return ret

let get : src -> Xmlm.source Lwt.t = function
  | `Uri src -> get src
  | `Data data -> Lwt.return (`String (0, data))

let parse ?xmlbase = function
  | `Rss1 -> fun src -> `Rss1 (Syndic.Rss1.parse ?xmlbase src)
  | `Rss2 -> fun src -> `Rss2 (Syndic.Rss2.parse ?xmlbase src)
  | `Atom -> fun src -> `Atom (Syndic.Atom.parse ?xmlbase src)
  | `Opml1 -> fun src -> `Opml1 (Syndic.Opml1.parse ?xmlbase src)

let string_of_src = function
  | `Uri uri -> Printf.sprintf "'%s'" (Uri.to_string uri)
  | `Data data ->
      let buffer = Buffer.create 16 in
      Buffer.add_string buffer (String.sub data 0 16) ;
      Buffer.add_string buffer "…" ;
      Buffer.contents buffer

let string_of_fmt = function
  | `Rss1 -> "RSS 1.0"
  | `Rss2 -> "RSS 2.0"
  | `Atom -> "Atom"
  | `Opml1 -> "OPML 1.0"

let tests : ([> src] * [< fmt] * result) list =
  [ (`Uri (Uri.of_string "https://16andcounting.libsyn.com/rss"), `Rss2, Ok)
  ; (`Uri (Uri.of_string "http://ocaml.org/feed.xml"), `Atom, Ok)
  ; (`Uri (Uri.of_string "http://korben.info/feed"), `Rss2, Ok)
  ; (`Uri (Uri.of_string "http://linuxfr.org/journaux.atom"), `Atom, Ok)
  ; (`Uri (Uri.of_string "https://www.reddit.com/r/ocaml/.rss"), `Atom, Ok)
  ; ( `Data
        "<?xml version='1.0' encoding='utf-8'?> <feed \
         xmlns='http://www.w3.org/2015/Atom'> <title></title></feed>"
    , `Atom
    , W3CError [] ) ]

let () =
  Printexc.register_printer (function
    | Syndic.Rss1.Error.Error _ as exn ->
        Some (Syndic.Rss1.Error.to_string exn)
    | Syndic.Rss2.Error.Error _ as exn ->
        Some (Syndic.Rss2.Error.to_string exn)
    | Syndic.Atom.Error.Error _ as exn ->
        Some (Syndic.Atom.Error.to_string exn)
    | Syndic.Opml1.Error.Error _ as exn ->
        Some (Syndic.Opml1.Error.to_string exn)
    | _ -> None )

module Printf = struct<commit_msg>Don't use https?<commit_after>let () = Printexc.record_backtrace true

open Lwt

type result =
  | Ok
  | SyndicError of (Xmlm.pos * string)
  | W3CError of (Xmlm.pos * string) list
  | AnotherError of exn

exception Is_not_a_file

type src = [`Data of string | `Uri of Uri.t]
type fmt = [`Atom | `Rss1 | `Rss2 | `Opml1]

let curl_setup_simple h =
  let open Curl in
  set_useragent h "Syndic" ;
  set_nosignal h true ;
  set_connecttimeout h 5 ;
  set_timeout h 10 ;
  set_followlocation h true ;
  set_maxredirs h 10 ;
  set_ipresolve h IPRESOLVE_V4 ;
  set_encoding h CURL_ENCODING_ANY

let download h =
  let b = Buffer.create 16 in
  Curl.set_writefunction h (fun s -> Buffer.add_string b s ; String.length s) ;
  Lwt.bind (Curl_lwt.perform h) (fun code ->
      Lwt.return (code, Buffer.contents b) )

let get url =
  let open Lwt.Infix in
  let h = Curl.init () in
  Curl.set_url h (Uri.to_string url) ;
  curl_setup_simple h ;
  Lwt.try_bind
    (fun () -> download h)
    (fun (_code, contents) -> Lwt.return (`String (0, contents)))
    (fun exn -> Lwt.fail exn)
  >>= fun ret -> Curl.cleanup h ; Lwt.return ret

let get : src -> Xmlm.source Lwt.t = function
  | `Uri src -> get src
  | `Data data -> Lwt.return (`String (0, data))

let parse ?xmlbase = function
  | `Rss1 -> fun src -> `Rss1 (Syndic.Rss1.parse ?xmlbase src)
  | `Rss2 -> fun src -> `Rss2 (Syndic.Rss2.parse ?xmlbase src)
  | `Atom -> fun src -> `Atom (Syndic.Atom.parse ?xmlbase src)
  | `Opml1 -> fun src -> `Opml1 (Syndic.Opml1.parse ?xmlbase src)

let string_of_src = function
  | `Uri uri -> Printf.sprintf "'%s'" (Uri.to_string uri)
  | `Data data ->
      let buffer = Buffer.create 16 in
      Buffer.add_string buffer (String.sub data 0 16) ;
      Buffer.add_string buffer "…" ;
      Buffer.contents buffer

let string_of_fmt = function
  | `Rss1 -> "RSS 1.0"
  | `Rss2 -> "RSS 2.0"
  | `Atom -> "Atom"
  | `Opml1 -> "OPML 1.0"

let tests : ([> src] * [< fmt] * result) list =
  [ (`Uri (Uri.of_string "http://16andcounting.libsyn.com/rss"), `Rss2, Ok)
  ; (`Uri (Uri.of_string "http://ocaml.org/feed.xml"), `Atom, Ok)
  ; (`Uri (Uri.of_string "http://korben.info/feed"), `Rss2, Ok)
  ; (`Uri (Uri.of_string "http://linuxfr.org/journaux.atom"), `Atom, Ok)
  ; (`Uri (Uri.of_string "http://www.reddit.com/r/ocaml/.rss"), `Atom, Ok)
  ; ( `Data
        "<?xml version='1.0' encoding='utf-8'?> <feed \
         xmlns='http://www.w3.org/2015/Atom'> <title></title></feed>"
    , `Atom
    , W3CError [] ) ]

let () =
  Printexc.register_printer (function
    | Syndic.Rss1.Error.Error _ as exn ->
        Some (Syndic.Rss1.Error.to_string exn)
    | Syndic.Rss2.Error.Error _ as exn ->
        Some (Syndic.Rss2.Error.to_string exn)
    | Syndic.Atom.Error.Error _ as exn ->
        Some (Syndic.Atom.Error.to_string exn)
    | Syndic.Opml1.Error.Error _ as exn ->
        Some (Syndic.Opml1.Error.to_string exn)
    | _ -> None )

module Printf = struct
{'fraction_non_alphanumeric': 0.1538723404255319, 'fraction_numerical': 0.019063829787234043, 'mean_word_length': 2.8581746552856204, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 2, '>': 75, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '449875', 'n_tokens_mistral': 2430, 'n_tokens_neox': 2360, 'n_words': 583}