<commit_before>open Biocaml_internal_pervasives

type 'a t = [
| `track of (string * string) list
| `comment of string
| `browser of
    [ `position of string * int * int | `hide of [`all] | `unknown of string ]
| `content of 'a
]

type parse_error =
[ `incomplete_input of Biocaml_pos.t * string list * string option
| `wrong_browser_position of Biocaml_pos.t * string
| `wrong_key_value_format of (string * string) list * string * string ]

open Result

(*
  browser position chr19:49304200-49310700
  browser hide all
*)
let parse_chormpos position s =
  try begin match String.rindex s ':' with
  | Some colon ->
    let name = String.slice s 0 colon in
    begin match String.rindex s '-' with
    | Some dash ->
      let start = String.slice s (colon + 1) dash |! Int.of_string in
      let stop =  String.slice s (dash + 1) (String.length s) |! Int.of_string in
      return (`browser (`position (name, start, stop)))
    | None -> failwith "A"
    end
  | None -> failwith "B"
  end
  with
    e -> fail (`wrong_browser_position (position, s))

let parse_browser position line =
  let tokens =
    String.chop_prefix ~prefix:"browser " line
    |! Option.value ~default:""
    |! String.split_on_chars ~on:[' '; '\t'; '\r']
    |! List.filter ~f:((<>) "") in
  begin match tokens with
  | "position" :: pos :: [] -> parse_chormpos position pos
  | "hide" :: "all" :: [] -> return (`browser (`hide `all))
  | any -> return (`browser (`unknown line))
  end

let parse_track position stripped =
  let rec loop s acc =
    match Parse.escapable_string s ~stop_before:['='] with
    | (tag, Some '=', rest) ->
      begin match Parse.escapable_string rest ~stop_before:[' '; '\t'] with
      | (value, _, rest) ->
        let str = String.strip rest in
        if str = "" then return ((tag, value) :: acc)
        else loop str ((tag, value) :: acc)
      end
    | (str, _, rest) -> fail (`wrong_key_value_format (List.rev acc, str, rest))
  in
  loop stripped []
  >>= fun kv ->
  return (`track (List.rev kv))

let rec next p =
  let open Biocaml_transform.Line_oriented in
  let output_result = function
    | Ok o -> `output o
    | Error e -> `error e in
  match next_line p with
  | None -> `not_ready
  | Some "" -> `not_ready
  | Some l when String.(is_prefix (strip l) ~prefix:"#") ->
    `output (`comment String.(sub l ~pos:1 ~len:(length l - 1)))<commit_msg>Implement two corner cases in Biocaml_track<commit_after>open Biocaml_internal_pervasives

type 'a t = [
| `track of (string * string) list
| `comment of string
| `browser of
    [ `position of string * int * int | `hide of [`all] | `unknown of string ]
| `content of 'a
]

type parse_error =
[ `incomplete_input of Biocaml_pos.t * string list * string option
| `wrong_browser_position of Biocaml_pos.t * string
| `wrong_key_value_format of (string * string) list * string * string ]

open Result

(*
  browser position chr19:49304200-49310700
  browser hide all
*)
let parse_chormpos position s =
  try begin match String.rindex s ':' with
  | Some colon ->
    let name = String.slice s 0 colon in
    begin match String.rindex s '-' with
    | Some dash ->
      let start = String.slice s (colon + 1) dash |! Int.of_string in
      let stop =  String.slice s (dash + 1) (String.length s) |! Int.of_string in
      return (`browser (`position (name, start, stop)))
    | None -> failwith "A"
    end
  | None -> failwith "B"
  end
  with
    e -> fail (`wrong_browser_position (position, s))

let parse_browser position line =
  let tokens =
    String.chop_prefix ~prefix:"browser " line
    |! Option.value ~default:""
    |! String.split_on_chars ~on:[' '; '\t'; '\r']
    |! List.filter ~f:((<>) "") in
  begin match tokens with
  | "position" :: pos :: [] -> parse_chormpos position pos
  | "hide" :: "all" :: [] -> return (`browser (`hide `all))
  | any -> return (`browser (`unknown line))
  end

let parse_track position stripped =
  let rec loop s acc =
    match Parse.escapable_string s ~stop_before:['='] with
    | (tag, Some '=', rest) ->
      begin match Parse.escapable_string rest ~stop_before:[' '; '\t'] with
      | (value, _, rest) ->
        let str = String.strip rest in
        if str = "" then return ((tag, value) :: acc)
        else loop str ((tag, value) :: acc)
      end
    | (str, _, rest) -> fail (`wrong_key_value_format (List.rev acc, str, rest))
  in
  loop stripped []
  >>= fun kv ->
  return (`track (List.rev kv))

let rec next p =
  let open Biocaml_transform.Line_oriented in
  let output_result = function
    | Ok o -> `output o
    | Error e -> `error e in
  match next_line p with
  | None -> `not_ready
  | Some "" -> `not_ready
  | Some l when String.(is_prefix (strip l) ~prefix:"#") ->
    `output (`comment String.(sub l ~pos:1 ~len:(length l - 1)))
  | Some l when String.strip l = "track"-> `output (`track [])
  | Some l when String.strip l = "browser" -> `output (`browser (`unknown l))
{'fraction_non_alphanumeric': 0.14198659354052406, 'fraction_numerical': 0.009343895998374974, 'mean_word_length': 3.0163132137030995, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '85779', 'n_tokens_mistral': 1683, 'n_tokens_neox': 1616, 'n_words': 554}