<commit_before>#light (*
  exec fsi --exec $0 --quiet
*)

open System
open System.Collections.Generic
open System.IO
open System.Text
open System.Text.RegularExpressions

let group r = "(?:" + r + ")"
let many r = r + "*"
let choice rs = String.concat "|" rs |> group
let strings ss = Array.map Regex.Escape ss |> choice
let chars (s: string) = s.ToCharArray () |> Array.map string |> strings

let keywords =
  [|":>"; "->"; "="; 
    "[<"; ">]"; "[|"; "|]"; "?<-"; "??"; "?"; ":?>"; ":?"; ":>"; "::"; ":="; ":";
    "abstract";
    "do!"; "downto"; "do";
    "else";
    "false"; "finally"; "for"; "fun";
    "if"; "inline"; "in";
    "let!"; "let";
    "match"; "member"; "module";
    "namespace"; "new";
    "rec"; "return!"; "return";
    "static";
    "then"; "to"; "true"; "try"; "type";
    "upcast"; "use!"; "use";
    "val";
    "when"; "while"; "with";
    "yield!"; "yield";
    |]

let keywordSet = HashSet<_>(keywords)
let punctuation =
  [|","; "("; ")"; "{"; "}"; "["; "]"; "'"; "#"; ":"; ";;"; ";"|]
let nonIdSet = HashSet<_>(Array.concat [|keywords; punctuation; [|"."|]|])

let id = group ("[a-zA-Z_]" + many "[a-zA-Z0-9_']")
let num = group ("[0-9]" + many "[0-9.sxuylLfF]")
let symFirst = "!%&*+-./<=>@^|~"
let symRest = symFirst + "?"
let sym = group (chars symFirst + many (chars symRest))
let str = "\\\".*\\\""
let char = "\\'.*\\'"

let tokenRegex =
  let extra = Array.concat [|keywords; punctuation|] |> strings
  Regex (choice [|id; extra; sym; num; str; char|])

let tokenize (s: string) =
  let rec loop ts = function
   | [] -> []
   | [m: Match] -> List.rev ((m.Value, 0)::ts)
   | m::n::ms ->
     loop ((m.Value, n.Index - m.Index - m.Value.Length)::ts) (n::ms)
  tokenRegex.Matches s |> Seq.cast |> List.ofSeq |> loop []
<commit_msg>More careful handling of string literals.<commit_after>#light (*
  exec fsi --exec $0 --quiet
*)

open System
open System.Collections.Generic
open System.IO
open System.Text
open System.Text.RegularExpressions

let group r = "(?:" + r + ")"
let many r = r + "*"
let choice rs = String.concat "|" rs |> group
let strings ss = Array.map Regex.Escape ss |> choice
let chars (s: string) = s.ToCharArray () |> Array.map string |> strings

let keywords =
  [|":>"; "->"; "="; 
    "[<"; ">]"; "[|"; "|]"; "?<-"; "??"; "?"; ":?>"; ":?"; ":>"; "::"; ":="; ":";
    "abstract";
    "do!"; "downto"; "do";
    "else";
    "false"; "finally"; "for"; "fun";
    "if"; "inline"; "in";
    "let!"; "let";
    "match"; "member"; "module";
    "namespace"; "new";
    "rec"; "return!"; "return";
    "static";
    "then"; "to"; "true"; "try"; "type";
    "upcast"; "use!"; "use";
    "val";
    "when"; "while"; "with";
    "yield!"; "yield";
    |]

let keywordSet = HashSet<_>(keywords)
let punctuation =
  [|","; "("; ")"; "{"; "}"; "["; "]"; "'"; "#"; ":"; ";;"; ";"|]
let nonIdSet = HashSet<_>(Array.concat [|keywords; punctuation; [|"."|]|])

let id = group ("[a-zA-Z_]" + many "[a-zA-Z0-9_']")
let num = group ("[0-9]" + many "[0-9.sxuylLfF]")
let symFirst = "!%&*+-./<=>@^|~"
let symRest = symFirst + "?"
let sym = group (chars symFirst + many (chars symRest))
let str = "\\\"(?:[^\\\\\"]|\\\\.)*\\\""
let char = "\\'.*\\'"

let tokenRegex =
  let extra = Array.concat [|keywords; punctuation|] |> strings
  Regex (choice [|id; extra; sym; num; str; char|])

let tokenize (s: string) =
  let rec loop ts = function
   | [] -> []
   | [m: Match] -> List.rev ((m.Value, 0)::ts)
   | m::n::ms ->
     loop ((m.Value, n.Index - m.Index - m.Value.Length)::ts) (n::ms)
  tokenRegex.Matches s |> Seq.cast |> List.ofSeq |> loop []

{'fraction_non_alphanumeric': 0.2876674107142857, 'fraction_numerical': 0.004464285714285714, 'mean_word_length': 3.2476303317535544, 'pattern_counts': {'":': 16, '<': 13, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '455156', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1405, 'n_words': 335}