<commit_before>{

type token = LOC of string * (int * int) * (int * int)
           | CHR of char
           | EOI

let mkLOC fn (bline, bchar) (eline, echar) =
  let bline = int_of_string bline in
  let bchar = int_of_string bchar in
  let eline = int_of_string eline in
  let echar = int_of_string echar in
  LOC(fn, (bline, bchar), (eline, echar))
}

let int = ['0'-'9']+

rule loc = parse
  | "File \"" ([^'"']* as file_name) "\", line " (int as line) ", characters "
    (int as start) '-' int " (end at line " (int as stop_line)
    ", character " (int as stop) ')' [^'\n']* '\n'

    { mkLOC file_name (line, start) (stop_line, stop) }

  | "File \"" ([^'"']* as file_name) "\", line " (int as line) ", characters "
    (int as start) '-' (int as stop) ([^'0'-'9' '\n'] [^'\n']*)? '\n'

    { mkLOC file_name (line, start) (line, stop) }

  | eof { EOI }

  | _ as c { CHR c }

and line = parse
  | [^ '\n']* '\n'? as ln { Some ln }
  | eof { None }<commit_msg>substloc: proper handling of \r in extraction<commit_after>{

type token = LOC of string * (int * int) * (int * int)
           | CHR of char
           | EOI

let mkLOC fn (bline, bchar) (eline, echar) =
  let bline = int_of_string bline in
  let bchar = int_of_string bchar in
  let eline = int_of_string eline in
  let echar = int_of_string echar in
  LOC(fn, (bline, bchar), (eline, echar))
}

let int = ['0'-'9']+

rule loc = parse
  | "File \"" ([^'"']* as file_name) "\", line " (int as line) ", characters "
    (int as start) '-' int " (end at line " (int as stop_line)
    ", character " (int as stop) ')' [^'\n']* '\n'

    { mkLOC file_name (line, start) (stop_line, stop) }

  | "File \"" ([^'"']* as file_name) "\", line " (int as line) ", characters "
    (int as start) '-' (int as stop) ([^'0'-'9' '\n'] [^'\n']*)? '\n'

    { mkLOC file_name (line, start) (line, stop) }

  | eof { EOI }

  | _ as c { CHR c }

and line = parse
  | [^ '\n' '\r']* ('\n'|'\r'|"\r\n")? as ln { Some ln }
  | eof { None }
{'fraction_non_alphanumeric': 0.20332159033719174, 'fraction_numerical': 0.004026170105686965, 'mean_word_length': 2.6477064220183486, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5853859', 'n_tokens_mistral': 828, 'n_tokens_neox': 791, 'n_words': 253}