<commit_before>{
(**********************************************************************
 *                                                                    *
 *                            OCamlFormat                             *
 *                                                                    *
 *  Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.  *
 *                                                                    *
 *  This source code is licensed under the MIT license found in the   *
 *  LICENSE file in the root directory of this source tree.           *
 *                                                                    *
 **********************************************************************)

open Caml

(* To buffer string literals *)

let string_buffer = Buffer.create 256
let reset_string_buffer () = Buffer.reset string_buffer
let get_stored_string () = Buffer.contents string_buffer

let store_string_char c = Buffer.add_char string_buffer c
let store_string s = Buffer.add_string string_buffer s

}

let newline = ('\013'* '\010')
let hex_digit =
  ['0'-'9' 'A'-'F' 'a'-'f']

rule string mode = parse
  | '\"'
      { reset_string_buffer ();
        string_aux mode lexbuf;
        get_stored_string () }
  | _ { user_error "not a string literal" [] }


and string_aux mode = parse
    '\"'
      { () }
  | '\\' newline ([' ' '\t'] *)
      { string_aux mode lexbuf }
  | '\\' ['\\' '\'' '\"' 't' 'b' 'r' ' ']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | "\\n"
      { begin match mode with
          | `Normalize_nl -> store_string "\n"
          | `Preserve -> store_string "\\n" end;
        string_aux mode lexbuf }
  | '\\' ['0'-'9'] ['0'-'9'] ['0'-'9']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'o' ['0'-'3'] ['0'-'7'] ['0'-'7']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'x' ['0'-'9' 'a'-'f' 'A'-'F'] ['0'-'9' 'a'-'f' 'A'-'F']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'u' '{' hex_digit+ '}'
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' _
      { (*  Should be an error, but we are very lax.
            raise (Error (Illegal_escape (Lexing.lexeme lexbuf),
            Location.curr lexbuf))
        *)
        string_aux mode lexbuf
      }
  | newline
      { store_string (Lexing.lexeme lexbuf);<commit_msg>Fix: string litteral<commit_after>{
(**********************************************************************
 *                                                                    *
 *                            OCamlFormat                             *
 *                                                                    *
 *  Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.  *
 *                                                                    *
 *  This source code is licensed under the MIT license found in the   *
 *  LICENSE file in the root directory of this source tree.           *
 *                                                                    *
 **********************************************************************)

open Caml

(* To buffer string literals *)

let string_buffer = Buffer.create 256
let reset_string_buffer () = Buffer.reset string_buffer
let get_stored_string () = Buffer.contents string_buffer

let store_string_char c = Buffer.add_char string_buffer c
let store_string s = Buffer.add_string string_buffer s

}

let newline = ('\013'* '\010')
let hex_digit =
  ['0'-'9' 'A'-'F' 'a'-'f']

rule string mode = parse
  | '\"'
      { reset_string_buffer ();
        string_aux mode lexbuf;
        get_stored_string () }
  | _ { user_error "not a string literal" [] }


and string_aux mode = parse
    '\"'
      { () }
  | '\\' newline ([' ' '\t'] *)
      { string_aux mode lexbuf }
  | '\\' ['\\' '\'' '\"' 't' 'b' 'r' ' ']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | "\\n"
      { begin match mode with
          | `Normalize_nl -> store_string "\n"
          | `Preserve -> store_string "\\n" end;
        string_aux mode lexbuf }
  | '\\' ['0'-'9'] ['0'-'9'] ['0'-'9']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'o' ['0'-'3'] ['0'-'7'] ['0'-'7']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'x' ['0'-'9' 'a'-'f' 'A'-'F'] ['0'-'9' 'a'-'f' 'A'-'F']
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' 'u' '{' hex_digit+ '}'
      { store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf }
  | '\\' _
      { (*  Should be an error, but we are very lax.
            raise (Error (Illegal_escape (Lexing.lexeme lexbuf),
            Location.curr lexbuf))
        *)
        store_string (Lexing.lexeme lexbuf);
        string_aux mode lexbuf
      }
  | newline
      { store_string (Lexing.lexeme lexbuf);
{'fraction_non_alphanumeric': 0.19270625747309686, 'fraction_numerical': 0.012355520127540853, 'mean_word_length': 1.6969371305749597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319176', 'n_tokens_mistral': 1593, 'n_tokens_neox': 1468, 'n_words': 366}