<commit_before>let addrlen = Big.of_int 64
let blockbytes = 4096
let blockbits = Big.of_int (blockbytes*8)
let disk_fd = ref Unix.stdout   (* just some Unix.file_descr object *)
let debug = true

type disk_state = { disk_fd : Unix.file_descr ref }

let addr_to_int a = Big.to_int (Word.wordToNat addrlen a)

let word_to_block (Word.W w) =
  Printf.printf "word_to_block %s\n" (Z.to_string w);
  let s = Z.to_bits w in
  let nbytes = String.length s in
  let res = String.concat "" [ s ; String.make (blockbytes-nbytes) (Char.chr 0) ] in
  res

let block_to_word b =
  let z = Z.of_bits b in
  Printf.printf "block_to_word %s\n" (Z.to_string z);
  Word.W z

let init_disk fn =
  let fd = Unix.openfile fn [ Unix.O_RDWR ; Unix.O_CREAT ] 0o666 in
  { disk_fd = ref fd }

let close_disk { disk_fd = fd } =
  Unix.close !fd

let read_disk { disk_fd = fd } b =
  let s = String.create blockbytes in
  let cc = ExtUnix.All.pread !fd (b * blockbytes) s 0 blockbytes in
  if cc != blockbytes then raise (Failure "read_disk");
  Printf.printf "read_disk: string of length %d\n" (String.length s);
  block_to_word s

let write_disk { disk_fd = fd } b v =
  let s = word_to_block v in
  Printf.printf "write_disk: string of length %d\n" (String.length s);
  let cc = ExtUnix.All.pwrite !fd (b * blockbytes) s 0 blockbytes in
  if cc != blockbytes then raise (Failure "write_disk")

let sync_disk { disk_fd = fd } b =
  ExtUnix.All.fsync !fd

let set_size_disk { disk_fd = fd } b =
  Unix.ftruncate !fd (b*blockbytes)

let rec run_dcode ds = function<commit_msg>kill some ocaml debug statements<commit_after>let addrlen = Big.of_int 64
let blockbytes = 4096
let blockbits = Big.of_int (blockbytes*8)
let disk_fd = ref Unix.stdout   (* just some Unix.file_descr object *)
let debug = true

type disk_state = { disk_fd : Unix.file_descr ref }

let addr_to_int a = Big.to_int (Word.wordToNat addrlen a)

let word_to_block (Word.W w) =
  let s = Z.to_bits w in
  let nbytes = String.length s in
  let res = String.concat "" [ s ; String.make (blockbytes-nbytes) (Char.chr 0) ] in
  res

let block_to_word b =
  let z = Z.of_bits b in
  Word.W z

let init_disk fn =
  let fd = Unix.openfile fn [ Unix.O_RDWR ; Unix.O_CREAT ] 0o666 in
  { disk_fd = ref fd }

let close_disk { disk_fd = fd } =
  Unix.close !fd

let read_disk { disk_fd = fd } b =
  let s = String.create blockbytes in
  let cc = ExtUnix.All.pread !fd (b * blockbytes) s 0 blockbytes in
  if cc != blockbytes then raise (Failure "read_disk");
  block_to_word s

let write_disk { disk_fd = fd } b v =
  let s = word_to_block v in
  let cc = ExtUnix.All.pwrite !fd (b * blockbytes) s 0 blockbytes in
  if cc != blockbytes then raise (Failure "write_disk")

let sync_disk { disk_fd = fd } b =
  ExtUnix.All.fsync !fd

let set_size_disk { disk_fd = fd } b =
  Unix.ftruncate !fd (b*blockbytes)

let rec run_dcode ds = function
{'fraction_non_alphanumeric': 0.10181311018131102, 'fraction_numerical': 0.009762900976290097, 'mean_word_length': 3.4070660522273424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3471411', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1102, 'n_words': 359}