<commit_before>(*
 * Copyright (C) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version,
 * with the special exception on linking described in file LICENSE.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *)

open Printf
open Lwt

module String = struct include String include BatString end
module S = Obs_server.Make(Obs_storage)(Obs_protocol_payload.Version_0_0_0)

let port = ref 12050
let db_dir = ref None
let debug = ref false
let gcommit_period = ref 0.010
let write_buffer_size = ref (4 * 1024 * 1024)
let block_size = ref 4096
let max_open_files = ref 1000
let master = ref None
let assume_page_fault = ref false
let unsafe_mode = ref false

let params =
  Arg.align
    [
      "-port", Arg.Set_int port, "PORT Port to listen at (default: 12050)";
      "-master", Arg.String (fun s -> master := Some s),
        "HOST:PORT Replicate database reachable on HOST:PORT.";
      "-debug", Arg.Set debug, " Dump debug info to stderr.";
      "-group-commit-period", Arg.Set_float gcommit_period,
        "DT Group commit period (default: 0.010s)";
      "-write-buffer-size", Arg.Set_int write_buffer_size, "N Write buffer size (default: 4MB)";
      "-block_size", Arg.Set_int block_size, "N Block size (default: 4KB)";
      "-max-open-files", Arg.Set_int max_open_files, "N Max open files (default: 1000)";
      "-assume-page-fault", Arg.Set assume_page_fault,
        " Assume working set doesn't fit in RAM and avoid blocking.";
      "-no-fsync", Arg.Set unsafe_mode,
        " Don't fsync after writes (may loss data on system crash).";
    ]

let usage_message = "Usage: obigstore [options] [database dir]"

let _ = Sys.set_signal Sys.sigpipe Sys.Signal_ignore
let _ = Sys.set_signal Sys.sigint (Sys.Signal_handle (fun _ -> exit 0))
let _ = Sys.set_signal Sys.sighup (Sys.Signal_handle (fun _ -> Gc.compact ()))
<commit_msg>obigstore: rename -group-commit-period to -commit-period.<commit_after>(*
 * Copyright (C) 2011 <NAME> <<EMAIL>>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version,
 * with the special exception on linking described in file LICENSE.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *)

open Printf
open Lwt

module String = struct include String include BatString end
module S = Obs_server.Make(Obs_storage)(Obs_protocol_payload.Version_0_0_0)

let port = ref 12050
let db_dir = ref None
let debug = ref false
let gcommit_period = ref 0.010
let write_buffer_size = ref (4 * 1024 * 1024)
let block_size = ref 4096
let max_open_files = ref 1000
let master = ref None
let assume_page_fault = ref false
let unsafe_mode = ref false

let params =
  Arg.align
    [
      "-port", Arg.Set_int port, "PORT Port to listen at (default: 12050)";
      "-master", Arg.String (fun s -> master := Some s),
        "HOST:PORT Replicate database reachable on HOST:PORT.";
      "-debug", Arg.Set debug, " Dump debug info to stderr.";
      "-commit-period", Arg.Set_float gcommit_period,
        "DT Group commit period (default: 0.010s)";
      "-write-buffer-size", Arg.Set_int write_buffer_size, "N Write buffer size (default: 4MB)";
      "-block_size", Arg.Set_int block_size, "N Block size (default: 4KB)";
      "-max-open-files", Arg.Set_int max_open_files, "N Max open files (default: 1000)";
      "-assume-page-fault", Arg.Set assume_page_fault,
        " Assume working set doesn't fit in RAM and avoid blocking.";
      "-no-fsync", Arg.Set unsafe_mode,
        " Don't fsync after writes (may loss data on system crash).";
    ]

let usage_message = "Usage: obigstore [options] [database dir]"

let _ = Sys.set_signal Sys.sigpipe Sys.Signal_ignore
let _ = Sys.set_signal Sys.sigint (Sys.Signal_handle (fun _ -> exit 0))
let _ = Sys.set_signal Sys.sighup (Sys.Signal_handle (fun _ -> Gc.compact ()))

{'fraction_non_alphanumeric': 0.09535160905840286, 'fraction_numerical': 0.025824394119984106, 'mean_word_length': 3.6448339483394836, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3778649', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1479, 'n_words': 620}