<commit_before>(*
 * Copyright (C) 2006-2009 Citrix Systems Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)
open Pervasiveext

exception Unix_error of int

external _exit : int -> unit = "unix_exit"

(** remove a file, but doesn't raise an exception if the file is already removed *)
let unlink_safe file =
	try Unix.unlink file with (* Unix.Unix_error (Unix.ENOENT, _ , _)*) _ -> ()

(** create a directory but doesn't raise an exception if the directory already exist *)
let mkdir_safe dir perm =
	try Unix.mkdir dir perm with Unix.Unix_error (Unix.EEXIST, _, _) -> ()

(** create a directory, and create parent if doesn't exist *)
let mkdir_rec dir perm =
	let rec p_mkdir dir =
		let p_name = Filename.dirname dir in
		if p_name <> "/" && p_name <> "." 
		then p_mkdir p_name;
		mkdir_safe dir perm in
	p_mkdir dir

(** write a pidfile file *)
let pidfile_write filename =
	let fd = Unix.openfile filename
	                       [ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; ]
			       0o640 in
	finally
	(fun () ->
		let pid = Unix.getpid () in
		let buf = string_of_int pid ^ "\n" in
		let len = String.length buf in
		if Unix.write fd buf 0 len <> len 
		then failwith "pidfile_write failed";
	)
	(fun () -> Unix.close fd)

(** read a pidfile file, return either Some pid or None *)
let pidfile_read filename =
	let fd = Unix.openfile filename [ Unix.O_RDONLY ] 0o640 in
	finally
	(fun () ->
		try
			let buf = String.create 80  in
			let rd = Unix.read fd buf 0 (String.length buf) in
			if rd = 0 then
				failwith "pidfile_read failed";
			Scanf.sscanf (String.sub buf 0 rd) "%d" (fun i -> Some i)
		with exn -> None)
	(fun () -> Unix.close fd)

(** daemonize a process *)
(* !! Must call this before spawning any threads !! *)
let daemonize () =
	match Unix.fork () with
	| 0 ->
		if Unix.setsid () == -1 then
			failwith "Unix.setsid failed";

		begin match Unix.fork () with
		| 0 ->
			let nullfd = Unix.openfile "/dev/null" [ Unix.O_WRONLY ] 0 in
			begin try
				Unix.close Unix.stdin;
				Unix.dup2 nullfd Unix.stdout;
				Unix.dup2 nullfd Unix.stderr;
			with exn -> Unix.close nullfd; raise exn
			end;
			Unix.close nullfd
		| _ -> exit 0
		end
	| _ -> exit 0

exception Break

let lines_fold f start input =
	let rec fold accumulator =
		let line =
			try Some (input_line input)
			with End_of_file -> None in
		match line with
			| Some line -> (try fold (f accumulator line) with Break -> accumulator)
			| None -> accumulator in
	fold start

let lines_iter f = lines_fold (fun () line -> ignore(f line)) ()

(** open a file, and make sure the close is always done *)
let with_input_channel file f =
	let input = open_in file in
	finally<commit_msg>XOP-113: Modify Unixext.fold_lines to avoid stack overflow.<commit_after>(*
 * Copyright (C) 2006-2009 Citrix Systems Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)
open Pervasiveext

exception Unix_error of int

external _exit : int -> unit = "unix_exit"

(** remove a file, but doesn't raise an exception if the file is already removed *)
let unlink_safe file =
	try Unix.unlink file with (* Unix.Unix_error (Unix.ENOENT, _ , _)*) _ -> ()

(** create a directory but doesn't raise an exception if the directory already exist *)
let mkdir_safe dir perm =
	try Unix.mkdir dir perm with Unix.Unix_error (Unix.EEXIST, _, _) -> ()

(** create a directory, and create parent if doesn't exist *)
let mkdir_rec dir perm =
	let rec p_mkdir dir =
		let p_name = Filename.dirname dir in
		if p_name <> "/" && p_name <> "." 
		then p_mkdir p_name;
		mkdir_safe dir perm in
	p_mkdir dir

(** write a pidfile file *)
let pidfile_write filename =
	let fd = Unix.openfile filename
	                       [ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; ]
			       0o640 in
	finally
	(fun () ->
		let pid = Unix.getpid () in
		let buf = string_of_int pid ^ "\n" in
		let len = String.length buf in
		if Unix.write fd buf 0 len <> len 
		then failwith "pidfile_write failed";
	)
	(fun () -> Unix.close fd)

(** read a pidfile file, return either Some pid or None *)
let pidfile_read filename =
	let fd = Unix.openfile filename [ Unix.O_RDONLY ] 0o640 in
	finally
	(fun () ->
		try
			let buf = String.create 80  in
			let rd = Unix.read fd buf 0 (String.length buf) in
			if rd = 0 then
				failwith "pidfile_read failed";
			Scanf.sscanf (String.sub buf 0 rd) "%d" (fun i -> Some i)
		with exn -> None)
	(fun () -> Unix.close fd)

(** daemonize a process *)
(* !! Must call this before spawning any threads !! *)
let daemonize () =
	match Unix.fork () with
	| 0 ->
		if Unix.setsid () == -1 then
			failwith "Unix.setsid failed";

		begin match Unix.fork () with
		| 0 ->
			let nullfd = Unix.openfile "/dev/null" [ Unix.O_WRONLY ] 0 in
			begin try
				Unix.close Unix.stdin;
				Unix.dup2 nullfd Unix.stdout;
				Unix.dup2 nullfd Unix.stderr;
			with exn -> Unix.close nullfd; raise exn
			end;
			Unix.close nullfd
		| _ -> exit 0
		end
	| _ -> exit 0

exception Break

let lines_fold f start input =
	let accumulator = ref start in
	let running = ref true in
	while !running do
		let line =
			try Some (input_line input)
			with End_of_file -> None
		in
		match line with
		| Some line ->
			begin
				try accumulator := (f !accumulator line)
				with Break -> running := false
			end
		| None ->
				running := false
	done;
	!accumulator

let lines_iter f = lines_fold (fun () line -> ignore(f line)) ()

(** open a file, and make sure the close is always done *)
let with_input_channel file f =
	let input = open_in file in
	finally
{'fraction_non_alphanumeric': 0.09587772116720704, 'fraction_numerical': 0.010344295198394318, 'mean_word_length': 3.0462211118051217, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5478305', 'n_tokens_mistral': 2354, 'n_tokens_neox': 2052, 'n_words': 845}