<commit_before>(*
 * Copyright (C) Citrix Systems Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)

open Lwt

let xenstored_proc_port = "/proc/xen/xsd_port"
let xenstored_proc_kva = "/proc/xen/xsd_kva"

let page_size = 4096 (* xenstore doesn't support anything else *)

let open_ring0 () =
	let fd = Unix.openfile xenstored_proc_kva [ Unix.O_RDWR ] 0o600 in
	let page = Domains.map_fd fd page_size in
	Unix.close fd;
	Cstruct.of_bigarray page

let open_ringU domid mfn =
	let len = 4096 in
	let page = Domains.map_foreign domid mfn in
	Cstruct.of_bigarray page

let open_ring domid mfn : Cstruct.t =
	if domid = 0
	then open_ring0 ()
	else open_ringU domid mfn

let load_ring filename : Cstruct.t =
	let ba = Bigarray.(Array1.create char c_layout page_size) in
	let c = Cstruct.of_bigarray ba in
	let s = String.make page_size '\000' in
	let f = Unix.openfile filename [ Unix.O_RDONLY ] 0o0 in
	let n = Unix.read f s 0 page_size in
	if n <> page_size
	then failwith (Printf.sprintf "Failed to read a page of data from %s" filename);
	Unix.close f;
	Cstruct.blit_from_string s 0 c 0 page_size;
	c

let save_ring ring filename =
	let f = Unix.openfile filename [ Unix.O_WRONLY; Unix.O_CREAT ] 0o644 in
	let s = String.make 4096 '\000' in
	Cstruct.blit_to_string ring 0 s 0 (String.length s);
	Unix.write f s 0 (String.length s);
	Unix.close f

cstruct ring {
	uint8_t output[1024];
	uint8_t input[1024];
	uint32_t output_cons;
	uint32_t output_prod;
	uint32_t input_cons;
	uint32_t input_prod
} as little_endian

(* true if we are sure the header is invalid, false otherwise *)
let is_header_invalid c =
	true
	&& (Cstruct.len c >= Protocol.Header.sizeof)
	&& (
		match Protocol.Header.unmarshal c with
		| `Error _ -> false
		| `Ok _ -> true
	)

let fold_over_packets f init c =
	let rec loop remaining acc =
		match Protocol.Header.unmarshal remaining with
		| `Error _ -> acc
		| `Ok hdr ->
			let remaining = Cstruct.shift remaining Protocol.Header.sizeof in
			if Cstruct.len remaining < hdr.Protocol.Header.len
			then acc
			else
				let data = Cstruct.sub remaining 0 hdr.Protocol.Header.len in
				let remaining = Cstruct.shift remaining hdr.Protocol.Header.len in
				loop remaining (f acc (hdr, data)) in
	loop c init	

let count_packets c = List.length (fold_over_packets (fun acc p -> p :: acc) [] c)

let printable = function 
	| 'a' .. 'z'
	| 'A' .. 'Z'<commit_msg>dbgring: ignore blocks of 0s<commit_after>(*
 * Copyright (C) Citrix Systems Inc.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation; version 2.1 only. with the special
 * exception on linking described in file LICENSE.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *)

open Lwt

let xenstored_proc_port = "/proc/xen/xsd_port"
let xenstored_proc_kva = "/proc/xen/xsd_kva"

let page_size = 4096 (* xenstore doesn't support anything else *)

let open_ring0 () =
	let fd = Unix.openfile xenstored_proc_kva [ Unix.O_RDWR ] 0o600 in
	let page = Domains.map_fd fd page_size in
	Unix.close fd;
	Cstruct.of_bigarray page

let open_ringU domid mfn =
	let len = 4096 in
	let page = Domains.map_foreign domid mfn in
	Cstruct.of_bigarray page

let open_ring domid mfn : Cstruct.t =
	if domid = 0
	then open_ring0 ()
	else open_ringU domid mfn

let load_ring filename : Cstruct.t =
	let ba = Bigarray.(Array1.create char c_layout page_size) in
	let c = Cstruct.of_bigarray ba in
	let s = String.make page_size '\000' in
	let f = Unix.openfile filename [ Unix.O_RDONLY ] 0o0 in
	let n = Unix.read f s 0 page_size in
	if n <> page_size
	then failwith (Printf.sprintf "Failed to read a page of data from %s" filename);
	Unix.close f;
	Cstruct.blit_from_string s 0 c 0 page_size;
	c

let save_ring ring filename =
	let f = Unix.openfile filename [ Unix.O_WRONLY; Unix.O_CREAT ] 0o644 in
	let s = String.make 4096 '\000' in
	Cstruct.blit_to_string ring 0 s 0 (String.length s);
	Unix.write f s 0 (String.length s);
	Unix.close f

cstruct ring {
	uint8_t output[1024];
	uint8_t input[1024];
	uint32_t output_cons;
	uint32_t output_prod;
	uint32_t input_cons;
	uint32_t input_prod
} as little_endian

(* true if we are sure the header is invalid, false otherwise *)
let is_header_invalid c =
	true
	&& (Cstruct.len c >= Protocol.Header.sizeof)
	&& (
		match Protocol.Header.unmarshal c with
		| `Error _ -> false
		| `Ok _ -> true
	)

let fold_over_packets f init c =
	let rec loop remaining acc =
		match Protocol.Header.unmarshal remaining with
		| `Error _ -> acc
		| `Ok hdr ->
			(* Ignore "debug" packets which have length = rid = tid = 0: this is
			   more likely to be empty space. *)
			if true
			  && hdr.Protocol.Header.ty = Protocol.Op.Debug
			  && hdr.Protocol.Header.tid = 0l
			  && hdr.Protocol.Header.rid = 0l
			  && hdr.Protocol.Header.len = 0
			then acc
			else begin
				let remaining = Cstruct.shift remaining Protocol.Header.sizeof in
				if Cstruct.len remaining < hdr.Protocol.Header.len
				then acc
				else
					let data = Cstruct.sub remaining 0 hdr.Protocol.Header.len in
					let remaining = Cstruct.shift remaining hdr.Protocol.Header.len in
					loop remaining (f acc (hdr, data))
			end in
	loop c init	

let count_packets c = List.length (fold_over_packets (fun acc p -> p :: acc) [] c)

let printable = function 
	| 'a' .. 'z'
	| 'A' .. 'Z'
{'fraction_non_alphanumeric': 0.0836489379219496, 'fraction_numerical': 0.020253581425983862, 'mean_word_length': 3.4959289415247965, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2931635', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2060, 'n_words': 757}