<commit_before>open Batteries
open Log
module N = RamenSharedTypes.Node

type options = { debug : bool ; monitor : bool }

let options debug monitor = { debug ; monitor }

let rebase dataset_name name =
  if dataset_name = "" then name else dataset_name ^"/"^ name

(* Get the operations to import the dataset and do basic transformations.
 * Named streams belonging to this base layer:
 * - ${dataset_name}/csv: Raw imported tuples straight from the CSV;
 * - ${dataset_name}/c2s: The first of a pair of streams of traffic info
 *                        (source to dest rather than client to server);
 * - ${dataset_name}/s2c: The second stream of traffic info;
 *
 * If you wish to process traffic info you must feed on both c2s and s2c.
 *)
let base_layer dataset_name delete csv_dir =
  let make_node ?(parents=[]) name operation =
    let parents = List.map (rebase dataset_name) parents in<commit_msg>Fix bug when layer name is empty.<commit_after>open Batteries
open Log
module N = RamenSharedTypes.Node

type options = { debug : bool ; monitor : bool }

let options debug monitor = { debug ; monitor }

let rebase dataset_name name = dataset_name ^"/"^ name

(* Get the operations to import the dataset and do basic transformations.
 * Named streams belonging to this base layer:
 * - ${dataset_name}/csv: Raw imported tuples straight from the CSV;
 * - ${dataset_name}/c2s: The first of a pair of streams of traffic info
 *                        (source to dest rather than client to server);
 * - ${dataset_name}/s2c: The second stream of traffic info;
 *
 * If you wish to process traffic info you must feed on both c2s and s2c.
 *)
let base_layer dataset_name delete csv_dir =
  let make_node ?(parents=[]) name operation =
    let parents = List.map (rebase dataset_name) parents in
{'fraction_non_alphanumeric': 0.0801781737193764, 'fraction_numerical': 0.004454342984409799, 'mean_word_length': 3.572519083969466, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1992950', 'n_tokens_mistral': 502, 'n_tokens_neox': 493, 'n_words': 228}