<commit_before>(*
 * Please imagine a long and boring gnu-style copyright notice 
 * appearing just here.
 *)
open Common

(*****************************************************************************)
(* Purpose *)
(*****************************************************************************)
(* 
 * A more precise etags/ctags. 
 *
 * Both etags and ctags are working at a lexical level
 * and so have lots of false positives for the definitions. For instance
 * on PHP, a line containing $idx will be considered as a candidate when
 * searching for idx. By using a real parser tagging a file becomes
 * trivial and correct (but slower I have to admit).
 * 
 * Note that for OCaml the situation is kinda reversed. 'otags' 
 * can work only at the AST level, which requires to
 * correctly parse the file. Nevertheless many files using camlp4 are
 * causing otags to fatal. One option is to help otags by passing it
 * the correct -pp flags. Another option is to at least default to
 * a lexical-level tag generator which is what I do for OCaml.
 * 
 * usage: 
 *  $ stags -lang web -o TAGS *
 *)

(*****************************************************************************)
(* Flags *)
(*****************************************************************************)

(* In addition to flags that can be tweaked via -xxx options (cf the
 * full list of options in the "the options" section below).
 *)
let verbose = ref false

(* action mode *)
let action = ref ""

(* obsolete ? *)
let heavy_tagging = ref false<commit_msg>* main_stags.ml: comments<commit_after>(*
 * Please imagine a long and boring gnu-style copyright notice 
 * appearing just here.
 *)
open Common

(*****************************************************************************)
(* Purpose *)
(*****************************************************************************)
(* 
 * A more precise etags/ctags. 
 *
 * Both etags and ctags are working at a lexical level
 * and so have lots of false positives for the definitions. For instance
 * on PHP, a line containing $idx will be considered as a candidate when
 * searching for idx. By using a real parser tagging a file becomes
 * trivial and correct (but slower I have to admit).
 * 
 * Note that for OCaml the situation was kinda reversed. 'otags' 
 * can work only at the AST level, which requires to
 * correctly parse the file. Nevertheless many files using camlp4 are
 * causing otags to fatal. One option is to help otags by passing it
 * the correct -pp flags. Another option is to at least default to
 * a lexical-level tag generator which is what I do for OCaml.
 * update: with the new .cmt format, one can finally have a precise
 * tag generator for OCaml code too.
 * 
 * usage: 
 *  $ stags -lang web -o TAGS *
 *)

(*****************************************************************************)
(* Flags *)
(*****************************************************************************)

(* In addition to flags that can be tweaked via -xxx options (cf the
 * full list of options in the "the options" section below).
 *)
let verbose = ref false

(* action mode *)
let action = ref ""

(* obsolete ? *)
let heavy_tagging = ref false
{'fraction_non_alphanumeric': 0.2577092511013216, 'fraction_numerical': 0.0006293266205160479, 'mean_word_length': 4.351851851851852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491366', 'n_tokens_mistral': 790, 'n_tokens_neox': 723, 'n_words': 391}