<commit_before>(*s: parsing2.ml *)
(*s: Facebook copyright *)
(* <NAME>
 * 
 * Copyright (C) 2010 Facebook
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * version 2.1 as published by the Free Software Foundation, with the
 * special exception on linking described in file license.txt.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
 * license.txt for more details.
 *)
(*e: Facebook copyright *)

open Common

module FT = File_type

module HC = Highlight_code
module Db = Database_code

open Highlight_code

(*****************************************************************************)
(* Parsing helpers *)
(*****************************************************************************)

(* This type is needed if we want to use a single hashtbl to memoize
 * all the parsed file. Having a single hash helps for 
 * disable_file_in_cache below.
 *)
type ast = 
  | ML  of Parse_ml.program2
  | Php of Parse_php.program2
  | Cpp of Parse_cpp.program2
  | Js  of Parse_js.program2

  | Noweb of Parse_nw.program2


let _hmemo_file = Hashtbl.create 101

let parse_ml2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    ML (Parse_ml.parse file +> fst))
let parse_ml_cache a = 
  Common.profile_code "View.parse_ml_cache" (fun () -> 
    match parse_ml2 a with | ML a -> a | _ -> raise Impossible
  )

let parse_nw2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Noweb (Parse_nw.parse file +> fst))
let parse_nw_cache a = 
  Common.profile_code "View.parse_nw_cache" (fun () -> 
    match parse_nw2 a with | Noweb a -> a | _ -> raise Impossible
  )


let parse_php2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    let (ast2, stat) = Parse_php.parse file in
    let ast = Parse_php.program_of_program2 ast2 in
    (* work by side effect on ast2 too *)
    Check_variables_php.check_and_annotate_program
      ast;
    Php ast2
  )
let parse_php_cache a = 
  Common.profile_code "View.parse_php_cache" (fun () -> 
    match parse_php2 a with | Php a -> a | _ -> raise Impossible
  )

let parse_js2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Js (Parse_js.parse file +> fst))
let parse_js_cache a = 
  Common.profile_code "View.parse_js_cache" (fun () -> 
    match parse_js2 a with | Js a -> a | _ -> raise Impossible
  )

let parse_cpp2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Cpp (Parse_cpp.parse file +> fst))
let parse_cpp_cache a = 
  Common.profile_code "View.parse_cpp_cache" (fun () -> 
    match parse_cpp2 a with | Cpp a -> a | _ -> raise Impossible
  )




let disable_file_in_cache file =
  Hashtbl.remove _hmemo_file file


(*****************************************************************************)
(* Semantic ehancement *)
(*****************************************************************************)

let use_arity_of_use_count n =
  match () with
  (* note that because my PHP object analysis have some threshold
   * on the number of callers (see threshold_callers_indirect_db)
   * the number for HugeUse can not be more than this one otherwise
   * you will miss some cases
   *)
  | _ when n >= 100 -> HugeUse

  | _ when n > 20 -> LotsOfUse
  | _ when n >= 10  -> MultiUse
  | _ when n >= 2 -> SomeUse
  | _ when n = 1 -> UniqueUse
  | _ -> NoUse

let rewrite_categ_using_entities s categ file entities =

  let e_kind_opt = 
    try Some (Db.entity_kind_of_highlight_category categ)
    with _ -> None
  in
  match e_kind_opt with
  | None -> categ
  | Some e_kind ->

  
   let entities = 
    Hashtbl.find_all entities s +> List.filter (fun e ->
      (* we could have the full www dbcode but run the treemap on
       * a subdir in which case the root will not be the same.
       * It's a good approximation to just look at the basename.
       * The only false positive we will get if another file,
       * with the same name happened to also define entities
       * with the same name, which would be rare.
       * 
       * update: TODO use Model2.readable_to_absolute_filename_under_root ?
       *)
      Filename.basename e.Db.e_file = Filename.basename file &&
      (* some file have both a function and class with the same name *)
      e.Db.e_kind = e_kind
    )
  in
  match entities with
  | [] -> categ
  | [e] ->
      let use_cnt = e.Db.e_number_external_users in
      let arity = use_arity_of_use_count use_cnt in
      (* coupling: you must also say in style2 to take into account
       * the arity for the category
       *)
      (match categ with
      | Function (Def2 _) ->
          Function (Def2 arity)
      | FunctionDecl _ ->
          FunctionDecl (arity)
      | Class (Def2 _) ->
          Class (Def2 arity)
      | Method (Def2 _) ->
          Method (Def2 arity)
      | StaticMethod (Def2 _) ->
          StaticMethod (Def2 arity)
      | _ -> categ
      )
  | x::y::xs ->
      (* TODO: handle __construct directly *)
      if not (List.mem s ["__construct"])
      then
        pr2_once (spf "multi def found for %s in %s" s file);
      categ

(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)

(* coupling: right now if you add a language here, you need to whitelist it
 * also in draw2.draw_contents2.<commit_msg>visual: adding support for fields and macros<commit_after>(*s: parsing2.ml *)
(*s: Facebook copyright *)
(* <NAME>
 * 
 * Copyright (C) 2010 Facebook
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * version 2.1 as published by the Free Software Foundation, with the
 * special exception on linking described in file license.txt.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
 * license.txt for more details.
 *)
(*e: Facebook copyright *)

open Common

module FT = File_type

module HC = Highlight_code
module Db = Database_code

open Highlight_code

(*****************************************************************************)
(* Parsing helpers *)
(*****************************************************************************)

(* This type is needed if we want to use a single hashtbl to memoize
 * all the parsed file. Having a single hash helps for 
 * disable_file_in_cache below.
 *)
type ast = 
  | ML  of Parse_ml.program2
  | Php of Parse_php.program2
  | Cpp of Parse_cpp.program2
  | Js  of Parse_js.program2

  | Noweb of Parse_nw.program2


let _hmemo_file = Hashtbl.create 101

let parse_ml2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    ML (Parse_ml.parse file +> fst))
let parse_ml_cache a = 
  Common.profile_code "View.parse_ml_cache" (fun () -> 
    match parse_ml2 a with | ML a -> a | _ -> raise Impossible
  )

let parse_nw2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Noweb (Parse_nw.parse file +> fst))
let parse_nw_cache a = 
  Common.profile_code "View.parse_nw_cache" (fun () -> 
    match parse_nw2 a with | Noweb a -> a | _ -> raise Impossible
  )


let parse_php2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    let (ast2, stat) = Parse_php.parse file in
    let ast = Parse_php.program_of_program2 ast2 in
    (* work by side effect on ast2 too *)
    Check_variables_php.check_and_annotate_program
      ast;
    Php ast2
  )
let parse_php_cache a = 
  Common.profile_code "View.parse_php_cache" (fun () -> 
    match parse_php2 a with | Php a -> a | _ -> raise Impossible
  )

let parse_js2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Js (Parse_js.parse file +> fst))
let parse_js_cache a = 
  Common.profile_code "View.parse_js_cache" (fun () -> 
    match parse_js2 a with | Js a -> a | _ -> raise Impossible
  )

let parse_cpp2 file = 
  Common.memoized _hmemo_file file (fun () -> 
    Cpp (Parse_cpp.parse file +> fst))
let parse_cpp_cache a = 
  Common.profile_code "View.parse_cpp_cache" (fun () -> 
    match parse_cpp2 a with | Cpp a -> a | _ -> raise Impossible
  )




let disable_file_in_cache file =
  Hashtbl.remove _hmemo_file file


(*****************************************************************************)
(* Semantic ehancement *)
(*****************************************************************************)

let use_arity_of_use_count n =
  match () with
  (* note that because my PHP object analysis have some threshold
   * on the number of callers (see threshold_callers_indirect_db)
   * the number for HugeUse can not be more than this one otherwise
   * you will miss some cases
   *)
  | _ when n >= 100 -> HugeUse

  | _ when n > 20 -> LotsOfUse
  | _ when n >= 10  -> MultiUse
  | _ when n >= 2 -> SomeUse
  | _ when n = 1 -> UniqueUse
  | _ -> NoUse

let rewrite_categ_using_entities s categ file entities =

  let e_kind_opt = 
    try Some (Db.entity_kind_of_highlight_category categ)
    with _ -> None
  in
  match e_kind_opt with
  | None -> categ
  | Some e_kind ->

  
   let entities = 
    Hashtbl.find_all entities s +> List.filter (fun e ->
      (* we could have the full www dbcode but run the treemap on
       * a subdir in which case the root will not be the same.
       * It's a good approximation to just look at the basename.
       * The only false positive we will get if another file,
       * with the same name happened to also define entities
       * with the same name, which would be rare.
       * 
       * update: TODO use Model2.readable_to_absolute_filename_under_root ?
       *)
      Filename.basename e.Db.e_file = Filename.basename file &&
      (* some file have both a function and class with the same name *)
      e.Db.e_kind = e_kind
    )
  in
  match entities with
  | [] -> categ
  | [e] ->
      let use_cnt = e.Db.e_number_external_users in
      let arity = use_arity_of_use_count use_cnt in
      (* coupling: you must also say in style2 to take into account
       * the arity for the category
       *)
      (match categ with
      | Function (Def2 _) ->
          Function (Def2 arity)
      | FunctionDecl _ ->
          FunctionDecl (arity)
      | Class (Def2 _) ->
          Class (Def2 arity)
      | Method (Def2 _) ->
          Method (Def2 arity)
      | StaticMethod (Def2 _) ->
          StaticMethod (Def2 arity)
      | Field (Def2 _) ->
          Field (Def2 arity)
      | MacroVar (Def2 _) ->
          MacroVar (Def2 arity)
      | Macro (Def2 _) ->
          Macro (Def2 arity)
      | _ -> categ
      )
  | x::y::xs ->
      (* TODO: handle __construct directly *)
      if not (List.mem s ["__construct"])
      then
        pr2_once (spf "multi def found for %s in %s" s file);
      categ

(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)

(* coupling: right now if you add a language here, you need to whitelist it
 * also in draw2.draw_contents2.
{'fraction_non_alphanumeric': 0.16238255628434675, 'fraction_numerical': 0.009750044318383265, 'mean_word_length': 2.9812985179957656, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509890', 'n_tokens_mistral': 3550, 'n_tokens_neox': 3347, 'n_words': 1184}