<commit_before>(* <NAME>
 *
 * Copyright (C) 2010-2013 Facebook
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * version 2.1 as published by the Free Software Foundation, with the
 * special exception on linking described in file license.txt.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
 * license.txt for more details.
 *)
open Common 

(*****************************************************************************)
(* Prelude *)
(*****************************************************************************)

(*****************************************************************************)
(* Types *)
(*****************************************************************************)

(* see also dircolors.el and LFS *)
type file_type = 
  | PL of pl_type
  | Obj  of string (* .o, .a, .aux, .bak, etc *)
  | Binary of string
  | Text of string (* tex, txt, readme, noweb, org, etc *)
  | Doc  of string (* ps, pdf *)
  | Media of media_type
  | Archive of string (* tgz, rpm, etc *)
  | Other of string

 and pl_type = 
  | ML of string  (* mli, ml, mly, mll *)
  | Haskell of string
  | Lisp of lisp_type
  | Prolog of string
  | Makefile
  | Script of string (* sh, csh, awk, sed, etc *)
  | C of string | Cplusplus of string | ObjectiveC of string 
  | Java | Csharp
  | Perl | Python | Ruby | Lua
  | Erlang | Go | Rust
  | Beta
  | Pascal
  | Haxe | Opa | Flash
  | Web of webpl_type
  | Bytecode of string
  | Asm
  | Thrift
  | MiscPL of string

   and lisp_type = CommonLisp | Elisp | Scheme

   and webpl_type = 
     | Php of string (* php or phpt or script *)
     | Js | Coffee
     | Css
     | Html | Xml | Json
     | Sql

 and media_type =
   | Sound of string
   | Picture of string
   | Video of string

(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)

(* this function is used by codemap and archi_parse and called for each
 * filenames, so it has to be fast!
 *)
let file_type_of_file2 file = 
  let (d,b,e) = Common2.dbe_of_filename_noext_ok file in
  match e with

  | "ml" | "mli" 
  | "mly" | "mll"    
      -> PL (ML e)
  | "mlb" (* mlburg *)
  | "mlp" (* used in some source *)
  | "eliom" (* ocsigen, obviously *)
      -> PL (ML e)

  | "sml" -> PL (ML e)
  (* fsharp *)
  | "fsi" | "fsx" | "fs"  -> PL (ML e)
  (* linear ML *)
  | "lml"  -> PL (ML e)

  | "hs" | "lhs" -> PL (Haskell e)

  | "erl" | "hrl" -> PL Erlang

  | "hx" | "hxp" | "hxml" -> PL Haxe
  | "opa" -> PL Opa

  | "as" -> PL Flash

  | "bet" -> PL Beta

  (* todo detect false C file, look for "Mode: Objective-C++" string in file ?
   * can also be a c++, use Parser_cplusplus.is_problably_cplusplus_file 
   *)
  | "c" -> PL (C e)
  | "h" -> PL (C e)
  (* todo? have a PL of xxx_kind * pl_kind ?  *)
  | "y" | "l" -> PL (C e)

  | "hpp" -> PL (Cplusplus e) | "hxx" -> PL (Cplusplus e) 
  | "hh" -> PL (Cplusplus e)
  | "cpp" -> PL (Cplusplus e) | "C" -> PL (Cplusplus e)
  | "cc" -> PL (Cplusplus e)  | "cxx" -> PL (Cplusplus e)
  (* used in libstdc++ *)
  | "tcc" -> PL (Cplusplus e)

  | "m" | "mm" -> PL (ObjectiveC e)

  | "java" -> PL Java
  | "cs" -> PL Csharp

  | "p" -> PL Pascal

  | "thrift" -> PL Thrift

  | "scm" | "rkt" | "ss" -> PL (Lisp Scheme)
  | "lisp" -> PL (Lisp CommonLisp)
  | "el" -> PL (Lisp Elisp)

  (* Perl or Prolog ... I made my choice *)
  | "pl" -> PL (Prolog "pl")
  | "logic" -> PL (Prolog "logic") (* datalog of logicblox *)
  | "dtl" -> PL (Prolog "dtl") (* bddbddb *)
  | "dl" -> PL (Prolog "dl") (* datalog *)
  | "perl" -> PL Perl 
  | "py" -> PL Python
  | "rb" -> PL Ruby

  | "clp" -> PL (Prolog e)

  | "s" | "S" | "asm" -> PL Asm

  | "c--" -> PL (MiscPL e)
  | "oz" -> PL (MiscPL e)
  | "R" | "Rd" -> PL (MiscPL e)

  | "scala" -> PL (MiscPL e)

  | "sh" | "rc" | "csh" | "bash" -> PL (Script e)
  | "m4" -> PL (MiscPL e)
  | "conf" -> PL (MiscPL e)

  (* <NAME>'s Tiger toy language *)
  | "tig" -> PL (MiscPL e)

  (* merd *)
  | "me" -> PL (MiscPL "me")

  | "vim" -> PL (MiscPL "vim")
  | "nanorc" -> PL (MiscPL "nanorc")

  (* from hex to bcc *)
  | "he" -> PL (MiscPL "he")
  | "bc" -> PL (MiscPL "bc")

  | "php" | "phpt" -> PL (Web (Php e))
  | "css" -> PL (Web Css)
  (* "javascript" | "es" | ? *)
  | "js" -> PL (Web Js)
  | "coffee" -> PL (Web Coffee)
  | "html" | "htm" -> PL (Web Html)<commit_msg>* commons/file_type.ml: add a few one<commit_after>(* <NAME>
 *
 * Copyright (C) 2010-2013 Facebook
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public License
 * version 2.1 as published by the Free Software Foundation, with the
 * special exception on linking described in file license.txt.
 * 
 * This library is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the file
 * license.txt for more details.
 *)
open Common 

(*****************************************************************************)
(* Prelude *)
(*****************************************************************************)

(*****************************************************************************)
(* Types *)
(*****************************************************************************)

(* see also dircolors.el and LFS *)
type file_type = 
  | PL of pl_type
  | Obj  of string (* .o, .a, .aux, .bak, etc *)
  | Binary of string
  | Text of string (* tex, txt, readme, noweb, org, etc *)
  | Doc  of string (* ps, pdf *)
  | Media of media_type
  | Archive of string (* tgz, rpm, etc *)
  | Other of string

 and pl_type = 
  | ML of string  (* mli, ml, mly, mll *)
  | Haskell of string
  | Lisp of lisp_type
  | Prolog of string
  | Makefile
  | Script of string (* sh, csh, awk, sed, etc *)
  | C of string | Cplusplus of string | ObjectiveC of string 
  | Java | Csharp
  | Perl | Python | Ruby | Lua
  | Erlang | Go | Rust
  | Beta
  | Pascal
  | Haxe | Opa | Flash
  | Web of webpl_type
  | Bytecode of string
  | Asm
  | Thrift
  | MiscPL of string

   and lisp_type = CommonLisp | Elisp | Scheme

   and webpl_type = 
     | Php of string (* php or phpt or script *)
     | Js | Coffee
     | Css
     | Html | Xml | Json
     | Sql

 and media_type =
   | Sound of string
   | Picture of string
   | Video of string

(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)

(* this function is used by codemap and archi_parse and called for each
 * filenames, so it has to be fast!
 *)
let file_type_of_file2 file = 
  let (d,b,e) = Common2.dbe_of_filename_noext_ok file in
  match e with

  | "ml" | "mli" 
  | "mly" | "mll"    
      -> PL (ML e)
  | "mlb" (* mlburg *)
  | "mlp" (* used in some source *)
  | "eliom" (* ocsigen, obviously *)
      -> PL (ML e)

  | "sml" -> PL (ML e)
  (* fsharp *)
  | "fsi" | "fsx" | "fs"  -> PL (ML e)
  (* linear ML *)
  | "lml"  -> PL (ML e)

  | "hs" | "lhs" -> PL (Haskell e)

  | "erl" | "hrl" -> PL Erlang

  | "hx" | "hxp" | "hxml" -> PL Haxe
  | "opa" -> PL Opa

  | "as" -> PL Flash

  | "bet" -> PL Beta

  (* todo detect false C file, look for "Mode: Objective-C++" string in file ?
   * can also be a c++, use Parser_cplusplus.is_problably_cplusplus_file 
   *)
  | "c" -> PL (C e)
  | "h" -> PL (C e)
  (* todo? have a PL of xxx_kind * pl_kind ?  *)
  | "y" | "l" -> PL (C e)

  | "hpp" -> PL (Cplusplus e) | "hxx" -> PL (Cplusplus e) 
  | "hh" -> PL (Cplusplus e)
  | "cpp" -> PL (Cplusplus e) | "C" -> PL (Cplusplus e)
  | "cc" -> PL (Cplusplus e)  | "cxx" -> PL (Cplusplus e)
  (* used in libstdc++ *)
  | "tcc" -> PL (Cplusplus e)

  | "m" | "mm" -> PL (ObjectiveC e)

  | "java" -> PL Java
  | "cs" -> PL Csharp

  | "p" -> PL Pascal

  | "thrift" -> PL Thrift

  | "scm" | "rkt" | "ss" | "lsp" -> PL (Lisp Scheme)
  | "lisp" -> PL (Lisp CommonLisp)
  | "el" -> PL (Lisp Elisp)

  (* Perl or Prolog ... I made my choice *)
  | "pl" -> PL (Prolog "pl")
  | "logic" -> PL (Prolog "logic") (* datalog of logicblox *)
  | "dtl" -> PL (Prolog "dtl") (* bddbddb *)
  | "dl" -> PL (Prolog "dl") (* datalog *)
  | "perl" -> PL Perl 
  | "py" -> PL Python
  | "rb" -> PL Ruby

  | "clp" -> PL (Prolog e)

  | "s" | "S" | "asm" -> PL Asm

  | "c--" -> PL (MiscPL e)
  | "oz" -> PL (MiscPL e)
  | "R" | "Rd" -> PL (MiscPL e)

  | "scala" -> PL (MiscPL e)
  | "groovy" -> PL (MiscPL e)

  | "sh" | "rc" | "csh" | "bash" -> PL (Script e)
  | "m4" -> PL (MiscPL e)
  | "conf" -> PL (MiscPL e)

  (* <NAME>'s Tiger toy language *)
  | "tig" -> PL (MiscPL e)

  (* merd *)
  | "me" -> PL (MiscPL "me")

  | "vim" -> PL (MiscPL "vim")
  | "nanorc" -> PL (MiscPL "nanorc")

  (* from hex to bcc *)
  | "he" -> PL (MiscPL "he")
  | "bc" -> PL (MiscPL "bc")

  | "php" | "phpt" -> PL (Web (Php e))
  | "css" -> PL (Web Css)
  (* "javascript" | "es" | ? *)
  | "js" -> PL (Web Js)
  | "coffee" -> PL (Web Coffee)
  | "html" | "htm" -> PL (Web Html)
{'fraction_non_alphanumeric': 0.258974358974359, 'fraction_numerical': 0.002777777777777778, 'mean_word_length': 2.7519038076152307, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 122, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2443941', 'n_tokens_mistral': 3613, 'n_tokens_neox': 3392, 'n_words': 1133}