<commit_before>(*
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

module Attrs = Attrs

module Maps = Maps

module Paths = Paths

module Types = Model

type 'a lookup_result = 'a Component_table.lookup_unit_result =
  | Forward_reference
  | Found of { root : 'a; hidden : bool }
  | Not_found

let core_types = Predefined.core_types

let core_exceptions = Predefined.core_exceptions

type 'a result =
  | Ok of 'a Model.Compilation_unit.t
  | Not_an_interface
  | Wrong_version
  | Corrupted
  | Not_a_typedtree
  | Not_an_implementation

let read_cmti root_fn filename =
  try
    let cmt_info = Cmt_format.read_cmt filename in
    match cmt_info.cmt_annots with
    | Interface intf -> begin
      match cmt_info.cmt_interface_digest with
      | Some digest ->
        let name = cmt_info.cmt_modname in
        let root = root_fn name digest in
        let (id, doc, items) = Cmti.read_interface root name intf in
        let imports =
          List.filter (fun (name', _) -> name <> name') cmt_info.cmt_imports
        in
        let imports =
          List.map (fun (s, d) ->
            Model.Compilation_unit.Import.Unresolved (s, d))
          imports
        in
        let interface = true in
        let hidden = Paths.contains_double_underscore name in
        let source =
          match cmt_info.cmt_sourcefile, cmt_info.cmt_source_digest with
          | Some file, Some digest ->
            let build_dir = cmt_info.cmt_builddir in
            Some {Model.Compilation_unit.Source.file; digest; build_dir}
          | _, _ -> None
        in
        let content = Model.Compilation_unit.Module items in
        let unit =
          {Model.Compilation_unit.id; doc; digest; imports; source;
           interface; hidden; content; expansion = None}
        in
        let unit = Lookup.lookup unit in
          Ok unit
      | None -> Corrupted
    end
    | _ -> Not_an_interface
  with
  | Cmi_format.Error (Not_an_interface _) -> Not_an_interface
  | Cmi_format.Error (Wrong_version_interface _) -> Wrong_version
  | Cmi_format.Error (Corrupted_interface _) -> Corrupted
  | Cmt_format.Error (Not_a_typedtree _) -> Not_a_typedtree

let read_cmt root_fn filename =
  let open Cmi_format in
  let open Cmt_format in
  let open Model.Compilation_unit in
  try
    let cmt_info = read_cmt filename in
    match cmt_info.cmt_annots with
    | Packed(_, files) ->
        let name = cmt_info.cmt_modname in
        let interface, digest =
          match cmt_info.cmt_interface_digest with
          | Some digest -> true, digest
          | None ->
            match List.assoc name cmt_info.cmt_imports with
            | Some digest -> false, digest
            | None -> assert false
            | exception Not_found -> assert false
        in
        let hidden = Paths.contains_double_underscore name in
        let root = root_fn name digest in
        let id = Paths.Identifier.Root(root, name) in
        let items =
          List.map<commit_msg>Doc_model.read_cmti: limit exception handler scope<commit_after>(*
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 *)

module Attrs = Attrs

module Maps = Maps

module Paths = Paths

module Types = Model

type 'a lookup_result = 'a Component_table.lookup_unit_result =
  | Forward_reference
  | Found of { root : 'a; hidden : bool }
  | Not_found

let core_types = Predefined.core_types

let core_exceptions = Predefined.core_exceptions

type 'a result =
  | Ok of 'a Model.Compilation_unit.t
  | Not_an_interface
  | Wrong_version
  | Corrupted
  | Not_a_typedtree
  | Not_an_implementation

let read_cmti root_fn filename =
  match Cmt_format.read_cmt filename with
  | exception Cmi_format.Error (Not_an_interface _) -> Not_an_interface
  | exception Cmi_format.Error (Wrong_version_interface _) -> Wrong_version
  | exception Cmi_format.Error (Corrupted_interface _) -> Corrupted
  | exception Cmt_format.Error (Not_a_typedtree _) -> Not_a_typedtree
  | cmt_info ->
    match cmt_info.cmt_annots with
    | Interface intf -> begin
      match cmt_info.cmt_interface_digest with
      | Some digest ->
        let name = cmt_info.cmt_modname in
        let root = root_fn name digest in
        let (id, doc, items) = Cmti.read_interface root name intf in
        let imports =
          List.filter (fun (name', _) -> name <> name') cmt_info.cmt_imports
        in
        let imports =
          List.map (fun (s, d) ->
            Model.Compilation_unit.Import.Unresolved (s, d))
          imports
        in
        let interface = true in
        let hidden = Paths.contains_double_underscore name in
        let source =
          match cmt_info.cmt_sourcefile, cmt_info.cmt_source_digest with
          | Some file, Some digest ->
            let build_dir = cmt_info.cmt_builddir in
            Some {Model.Compilation_unit.Source.file; digest; build_dir}
          | _, _ -> None
        in
        let content = Model.Compilation_unit.Module items in
        let unit =
          {Model.Compilation_unit.id; doc; digest; imports; source;
           interface; hidden; content; expansion = None}
        in
        let unit = Lookup.lookup unit in
          Ok unit
      | None -> Corrupted
    end
    | _ -> Not_an_interface

let read_cmt root_fn filename =
  let open Cmi_format in
  let open Cmt_format in
  let open Model.Compilation_unit in
  try
    let cmt_info = read_cmt filename in
    match cmt_info.cmt_annots with
    | Packed(_, files) ->
        let name = cmt_info.cmt_modname in
        let interface, digest =
          match cmt_info.cmt_interface_digest with
          | Some digest -> true, digest
          | None ->
            match List.assoc name cmt_info.cmt_imports with
            | Some digest -> false, digest
            | None -> assert false
            | exception Not_found -> assert false
        in
        let hidden = Paths.contains_double_underscore name in
        let root = root_fn name digest in
        let id = Paths.Identifier.Root(root, name) in
        let items =
          List.map
{'fraction_non_alphanumeric': 0.06588991562876657, 'fraction_numerical': 0.0010713807419311639, 'mean_word_length': 2.5938402309913378, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '318586', 'n_tokens_mistral': 2271, 'n_tokens_neox': 2201, 'n_words': 743}