<commit_before>(**************************************************************************)
(*                                                                        *)
(*    Copyright 2012-2013 OCamlPro                                        *)
(*    Copyright 2012 INRIA                                                *)
(*                                                                        *)
(*  All rights reserved.This file is distributed under the terms of the   *)
(*  GNU Lesser General Public License version 3.0 with linking            *)
(*  exception.                                                            *)
(*                                                                        *)
(*  OPAM is distributed in the hope that it will be useful, but WITHOUT   *)
(*  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY    *)
(*  or FITNESS FOR A PARTICULAR PURPOSE.See the GNU General Public        *)
(*  License for more details.                                             *)
(*                                                                        *)
(**************************************************************************)

open OpamTypes
open OpamState.Types

let log fmt = OpamGlobals.log "REPOSITORY" fmt

let print_updated_compilers updates =

  let print singular plural map =
    if not (OpamCompiler.Set.is_empty map) then (
      if OpamCompiler.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamCompiler.Set.iter (fun comp ->
        OpamGlobals.msg " - %s\n" (OpamCompiler.to_string comp)
      ) map
    ) in

  print
    "The following NEW compiler is available"
    "The following NEW compilers are available"
    updates.created;

  print
    "The following compiler description has been UPDATED"
    "The following compiler descriptions have been UPDATED"
    updates.updated;

  print
    "The following compiler description has been DELETED"
    "The following compiler descriptions are been DELETED"
    updates.deleted

let filter_compiler_checksums cs =
  let keep f = OpamFilename.check_suffix f ".comp" in
  List.filter keep cs

let fix_compiler_descriptions t ~verbose =
  OpamGlobals.msg "Updating %s/ ...\n"
    (OpamFilename.prettify_dir (OpamPath.compilers_dir t.root));
  let global_index = OpamState.compiler_state t in
  let repo_index = OpamState.compiler_repository_state t in
  let niet = String.concat ":" in
  log "global-index: %s" (OpamCompiler.Map.to_string niet global_index);
  log "repo-index  : %s" (OpamCompiler.Map.to_string niet repo_index);

  let updated_compilers, new_compilers =
    let updated_compilers =
      OpamCompiler.Map.fold (fun comp state set ->
          if not (OpamCompiler.Map.mem comp global_index)
          || OpamCompiler.Map.find comp global_index <> state then
            OpamCompiler.Set.add comp set
          else
            set
        ) repo_index OpamCompiler.Set.empty in
    OpamCompiler.Set.partition (OpamState.is_compiler_installed t) updated_compilers in
  log "updated-compilers: %s" (OpamCompiler.Set.to_string updated_compilers);
  log "new-compilers    : %s" (OpamCompiler.Set.to_string new_compilers);

  let deleted_compilers =
    OpamCompiler.Set.fold (fun comp map ->
        if comp = OpamCompiler.system             (* system *)
        || OpamCompiler.Map.mem comp repo_index   (* OR available *)
        || OpamState.is_compiler_installed t comp (* OR installed *)
        then
          map
        else
          OpamCompiler.Set.add comp map
      ) t.compilers OpamCompiler.Set.empty in
  log "deleted-compilers: %s" (OpamCompiler.Set.to_string deleted_compilers);

  (* Delete compiler descritions *)
  OpamCompiler.Set.iter (fun comp ->
      let dir = OpamPath.compilers t.root comp in
      OpamFilename.rmdir dir;
    ) deleted_compilers;

  (* Update the compiler description *)
  OpamCompiler.Set.iter (fun comp ->
      match OpamState.repository_and_prefix_of_compiler t comp with
      | None                -> ()
      | Some (repo, prefix) ->
        let files = OpamRepository.compiler_files repo prefix comp in
        let dir = OpamPath.compilers t.root comp in
        if OpamFilename.exists_dir dir then OpamFilename.rmdir dir;
        OpamFilename.mkdir dir;
        List.iter (fun file ->
            OpamFilename.copy_in file dir
          ) files;
    ) (OpamCompiler.Set.union updated_compilers new_compilers);

  let updates = {
    created = new_compilers;
    updated = updated_compilers;
    deleted = deleted_compilers;
    changed = OpamCompiler.Set.empty; (* we don't reinstall compilers yet *)
  } in

  if verbose then print_updated_compilers updates;

  updates

let print_updated_packages t updates =

  let print singular plural map fn =
    if not (OpamPackage.Set.is_empty map) then (
      if OpamPackage.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamPackage.Set.iter (fun nv ->
        match fn nv with
        | None   -> OpamGlobals.msg " - %s\n" (OpamPackage.to_string nv)
        | Some s -> OpamGlobals.msg " - %s [%s]\n" (OpamPackage.to_string nv) s
      ) map
    ) in

  let installed_switches nv =
    let installed = OpamState.installed_versions t (OpamPackage.name nv) in
    let installed =
      try OpamPackage.Map.find nv installed
      with Not_found -> [] in (* XXX: should never happen *)
    match installed with
    | [] -> None
    | _  -> Some (
        Printf.sprintf "%s (%s)"
          (OpamPackage.Version.to_string (OpamPackage.version nv))
          (OpamMisc.pretty_list (List.map OpamSwitch.to_string installed))
      ) in

  let none _ = None in

  print
    "The following NEW package is available"
    "The following NEW packages are available"
    updates.created
    none;

  print
    "The following package has been CHANGED upstream and needs to be recompiled"
    "The following packages have been CHANGED upstream and need to be recompiled"
    updates.changed
    installed_switches;

  print
    "The following package has been UPDATED upstream"
    "The following packages have been UPDATED upstream"
    updates.updated
    none;

  print
    "The following package has been DELETED"
    "The following packages have been DELETED"
    updates.deleted
    none

let print_updated_dev_packages pinned_packages =
  let print singular plural map =
    if not (OpamPackage.Set.is_empty map) then (
      if OpamPackage.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamPackage.Set.iter (fun nv ->
          OpamGlobals.msg " - %s\n" (OpamPackage.to_string nv)
        ) map
    ) in
  print
    "The following DEV package needs to be upgraded"
    "The following DEV packages need to be upgraded"
    pinned_packages

(* Check for updates in pinned packages *)
let update_dev_packages t ~verbose packages =
  log "update-dev-packages updates %s" (OpamPackage.Set.to_string packages);
  let updates = OpamState.update_dev_packages t in
  if verbose then print_updated_dev_packages updates;
  updates

let filter_package_checksums cs =
  let keep f =
    match OpamFilename.Base.to_string (OpamFilename.basename f) with
    | "opam" | "descr" -> false
    | _ -> true in
  List.filter keep cs

(* Update the package contents, display the new packages and update
   reinstall *)
let fix_package_descriptions t ~verbose =
  OpamGlobals.msg "Updating %s/ ...\n"
    (OpamFilename.prettify_dir (OpamPath.packages_dir t.root));

  let global_index = OpamState.package_state t in
  let repo_index   = OpamState.package_repository_state t in
  let niet = String.concat ":" in
  log "global-index: %s" (OpamPackage.Map.to_string niet global_index);
  log "repo-index  : %s" (OpamPackage.Map.to_string niet repo_index);

  let updated_packages, new_packages =
    let updated_packages =
      OpamPackage.Map.fold (fun nv state set ->
          if not (OpamPackage.Map.mem nv global_index)
          || OpamPackage.Map.find nv global_index <> state then
            OpamPackage.Set.add nv set
          else
            set
        ) repo_index OpamPackage.Set.empty in
    OpamPackage.Set.partition (fun nv ->
        OpamPackage.Map.mem nv global_index
      ) updated_packages in
  log "new-packages    : %s" (OpamPackage.Set.to_string new_packages);

  let all_installed = OpamState.all_installed t in
  let changed_packages, updated_packages =
    OpamPackage.Set.partition (fun nv ->
        OpamPackage.Set.mem nv all_installed
      ) updated_packages in
  log "updated-packages: %s" (OpamPackage.Set.to_string updated_packages);
  log "changed-packages: %s" (OpamPackage.Set.to_string changed_packages);

  let deleted_packages =
    OpamPackage.Set.filter (fun nv ->
        not (OpamPackage.Map.mem nv repo_index         (* available *)
             || OpamPackage.Set.mem nv all_installed)  (* OR installed *)
      ) t.packages in
  log "deleted-packages: %s" (OpamPackage.Set.to_string deleted_packages);

  (* Remove the deleted packages *)
  OpamPackage.Set.iter (fun nv ->
      OpamFilename.rmdir  (OpamPath.packages t.root nv);
      OpamFilename.remove (OpamPath.archive t.root nv);
    ) deleted_packages;

  (* Update the package descriptions *)
  let (++) = OpamPackage.Set.union in
  OpamPackage.Set.iter (fun nv ->
      match OpamState.repository_and_prefix_of_package t nv with
      | None                -> ()
      | Some (repo, prefix) ->
        let root = OpamPath.Repository.packages repo prefix nv in
        let files = OpamRepository.package_files repo prefix nv ~archive:false in
        let dir = OpamPath.packages t.root nv in
        if OpamFilename.exists_dir dir then OpamFilename.rmdir dir;
        OpamFilename.mkdir dir;
        List.iter (fun file ->
            OpamFilename.copy_in ~root file dir
          ) files;
        OpamFilename.remove (OpamPath.archive t.root nv);
    ) (new_packages ++ updated_packages ++ changed_packages);

  (* that's not a good idea *at all* to enable this hook if you
           are not in a testing environment *)
  OpamPackage.Map.iter (fun nv _ ->
      if !OpamGlobals.sync_archives then (<commit_msg>[update] do not display non-installed pinned packages as a DELETED package<commit_after>(**************************************************************************)
(*                                                                        *)
(*    Copyright 2012-2013 OCamlPro                                        *)
(*    Copyright 2012 INRIA                                                *)
(*                                                                        *)
(*  All rights reserved.This file is distributed under the terms of the   *)
(*  GNU Lesser General Public License version 3.0 with linking            *)
(*  exception.                                                            *)
(*                                                                        *)
(*  OPAM is distributed in the hope that it will be useful, but WITHOUT   *)
(*  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY    *)
(*  or FITNESS FOR A PARTICULAR PURPOSE.See the GNU General Public        *)
(*  License for more details.                                             *)
(*                                                                        *)
(**************************************************************************)

open OpamTypes
open OpamState.Types

let log fmt = OpamGlobals.log "REPOSITORY" fmt

let print_updated_compilers updates =

  let print singular plural map =
    if not (OpamCompiler.Set.is_empty map) then (
      if OpamCompiler.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamCompiler.Set.iter (fun comp ->
        OpamGlobals.msg " - %s\n" (OpamCompiler.to_string comp)
      ) map
    ) in

  print
    "The following NEW compiler is available"
    "The following NEW compilers are available"
    updates.created;

  print
    "The following compiler description has been UPDATED"
    "The following compiler descriptions have been UPDATED"
    updates.updated;

  print
    "The following compiler description has been DELETED"
    "The following compiler descriptions are been DELETED"
    updates.deleted

let filter_compiler_checksums cs =
  let keep f = OpamFilename.check_suffix f ".comp" in
  List.filter keep cs

let fix_compiler_descriptions t ~verbose =
  OpamGlobals.msg "Updating %s/ ...\n"
    (OpamFilename.prettify_dir (OpamPath.compilers_dir t.root));
  let global_index = OpamState.compiler_state t in
  let repo_index = OpamState.compiler_repository_state t in
  let niet = String.concat ":" in
  log "global-index: %s" (OpamCompiler.Map.to_string niet global_index);
  log "repo-index  : %s" (OpamCompiler.Map.to_string niet repo_index);

  let updated_compilers, new_compilers =
    let updated_compilers =
      OpamCompiler.Map.fold (fun comp state set ->
          if not (OpamCompiler.Map.mem comp global_index)
          || OpamCompiler.Map.find comp global_index <> state then
            OpamCompiler.Set.add comp set
          else
            set
        ) repo_index OpamCompiler.Set.empty in
    OpamCompiler.Set.partition (OpamState.is_compiler_installed t) updated_compilers in
  log "updated-compilers: %s" (OpamCompiler.Set.to_string updated_compilers);
  log "new-compilers    : %s" (OpamCompiler.Set.to_string new_compilers);

  let deleted_compilers =
    OpamCompiler.Set.fold (fun comp map ->
        if comp = OpamCompiler.system             (* system *)
        || OpamCompiler.Map.mem comp repo_index   (* OR available *)
        || OpamState.is_compiler_installed t comp (* OR installed *)
        then
          map
        else
          OpamCompiler.Set.add comp map
      ) t.compilers OpamCompiler.Set.empty in
  log "deleted-compilers: %s" (OpamCompiler.Set.to_string deleted_compilers);

  (* Delete compiler descritions *)
  OpamCompiler.Set.iter (fun comp ->
      let dir = OpamPath.compilers t.root comp in
      OpamFilename.rmdir dir;
    ) deleted_compilers;

  (* Update the compiler description *)
  OpamCompiler.Set.iter (fun comp ->
      match OpamState.repository_and_prefix_of_compiler t comp with
      | None                -> ()
      | Some (repo, prefix) ->
        let files = OpamRepository.compiler_files repo prefix comp in
        let dir = OpamPath.compilers t.root comp in
        if OpamFilename.exists_dir dir then OpamFilename.rmdir dir;
        OpamFilename.mkdir dir;
        List.iter (fun file ->
            OpamFilename.copy_in file dir
          ) files;
    ) (OpamCompiler.Set.union updated_compilers new_compilers);

  let updates = {
    created = new_compilers;
    updated = updated_compilers;
    deleted = deleted_compilers;
    changed = OpamCompiler.Set.empty; (* we don't reinstall compilers yet *)
  } in

  if verbose then print_updated_compilers updates;

  updates

let print_updated_packages t updates =

  let print singular plural map fn =
    if not (OpamPackage.Set.is_empty map) then (
      if OpamPackage.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamPackage.Set.iter (fun nv ->
        match fn nv with
        | None   -> OpamGlobals.msg " - %s\n" (OpamPackage.to_string nv)
        | Some s -> OpamGlobals.msg " - %s [%s]\n" (OpamPackage.to_string nv) s
      ) map
    ) in

  let installed_switches nv =
    let installed = OpamState.installed_versions t (OpamPackage.name nv) in
    let installed =
      try OpamPackage.Map.find nv installed
      with Not_found -> [] in (* XXX: should never happen *)
    match installed with
    | [] -> None
    | _  -> Some (
        Printf.sprintf "%s (%s)"
          (OpamPackage.Version.to_string (OpamPackage.version nv))
          (OpamMisc.pretty_list (List.map OpamSwitch.to_string installed))
      ) in

  let none _ = None in

  print
    "The following NEW package is available"
    "The following NEW packages are available"
    updates.created
    none;

  print
    "The following package has been CHANGED upstream and needs to be recompiled"
    "The following packages have been CHANGED upstream and need to be recompiled"
    updates.changed
    installed_switches;

  print
    "The following package has been UPDATED upstream"
    "The following packages have been UPDATED upstream"
    updates.updated
    none;

  print
    "The following package has been DELETED"
    "The following packages have been DELETED"
    updates.deleted
    none

let print_updated_dev_packages pinned_packages =
  let print singular plural map =
    if not (OpamPackage.Set.is_empty map) then (
      if OpamPackage.Set.cardinal map = 1 then
        OpamGlobals.msg "%s:\n" singular
      else
        OpamGlobals.msg "%s:\n" plural;
      OpamPackage.Set.iter (fun nv ->
          OpamGlobals.msg " - %s\n" (OpamPackage.to_string nv)
        ) map
    ) in
  print
    "The following DEV package needs to be upgraded"
    "The following DEV packages need to be upgraded"
    pinned_packages

(* Check for updates in pinned packages *)
let update_dev_packages t ~verbose packages =
  log "update-dev-packages updates %s" (OpamPackage.Set.to_string packages);
  let updates = OpamState.update_dev_packages t in
  if verbose then print_updated_dev_packages updates;
  updates

let filter_package_checksums cs =
  let keep f =
    match OpamFilename.Base.to_string (OpamFilename.basename f) with
    | "opam" | "descr" -> false
    | _ -> true in
  List.filter keep cs

(* Update the package contents, display the new packages and update
   reinstall *)
let fix_package_descriptions t ~verbose =
  OpamGlobals.msg "Updating %s/ ...\n"
    (OpamFilename.prettify_dir (OpamPath.packages_dir t.root));

  let global_index = OpamState.package_state t in
  let repo_index   = OpamState.package_repository_state t in
  let niet = String.concat ":" in
  log "global-index: %s" (OpamPackage.Map.to_string niet global_index);
  log "repo-index  : %s" (OpamPackage.Map.to_string niet repo_index);

  let updated_packages, new_packages =
    let updated_packages =
      OpamPackage.Map.fold (fun nv state set ->
          if not (OpamPackage.Map.mem nv global_index)
          || OpamPackage.Map.find nv global_index <> state then
            OpamPackage.Set.add nv set
          else
            set
        ) repo_index OpamPackage.Set.empty in
    OpamPackage.Set.partition (fun nv ->
        OpamPackage.Map.mem nv global_index
      ) updated_packages in
  log "new-packages    : %s" (OpamPackage.Set.to_string new_packages);

  let all_installed = OpamState.all_installed t in
  let changed_packages, updated_packages =
    OpamPackage.Set.partition (fun nv ->
        OpamPackage.Set.mem nv all_installed
      ) updated_packages in
  log "updated-packages: %s" (OpamPackage.Set.to_string updated_packages);
  log "changed-packages: %s" (OpamPackage.Set.to_string changed_packages);

  let deleted_packages =
    OpamPackage.Set.filter (fun nv ->
        not (OpamPackage.is_pinned nv                         (* pinned*)
             || OpamPackage.Map.mem nv repo_index      (* OR available *)
             || OpamPackage.Set.mem nv all_installed)  (* OR installed *)
      ) t.packages in
  log "deleted-packages: %s" (OpamPackage.Set.to_string deleted_packages);

  (* Remove the deleted packages *)
  OpamPackage.Set.iter (fun nv ->
      OpamFilename.rmdir  (OpamPath.packages t.root nv);
      OpamFilename.remove (OpamPath.archive t.root nv);
    ) deleted_packages;

  (* Update the package descriptions *)
  let (++) = OpamPackage.Set.union in
  OpamPackage.Set.iter (fun nv ->
      match OpamState.repository_and_prefix_of_package t nv with
      | None                -> ()
      | Some (repo, prefix) ->
        let root = OpamPath.Repository.packages repo prefix nv in
        let files = OpamRepository.package_files repo prefix nv ~archive:false in
        let dir = OpamPath.packages t.root nv in
        if OpamFilename.exists_dir dir then OpamFilename.rmdir dir;
        OpamFilename.mkdir dir;
        List.iter (fun file ->
            OpamFilename.copy_in ~root file dir
          ) files;
        OpamFilename.remove (OpamPath.archive t.root nv);
    ) (new_packages ++ updated_packages ++ changed_packages);

  (* that's not a good idea *at all* to enable this hook if you
           are not in a testing environment *)
  OpamPackage.Map.iter (fun nv _ ->
      if !OpamGlobals.sync_archives then (
{'fraction_non_alphanumeric': 0.09503491078355314, 'fraction_numerical': 0.0016485647788983709, 'mean_word_length': 2.4130398808538804, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '652698', 'n_tokens_mistral': 5970, 'n_tokens_neox': 5829, 'n_words': 1756}