<commit_before>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

open Base
open Utils
module Let_syntax = Lwt.Infix.Let_syntax

(*
 * Module for us to interface with Watchman, a file watching service.
 * https://facebook.github.io/watchman/
 *
 * TODO:
 *   * Connect directly to the Watchman server socket instead of spawning
 *     a client process each time
 *   * Use the BSER protocol for enhanced performance
 *)

exception Timeout

exception Watchman_error of string

exception Subscription_canceled_by_watchman

exception Watchman_restarted

type subscribe_mode =
  | All_changes
  | Defer_changes
  | Drop_changes
      (** See also Watchman docs on drop. This means the subscriber will not
          get a list of files changed during a repo update. Practically, this
          is not useful for the typechecker process which needs to actually
          know which files were changed. This is useful for the monitor to
          aggressively kill the server. *)

type timeout = float option

type init_settings = {
  debug_logging: bool;
  defer_states: string list;
  expression_terms: Hh_json.json list;  (** See watchman expression terms. *)
  mergebase_with: string;
  roots: Path.t list;  (** symbolic commit to find changes against *)
  subscribe_mode: subscribe_mode;
  subscription_prefix: string;
  sync_timeout: int option;
}

(** The message's clock. *)
type clock = string

type pushed_changes =
  (*
   * State name and metadata.
   *
   * For example:
   *   State name: "hg.update"
   * Metadata:
   *   {
   *    "partial":false,
   *    "rev":"780dab9ff0a01691c9b18a5ee1194810e555c78b",
   *    "distance":2,
   *    "status":"ok"
   *   }
   *
   * Note: The distance is HG Revision distance, not SVN revision distance.
   *)
  | State_enter of string * Hh_json.json option
  | State_leave of string * Hh_json.json option
  | Changed_merge_base of string * SSet.t * clock
  | Files_changed of SSet.t

type changes =
  | Watchman_unavailable
  | Watchman_pushed of pushed_changes

module Jget = Hh_json_helpers.Jget
module J = Hh_json_helpers.AdhocJsonHelpers

(* Looks for common errors in watchman responses *)
let assert_no_error obj =
  let obj = Some obj in
  (match Jget.string_opt obj "warning" with
  | None -> ()
  | Some warning ->
    EventLogger.watchman_warning warning;
    Hh_logger.log "Watchman warning: %s\n" warning);
  (match Jget.string_opt obj "error" with
  | None -> ()
  | Some error ->
    EventLogger.watchman_error error;
    raise @@ Watchman_error error);
  (match Jget.bool_opt obj "canceled" with
  | None
  | Some false ->
    ()
  | Some true ->
    EventLogger.watchman_error "Subscription canceled by watchman";
    raise @@ Subscription_canceled_by_watchman);
  ()

(* Verifies that a watchman response is valid JSON and free from common errors *)
let sanitize_watchman_response ~debug_logging output =
  if debug_logging then Hh_logger.info "Watchman response: %s" output;
  let response =
    try Hh_json.json_of_string output
    with e ->
      let exn = Exception.wrap e in
      Hh_logger.error
        "Failed to parse string as JSON: %s\nEXCEPTION:%s\nSTACK:%s\n"
        output
        (Exception.get_ctor_string exn)
        (Exception.get_backtrace_string exn);
      Exception.reraise exn<commit_msg>[watchman] log parse errors<commit_after>(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

open Base
open Utils
module Let_syntax = Lwt.Infix.Let_syntax

(*
 * Module for us to interface with Watchman, a file watching service.
 * https://facebook.github.io/watchman/
 *
 * TODO:
 *   * Connect directly to the Watchman server socket instead of spawning
 *     a client process each time
 *   * Use the BSER protocol for enhanced performance
 *)

exception Timeout

exception Watchman_error of string

exception Subscription_canceled_by_watchman

exception Watchman_restarted

type subscribe_mode =
  | All_changes
  | Defer_changes
  | Drop_changes
      (** See also Watchman docs on drop. This means the subscriber will not
          get a list of files changed during a repo update. Practically, this
          is not useful for the typechecker process which needs to actually
          know which files were changed. This is useful for the monitor to
          aggressively kill the server. *)

type timeout = float option

type init_settings = {
  debug_logging: bool;
  defer_states: string list;
  expression_terms: Hh_json.json list;  (** See watchman expression terms. *)
  mergebase_with: string;
  roots: Path.t list;  (** symbolic commit to find changes against *)
  subscribe_mode: subscribe_mode;
  subscription_prefix: string;
  sync_timeout: int option;
}

(** The message's clock. *)
type clock = string

type pushed_changes =
  (*
   * State name and metadata.
   *
   * For example:
   *   State name: "hg.update"
   * Metadata:
   *   {
   *    "partial":false,
   *    "rev":"780dab9ff0a01691c9b18a5ee1194810e555c78b",
   *    "distance":2,
   *    "status":"ok"
   *   }
   *
   * Note: The distance is HG Revision distance, not SVN revision distance.
   *)
  | State_enter of string * Hh_json.json option
  | State_leave of string * Hh_json.json option
  | Changed_merge_base of string * SSet.t * clock
  | Files_changed of SSet.t

type changes =
  | Watchman_unavailable
  | Watchman_pushed of pushed_changes

module Jget = Hh_json_helpers.Jget
module J = Hh_json_helpers.AdhocJsonHelpers

(* Looks for common errors in watchman responses *)
let assert_no_error obj =
  let obj = Some obj in
  (match Jget.string_opt obj "warning" with
  | None -> ()
  | Some warning ->
    EventLogger.watchman_warning warning;
    Hh_logger.log "Watchman warning: %s\n" warning);
  (match Jget.string_opt obj "error" with
  | None -> ()
  | Some error ->
    EventLogger.watchman_error error;
    raise @@ Watchman_error error);
  (match Jget.bool_opt obj "canceled" with
  | None
  | Some false ->
    ()
  | Some true ->
    EventLogger.watchman_error "Subscription canceled by watchman";
    raise @@ Subscription_canceled_by_watchman);
  ()

(* Verifies that a watchman response is valid JSON and free from common errors *)
let sanitize_watchman_response ~debug_logging output =
  if debug_logging then Hh_logger.info "Watchman response: %s" output;
  let response =
    try Hh_json.json_of_string output
    with e ->
      let exn = Exception.wrap e in
      let msg = spf "Failed to parse string as JSON: %s" output in
      EventLogger.watchman_error
        (spf
           "%s\nEXCEPTION:%s\nSTACK:%s\n"
           msg
           (Exception.get_ctor_string exn)
           (Exception.get_backtrace_string exn));
      Hh_logger.error ~exn "%s" msg;
      Exception.reraise exn
{'fraction_non_alphanumeric': 0.08057553956834532, 'fraction_numerical': 0.00776978417266187, 'mean_word_length': 3.2670349907918967, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4609331', 'n_tokens_mistral': 2179, 'n_tokens_neox': 2132, 'n_words': 711}