<commit_before>;;; Copyright 2021 Google LLC
;;;
;;; Use of this source code is governed by an MIT-style
;;; license that can be found in the LICENSE file or at
;;; https://opensource.org/licenses/MIT.

;;;; Public Interface for gRPC

(in-package #:grpc)

(defvar *completion-queue* nil "The global completion queue used to
manage grpc calls.")

(define-condition grpc-call-error (error)
  ((call-error :initarg :call-error
               :initform nil
               :accessor call-error))
  (:report (lambda (condition stream)
             (format stream "GRPC CALL ERROR: ~A.~&" (call-error condition)))))

;; Functions for gRPC Client

(defun create-channel (target &optional (creds (cffi:null-pointer)) (args (cffi:null-pointer)))
  "A wrapper to create a channel for the client to TARGET with
additional args ARGS for client information. If CREDS is passed then a secure
channel will be created using CREDS else an insecure channel will be used."
  (c-grpc-client-new-channel creds target args))

(defun service-method-call (channel call-name cq)
  "A wrapper to create a grpc_call pointer that will be used to call CALL-NAME
on the CHANNEL provided and store the result in the completion queue CQ."
  (cffi:foreign-funcall "lisp_grpc_channel_create_call"
                        :pointer channel :string call-name :pointer cq
                        :pointer))

(cffi:defcfun ("lisp_grpc_call_start_batch" call-start-batch )
  grpc-call-error
  (call :pointer)
  (ops :pointer)
  (num-ops :int)
  (tag :pointer))

(cffi:defcfun ("lisp_grpc_completion_queue_pluck" completion-queue-pluck )
  :bool
  (completion-queue :pointer)
  (tag :pointer))

;; Wrappers to create operations

(defun make-send-metadata-op (op metadata
                              &key count flag
                                index)
  "Sets OP[INDEX] to a Send Initial Metadata operation by adding metadata
METADATA, the count of metadata COUNT, and the flag FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_send_metadata_op"
                        :pointer op
                        :int index
                        :pointer metadata
                        :int count
                        :int (convert-metadata-flag-to-integer flag)
                        :void))

(defun make-send-message-op (op message &key index)
  "Sets OP[INDEX] to a 'Send Message' operation that sends MESSAGE
to the server."
  (cffi:foreign-funcall "lisp_grpc_make_send_message_op"
                        :pointer op
                        :int index
                        :pointer message
                        :void))

(defun make-client-recv-status-op (op &key flag index)
  "Sets OP[INDEX] to a 'RECEIVE STATUS' operation, sets the FLAG of the op."
  (cffi:foreign-funcall "lisp_grpc_client_make_recv_status_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-recv-message-op (op &key flag index)
  "Sets OP[INDEX] to a Receive Message operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_recv_message_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-recv-metadata-op (op &key index)
  "Set OP[INDEX] to a Receive Initial Metadata operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_recv_metadata_op"
                        :pointer op
                        :int index
                        :void))

(defun make-client-close-op (op &key flag index)
  "Sets OP[INDEX] to a Send Close From Client operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_client_make_close_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-send-status-from-server-op (op &key metadata count status flag index)
  "Sets OP[INDEX] to a Send Status from server operation by adding metadata
METADATA, the server STATUS, the count of metadata COUNT, and the flag FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_send_status_from_server_op"
                        :pointer op
                        :int index
                        :pointer metadata
                        :int count
                        grpc-status-code status
                        :int flag
                        :void))

(defun make-recv-close-on-server-op (op &key cancelled flag index)
  "Sets OP[INDEX] to a Receive Close on Server operation by adding cancelled CANCELLED
and the flag FLAG"
  (cffi:foreign-funcall "lisp_grpc_server_make_close_op"
                        :pointer op
                        :int index
                        :pointer cancelled
                        :int flag
                        :void))

(defun prepare-ops (ops message
                    &key
                      send-metadata send-message client-close
                      client-recv-status recv-metadata
                      recv-message server-recv-close server-send-status)
  "Prepares OPS to send MESSAGE to the server. The keys SEND-METADATA
SEND-MESSAGE CLIENT-CLOSE CLIENT-RECV-STATUS RECV-METADATA RECV-MESSAGE
SERVER-RECV-CLOSE SERVER-SEND-STATUS are all different types of ops that the user may
want. Returns a plist containing keys being the op type and values being the index."
  (let ((cur-index -1)
        ops-plist)
    (flet ((next-marker (message-type)
             (setf (getf ops-plist message-type) (incf cur-index))))

      (when send-metadata
        (make-send-metadata-op ops (cffi:null-pointer)
                               :count 0 :flag 0 :index (next-marker :send-metadata)))
      (when send-message
        (make-send-message-op ops message :index (next-marker :send-message)))
      (when client-close
        (make-client-close-op ops :flag 0 :index (next-marker :client-close)))
      (when client-recv-status
        (make-client-recv-status-op ops :flag 0 :index (next-marker :client-recv-status)))
      (when recv-metadata
        (make-recv-metadata-op ops :index (next-marker :recv-metadata)))
      (when recv-message
        (make-recv-message-op ops :flag 0 :index (next-marker :recv-message)))
      (when server-recv-close
        (make-recv-close-on-server-op ops :cancelled (cffi:foreign-alloc :int) :flag 0 :index (next-marker :server-close)))
      (when server-send-status
        (make-send-status-from-server-op ops :metadata (cffi:null-pointer) :count 0 :status server-send-status :flag 0 :index (next-marker :server-send-status))))
    ops-plist))

(cffi:defcfun ("lisp_grpc_op_recv_message" get-grpc-op-recv-message ) :pointer
  (op :pointer)
  (index :int))

;; Auxiliary Functions

(cffi:defcfun ("create_new_grpc_ops" create-new-grpc-ops) :pointer
  "Creates a grpc_op* that is used to add NUM-OPS operations to,
these operation guide the interaction between the client and server."
  (num-ops :int))

(defun convert-grpc-slice-to-grpc-byte-buffer (slice)
  "Takes a grpc_slice* SLICE and returns a pointer to the corresponding
grpc_byte_buffer*."
  (cffi:foreign-funcall "convert_grpc_slice_to_grpc_byte_buffer"
                        :pointer slice
                        :pointer))

(defun convert-bytes-to-grpc-slice (bytes)
  "Takes a list of bytes BYTES and returns a pointer to the corresponding
grpc_slice*."
  (let ((array (cffi:foreign-alloc :char :initial-contents bytes)))
    (cffi:foreign-funcall "convert_bytes_to_grpc_slice"
                          :pointer array
                          :size (length bytes)
                          :pointer)))

(defun convert-grpc-slice-to-bytes (slice)
  "Takes SLICE and returns its content as a vector of bytes."
  (let* ((slice-string-pointer (cffi:foreign-funcall
                                "convert_grpc_slice_to_string" :pointer slice
                                :pointer)))
    (cffi:foreign-array-to-lisp slice-string-pointer
                                (list :array :uint8
                                      (cffi:foreign-funcall
                                           "strlen"
                                           :pointer slice-string-pointer :int)))))

;; Exported Functions

(defun init-grpc ()
  "Initializes the grpc library and the global *completion-queue* so that
grpc functions can be used and the queue can be managed. Call before any gRPC<commit_msg>Add :size defctype so grpc can compile<commit_after>;;; Copyright 2021 Google LLC
;;;
;;; Use of this source code is governed by an MIT-style
;;; license that can be found in the LICENSE file or at
;;; https://opensource.org/licenses/MIT.

;;;; Public Interface for gRPC

(in-package #:grpc)

(defvar *completion-queue* nil "The global completion queue used to
manage grpc calls.")

(define-condition grpc-call-error (error)
  ((call-error :initarg :call-error
               :initform nil
               :accessor call-error))
  (:report (lambda (condition stream)
             (format stream "GRPC CALL ERROR: ~A.~&" (call-error condition)))))

;; Functions for gRPC Client

(defun create-channel (target &optional (creds (cffi:null-pointer)) (args (cffi:null-pointer)))
  "A wrapper to create a channel for the client to TARGET with
additional args ARGS for client information. If CREDS is passed then a secure
channel will be created using CREDS else an insecure channel will be used."
  (c-grpc-client-new-channel creds target args))

(defun service-method-call (channel call-name cq)
  "A wrapper to create a grpc_call pointer that will be used to call CALL-NAME
on the CHANNEL provided and store the result in the completion queue CQ."
  (cffi:foreign-funcall "lisp_grpc_channel_create_call"
                        :pointer channel :string call-name :pointer cq
                        :pointer))

(cffi:defcfun ("lisp_grpc_call_start_batch" call-start-batch )
  grpc-call-error
  (call :pointer)
  (ops :pointer)
  (num-ops :int)
  (tag :pointer))

(cffi:defcfun ("lisp_grpc_completion_queue_pluck" completion-queue-pluck )
  :bool
  (completion-queue :pointer)
  (tag :pointer))

;; Wrappers to create operations

(defun make-send-metadata-op (op metadata
                              &key count flag
                                index)
  "Sets OP[INDEX] to a Send Initial Metadata operation by adding metadata
METADATA, the count of metadata COUNT, and the flag FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_send_metadata_op"
                        :pointer op
                        :int index
                        :pointer metadata
                        :int count
                        :int (convert-metadata-flag-to-integer flag)
                        :void))

(defun make-send-message-op (op message &key index)
  "Sets OP[INDEX] to a 'Send Message' operation that sends MESSAGE
to the server."
  (cffi:foreign-funcall "lisp_grpc_make_send_message_op"
                        :pointer op
                        :int index
                        :pointer message
                        :void))

(defun make-client-recv-status-op (op &key flag index)
  "Sets OP[INDEX] to a 'RECEIVE STATUS' operation, sets the FLAG of the op."
  (cffi:foreign-funcall "lisp_grpc_client_make_recv_status_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-recv-message-op (op &key flag index)
  "Sets OP[INDEX] to a Receive Message operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_recv_message_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-recv-metadata-op (op &key index)
  "Set OP[INDEX] to a Receive Initial Metadata operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_recv_metadata_op"
                        :pointer op
                        :int index
                        :void))

(defun make-client-close-op (op &key flag index)
  "Sets OP[INDEX] to a Send Close From Client operation with FLAG."
  (cffi:foreign-funcall "lisp_grpc_client_make_close_op"
                        :pointer op
                        :int index
                        :int flag
                        :void))

(defun make-send-status-from-server-op (op &key metadata count status flag index)
  "Sets OP[INDEX] to a Send Status from server operation by adding metadata
METADATA, the server STATUS, the count of metadata COUNT, and the flag FLAG."
  (cffi:foreign-funcall "lisp_grpc_make_send_status_from_server_op"
                        :pointer op
                        :int index
                        :pointer metadata
                        :int count
                        grpc-status-code status
                        :int flag
                        :void))

(defun make-recv-close-on-server-op (op &key cancelled flag index)
  "Sets OP[INDEX] to a Receive Close on Server operation by adding cancelled CANCELLED
and the flag FLAG"
  (cffi:foreign-funcall "lisp_grpc_server_make_close_op"
                        :pointer op
                        :int index
                        :pointer cancelled
                        :int flag
                        :void))

(defun prepare-ops (ops message
                    &key
                      send-metadata send-message client-close
                      client-recv-status recv-metadata
                      recv-message server-recv-close server-send-status)
  "Prepares OPS to send MESSAGE to the server. The keys SEND-METADATA
SEND-MESSAGE CLIENT-CLOSE CLIENT-RECV-STATUS RECV-METADATA RECV-MESSAGE
SERVER-RECV-CLOSE SERVER-SEND-STATUS are all different types of ops that the user may
want. Returns a plist containing keys being the op type and values being the index."
  (let ((cur-index -1)
        ops-plist)
    (flet ((next-marker (message-type)
             (setf (getf ops-plist message-type) (incf cur-index))))

      (when send-metadata
        (make-send-metadata-op ops (cffi:null-pointer)
                               :count 0 :flag 0 :index (next-marker :send-metadata)))
      (when send-message
        (make-send-message-op ops message :index (next-marker :send-message)))
      (when client-close
        (make-client-close-op ops :flag 0 :index (next-marker :client-close)))
      (when client-recv-status
        (make-client-recv-status-op ops :flag 0 :index (next-marker :client-recv-status)))
      (when recv-metadata
        (make-recv-metadata-op ops :index (next-marker :recv-metadata)))
      (when recv-message
        (make-recv-message-op ops :flag 0 :index (next-marker :recv-message)))
      (when server-recv-close
        (make-recv-close-on-server-op ops :cancelled (cffi:foreign-alloc :int) :flag 0 :index (next-marker :server-close)))
      (when server-send-status
        (make-send-status-from-server-op ops :metadata (cffi:null-pointer) :count 0 :status server-send-status :flag 0 :index (next-marker :server-send-status))))
    ops-plist))

(cffi:defcfun ("lisp_grpc_op_recv_message" get-grpc-op-recv-message ) :pointer
  (op :pointer)
  (index :int))

;; Auxiliary Functions

(cffi:defcfun ("create_new_grpc_ops" create-new-grpc-ops) :pointer
  "Creates a grpc_op* that is used to add NUM-OPS operations to,
these operation guide the interaction between the client and server."
  (num-ops :int))

(defun convert-grpc-slice-to-grpc-byte-buffer (slice)
  "Takes a grpc_slice* SLICE and returns a pointer to the corresponding
grpc_byte_buffer*."
  (cffi:foreign-funcall "convert_grpc_slice_to_grpc_byte_buffer"
                        :pointer slice
                        :pointer))

;; Hack since :size defctype doesn't work in
;; cffi:foreign-funcall externally
(cffi:defctype :size #+64-bit :uint64 #+32-bit :uint32)

(defun convert-bytes-to-grpc-slice (bytes)
  "Takes a list of bytes BYTES and returns a pointer to the corresponding
grpc_slice*."
  (let ((array (cffi:foreign-alloc :char :initial-contents bytes)))
    (cffi:foreign-funcall "convert_bytes_to_grpc_slice"
                          :pointer array
                          :size (length bytes)
                          :pointer)))

(defun convert-grpc-slice-to-bytes (slice)
  "Takes SLICE and returns its content as a vector of bytes."
  (let* ((slice-string-pointer (cffi:foreign-funcall
                                "convert_grpc_slice_to_string" :pointer slice
                                :pointer)))
    (cffi:foreign-array-to-lisp slice-string-pointer
                                (list :array :uint8
                                      (cffi:foreign-funcall
                                           "strlen"
                                           :pointer slice-string-pointer :int)))))

;; Exported Functions

(defun init-grpc ()
  "Initializes the grpc library and the global *completion-queue* so that
grpc functions can be used and the queue can be managed. Call before any gRPC
{'fraction_non_alphanumeric': 0.09054932073242765, 'fraction_numerical': 0.00212640283520378, 'mean_word_length': 2.0583453757225434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4754730', 'n_tokens_mistral': 4976, 'n_tokens_neox': 4792, 'n_words': 1500}