<commit_before>;; ChatLang DSL for chat authoring rules
;;
;; A partial implementation of the top level translator that produces
;; PSI rules.
(use-modules (opencog)
             (opencog nlp)
             (opencog exec)
             (opencog openpsi)
             (opencog eva-behavior)
             (srfi srfi-1))

;; Shared variables for all terms
(define atomese-variable-template (list (TypedVariable (Variable "$S")
                                                       (Type "SentenceNode"))
                                        (TypedVariable (Variable "$P")
                                                       (Type "ParseNode"))))

;; Shared conditions for all terms
(define atomese-condition-template (list (Parse (Variable "$P")
                                                (Variable "$S"))
                                         (State (Anchor "Currently Processing")
                                                (Variable "$S"))))

(define (process-pattern-term term atomese-pattern)
  "Process a single term -- calls the term function and appends the new
   variables and conditions to the existing pair."
  (let* ((atomese-for-term (primitive-eval term))
         (vars (append (car atomese-pattern) (car atomese-for-term)))
         (conds (append (cdr atomese-pattern) (cdr atomese-for-term))))
    (cons vars conds)))

; XXX TODO FIXME
; This is a temporary hack for testing purpose
; Will think more on this and update it accordingly
(define (term-sequence-check terms)
  "Checks terms occur in the desired order. To be implemented."
  (define word-list
    (map (lambda (w) (cond ((equal? 'concept (car w)) (Glob (car (cdr w))))
                           (else (Word (car (cdr w))))))
         terms))
  (Evaluation (GroundedPredicate "scm: check-word-sequence")
              (List (Variable "$S")
                    (List word-list))))

(define (get-word-lemma sent-node target-link-type)
  (List (append-map
    (lambda (w)
      ; Ignore LEFT-WALL and punctuations<commit_msg>Provide a way to get the lemma of a word during the conversion<commit_after>;; ChatLang DSL for chat authoring rules
;;
;; A partial implementation of the top level translator that produces
;; PSI rules.
(use-modules (opencog)
             (opencog nlp)
             (opencog exec)
             (opencog openpsi)
             (opencog eva-behavior)
             (srfi srfi-1))

;; Shared variables for all terms
(define atomese-variable-template (list (TypedVariable (Variable "$S")
                                                       (Type "SentenceNode"))
                                        (TypedVariable (Variable "$P")
                                                       (Type "ParseNode"))))

;; Shared conditions for all terms
(define atomese-condition-template (list (Parse (Variable "$P")
                                                (Variable "$S"))
                                         (State (Anchor "Currently Processing")
                                                (Variable "$S"))))

(define (process-pattern-term term atomese-pattern)
  "Process a single term -- calls the term function and appends the new
   variables and conditions to the existing pair."
  (let* ((atomese-for-term (primitive-eval term))
         (vars (append (car atomese-pattern) (car atomese-for-term)))
         (conds (append (cdr atomese-pattern) (cdr atomese-for-term))))
    (cons vars conds)))

(define (term-sequence-check terms)
  "Checks terms occur in the desired order. To be implemented."
  ; A hacky way to quickly find the lemma of a word using WordNet
  (define (get-lemma word)
    (let* ((cmd-string (string-append "wn " word " | grep \"Information available for .\\+\""))
           (port (open-input-pipe cmd-string))
           (lemma ""))
      (do ((line (get-line port) (get-line port)))
          ((eof-object? line))
        (let ((l (car (last-pair (string-split line #\ )))))
          (if (not (equal? word l))
            (set! lemma l))))
      (close-pipe port)
      (if (string-null? lemma) word lemma)))

  (define word-list
    (map (lambda (w) (cond ((equal? 'concept (car w)) (Glob (car (cdr w))))
                           (else (Word (car (cdr w))))))
         terms))
  (Evaluation (GroundedPredicate "scm: check-word-sequence")
              (List (Variable "$S")
                    (List word-list))))

(define (get-word-lemma sent-node target-link-type)
  (List (append-map
    (lambda (w)
      ; Ignore LEFT-WALL and punctuations
{'fraction_non_alphanumeric': 0.11737297537164411, 'fraction_numerical': 0.00044375416019525185, 'mean_word_length': 1.8495575221238938, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1768022', 'n_tokens_mistral': 1195, 'n_tokens_neox': 1166, 'n_words': 417}