<commit_before>;;; rcas-web - Web interface for RCAS
;;; Copyright © 2016  <NAME> <<EMAIL>>
;;;
;;; This program is free software: you can redistribute it and/or
;;; modify it under the terms of the GNU Affero General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;;; Affero General Public License for more details.
;;;
;;; You should have received a copy of the GNU Affero General Public
;;; License along with this program.  If not, see
;;; <http://www.gnu.org/licenses/>.

(define-module (rcas-web jobs)
  #:use-module (redis)
  #:export (enqueue
            done?
            process-next
            processing-num
            waiting-num))

(define (flatten lst)
  (cond ((null? lst) '())
        ((pair? lst)
         ((@@ (guile) append) (flatten (car lst))
                 (flatten (cdr lst))))
        (else (list lst))))

;; Take care of opening and closing a connection to redis.
(define-syntax with-redis
  (lambda (x)
    (syntax-case x ()
      ((_ exp ...)
       (with-syntax (($conn (datum->syntax x '$conn)))
         #'(let (($conn (redis-connect)))
             (dynamic-wind
               (lambda () #t)
               (lambda () (redis-send $conn
                                      (flatten (list exp ...))))
               (lambda () (redis-close $conn)))))))))

(define-syntax-rule (transaction exps ...)
  (list (multi)
        exps ...
        (exec)))

(define %prefix     "rcas-web:")
(define %waiting    (string-append %prefix "waiting"))
(define %processing (string-append %prefix "processing"))

(define (enqueue raw-file-name options)
  "Append the basename of RAW-FILE-NAME to the waiting queue and store
the OPTIONS."
  (let ((file-name (basename raw-file-name)))
    (with-redis
     (transaction
      (rpush %waiting (list file-name))
      (set (string-append %prefix ":options:" file-name)
           (format #f "~a" options))))))

(define (done? filename)
  "Return the processing result if the FILENAME has been processed or
#f if it has not."
  (let ((result
         (car (with-redis (get (string-append %prefix filename))))))<commit_msg>rcas-web: filename --> file-name.<commit_after>;;; rcas-web - Web interface for RCAS
;;; Copyright © 2016  <NAME> <<EMAIL>>
;;;
;;; This program is free software: you can redistribute it and/or
;;; modify it under the terms of the GNU Affero General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
;;; Affero General Public License for more details.
;;;
;;; You should have received a copy of the GNU Affero General Public
;;; License along with this program.  If not, see
;;; <http://www.gnu.org/licenses/>.

(define-module (rcas-web jobs)
  #:use-module (redis)
  #:export (enqueue
            done?
            process-next
            processing-num
            waiting-num))

(define (flatten lst)
  (cond ((null? lst) '())
        ((pair? lst)
         ((@@ (guile) append) (flatten (car lst))
                 (flatten (cdr lst))))
        (else (list lst))))

;; Take care of opening and closing a connection to redis.
(define-syntax with-redis
  (lambda (x)
    (syntax-case x ()
      ((_ exp ...)
       (with-syntax (($conn (datum->syntax x '$conn)))
         #'(let (($conn (redis-connect)))
             (dynamic-wind
               (lambda () #t)
               (lambda () (redis-send $conn
                                      (flatten (list exp ...))))
               (lambda () (redis-close $conn)))))))))

(define-syntax-rule (transaction exps ...)
  (list (multi)
        exps ...
        (exec)))

(define %prefix     "rcas-web:")
(define %waiting    (string-append %prefix "waiting"))
(define %processing (string-append %prefix "processing"))

(define (enqueue raw-file-name options)
  "Append the basename of RAW-FILE-NAME to the waiting queue and store
the OPTIONS."
  (let ((file-name (basename raw-file-name)))
    (with-redis
     (transaction
      (rpush %waiting (list file-name))
      (set (string-append %prefix ":options:" file-name)
           (format #f "~a" options))))))

(define (done? file-name)
  "Return the processing result if the FILE-NAME has been processed or
#f if it has not."
  (let ((result
         (car (with-redis (get (string-append %prefix file-name))))))
{'fraction_non_alphanumeric': 0.14276795005202914, 'fraction_numerical': 0.002081165452653486, 'mean_word_length': 2.6299093655589125, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4466244', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1406, 'n_words': 527}