<commit_before>scm
;
; wsd-process.scm
; Work through a bunch of RelEx files that are in the compact-file-format,
; and import these into opencog. Then WSD run processing on each one, as
; needed. This is a batch-processing script.
;
; Copyright (C) 2008 <NAME> <<EMAIL>>
;
(use-modules (ice-9 rdelim))
(use-modules (ice-9 popen))

; ---------------------------------------------------------------------
; Perform WSD-disjunct statistics processing.
;
; Process 'num-to-do' files in the input dir, and move them to done-dir
; The input files are assumed to be RelEx "compact-file format" files.
; These are converted into opencog data, and are loaded into opencog.
; At this point WSD should run automatically; this script then triggers
; disjunct processing manually. When this completes, the next file is
; processed ... 
;
(define (wsd-process-data input-dir done-dir num-to-do)

	(define (process-file filename)
		(let ((fullname (string-join (list input-dir filename) "/"))
		      (donename (string-join (list done-dir filename) "/"))
			)
			(system (string-join (list "echo \"" (string-join 
				(map (lambda (x) (string-join (list (object->string x) "\n"))) 
				(gc-stats))) "\"")))
			(system (string-join (list "echo \"" (string-join 
				(map (lambda (x) (string-join (list (object->string x) "\n"))) 
				(cog-report-counts))) "\"")))
			(system (string-join (list "echo start article: \"" filename "\"")))
			(system "date")
			(load-cff-data fullname)
			(system (string-join (list "echo done cff: \"" filename "\"")))
			(system "date")
			(cog-ad-hoc "do-wsd")
			(system (string-join (list "echo done wsd: \"" filename "\"")))
			(system "date")
			(ldj-process)
			(system (string-join (list "echo done ldj: \"" filename "\"")))
			(system "date")
			(system (string-join (list "mv \"" fullname "\" \"" donename "\"") ""))
			(delete-sentences)
			(system (string-join (list "echo done delete: \"" filename "\"")))
			(system "date")
		)
	)
	
	(for-each process-file
		(take (list-files input-dir) num-to-do)
	)
)
<commit_msg>Add progress count printing<commit_after>scm
;
; wsd-process.scm
; Work through a bunch of RelEx files that are in the compact-file-format,
; and import these into opencog. Then WSD run processing on each one, as
; needed. This is a batch-processing script.
;
; Copyright (C) 2008 <NAME> <<EMAIL>>
;
(use-modules (ice-9 rdelim))
(use-modules (ice-9 popen))

; ---------------------------------------------------------------------
; Perform WSD-disjunct statistics processing.
;
; Process 'num-to-do' files in the input dir, and move them to done-dir
; The input files are assumed to be RelEx "compact-file format" files.
; These are converted into opencog data, and are loaded into opencog.
; At this point WSD should run automatically; this script then triggers
; disjunct processing manually. When this completes, the next file is
; processed ... 
;
(define (wsd-process-data input-dir done-dir num-to-do)
	(define cnt 0)

	(define (process-file filename)
		(let ((fullname (string-join (list input-dir filename) "/"))
		      (donename (string-join (list done-dir filename) "/"))
			)
			(system (string-join (list "echo \"" (string-join 
				(map (lambda (x) (string-join (list (object->string x) "\n"))) 
				(gc-stats))) "\"")))
			(system (string-join (list "echo \"" (string-join 
				(map (lambda (x) (string-join (list (object->string x) "\n"))) 
				(cog-report-counts))) "\"")))
			(system (string-join (list "echo start article "
				(object->string cnt)
				": \"" filename "\""))
			)
			(system "date")
			(load-cff-data fullname)
			(system (string-join (list "echo done cff: \"" filename "\"")))
			(system "date")
			(cog-ad-hoc "do-wsd")
			(system (string-join (list "echo done wsd: \"" filename "\"")))
			(system "date")
			(ldj-process)
			(system (string-join (list "echo done ldj: \"" filename "\"")))
			(system "date")
			(system (string-join (list "mv \"" fullname "\" \"" donename "\"") ""))
			(delete-sentences)
			(system (string-join (list "echo done delete: \"" filename "\"")))
			(system "date")
			(set! cnt (+ cnt 1))
		)
	)
	
	(for-each process-file
		(take (list-files input-dir) num-to-do)
	)
)

{'fraction_non_alphanumeric': 0.20033468802295004, 'fraction_numerical': 0.0033468802295003584, 'mean_word_length': 2.950897072710104, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3395320', 'n_tokens_mistral': 1523, 'n_tokens_neox': 1382, 'n_words': 440}