<commit_before>;
; mst-bench.scm
;
; A sloppy, cheesey, quick-n-dirty tool to measure MST performance.
; <NAME> August 2017
;

(use-modules (opencog) (opencog sheaf))

; Step 1: create a bunch of random pair data.

; Create MPAIRS different random pairs, connecting NVERTS different
; vertexes.
(define (make-words MPAIRS NVERTS)
	(define (mkvert)
		(Concept (format #f "vertex ~D" (random NVERTS))))

	(define (mkpair)
		(Evaluation (Predicate "benchy") (List (mkvert) (mkvert))))

)<commit_msg>Done creating data<commit_after>;
; mst-bench.scm
;
; A sloppy, cheesey, quick-n-dirty tool to measure MST performance.
; <NAME> August 2017
;

(use-modules (opencog) (opencog sheaf))

; Step 1: create a bunch of random pair data.

(define mi-pred (Predicate "Faux MI Key"))

; Create MPAIRS different random pairs, connecting NVERTS different
; vertexes.
(define (make-words MPAIRS NVERTS)
	(define (mkvert)
		(Concept (format #f "vertex ~D" (random NVERTS))))

	(define (mkpair)
		(Evaluation (Predicate "benchy") (List (mkvert) (mkvert))))

	; Use a Gaussian distribution, centerd around 2, having a standard
	; deviation of about 5. This is close to reality for English
	; word-pairs, I think.
	(define (mkmi) (+ 4 (* 5 (random:normal))))

	; Create a random pair, and assign a random weight to it.
	(define (mkwpair)
		(cog-set-value! (mkpair) mi-pred (FloatValue (mkmi))))

	;; Loop M times.
	(define (mkmprs M)
		(if (< 0 M) (begin (mkwpair) (mkmprs (- M 1)))))

	(mkmprs MPAIRS)
)
{'fraction_non_alphanumeric': 0.13814016172506738, 'fraction_numerical': 0.01078167115902965, 'mean_word_length': 4.341726618705036, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2467082', 'n_tokens_mistral': 574, 'n_tokens_neox': 528, 'n_words': 185}