<commit_before>(ns lambda-blog.md-parser-test
  (:require [clojure.test :refer :all]
            [lambda-blog.parsers.md :refer [parse]]))

(def no-metadata "# Hello world!")
(def metadata "Meta: Test\n\n# Hello world!")
(def no-metadata-no-heading "Test test!")
(def metadata-no-heading "Meta: Test\n\nTest test!")

(deftest can-parse-md
  (is (= (parse no-metadata)
         {:metadata {}
          :contents "<h1>Hello world!</h1>"}))
  (is (= (parse metadata)
         {:metadata {:meta "Test"}
          :contents "<h1>Hello world!</h1>"}))
  (is (= (parse no-metadata-no-heading)
         {:metadata {}
          :contents "<p>Test test!</p>"}))
  (is (= (parse metadata-no-heading)
         {:metadata {:meta "Test"}
          :contents "<p>Test test!</p>"})))

(def metadata-malformed "Meta: Test\n# Test test!")
(def malformed "")
(def just-metadata "Meta: Test\n")
(def just-metadata2 "Meta: Test")

(deftest can-parse-malformed-files
  (is (= (parse metadata-malformed)
         {:metadata {}
          :contents "<p>Meta: Test<h1>Test test!</h1></p>"}))
  (is (= (parse malformed)
         {:metadata {}
          :contents ""}))
  (is (= (parse just-metadata)
         {:metadata {}
          :contents "<p>Meta: Test</p>"}))
  (is (= (parse just-metadata2)
         {:metadata {}
          :contents "<p>Meta: Test</p>"})))<commit_msg>Added some tests for the multi-valued metadata.<commit_after>(ns lambda-blog.md-parser-test
  (:require [clojure.test :refer :all]
            [lambda-blog.parsers.md :refer [parse]]))

(def no-metadata "# Hello world!")
(def metadata "Meta: Test\n\n# Hello world!")
(def no-metadata-no-heading "Test test!")
(def metadata-no-heading "Meta: Test\n\nTest test!")

(deftest can-parse-md
  (is (= (parse no-metadata)
         {:metadata {}
          :contents "<h1>Hello world!</h1>"}))
  (is (= (parse metadata)
         {:metadata {:meta "Test"}
          :contents "<h1>Hello world!</h1>"}))
  (is (= (parse no-metadata-no-heading)
         {:metadata {}
          :contents "<p>Test test!</p>"}))
  (is (= (parse metadata-no-heading)
         {:metadata {:meta "Test"}
          :contents "<p>Test test!</p>"})))

(def metadata-malformed "Meta: Test\n# Test test!")
(def malformed "")
(def just-metadata "Meta: Test\n")
(def just-metadata2 "Meta: Test")

(deftest can-parse-malformed-files
  (is (= (parse metadata-malformed)
         {:metadata {}
          :contents "<p>Meta: Test<h1>Test test!</h1></p>"}))
  (is (= (parse malformed)
         {:metadata {}
          :contents ""}))
  (is (= (parse just-metadata)
         {:metadata {}
          :contents "<p>Meta: Test</p>"}))
  (is (= (parse just-metadata2)
         {:metadata {}
          :contents "<p>Meta: Test</p>"})))

(def metadata-multi "Meta: test1
      test2
      test3
Data: foo bar baz

# Header")

(deftest can-add-multi-values-metadata
  (is (= (parse metadata-multi)
         {:metadata {:meta ["test1" "test2" "test3"]
                     :data "foo bar baz"}
          :contents "<h1>Header</h1>"})))
{'fraction_non_alphanumeric': 0.1990756025090789, 'fraction_numerical': 0.007923407065037967, 'mean_word_length': 2.740740740740741, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2474148', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1042, 'n_words': 246}