<commit_before>(ns vip.data-processor.s3-test
  (:require [vip.data-processor.s3 :refer :all]
            [clojure.test :refer [deftest testing is]]))

(deftest zip-filename*-test
  (testing "nicely formatted dates make nice filenames"
    (is (= "vipfeed-51-2015-03-24.zip"
           (zip-filename* "51" "2015-03-24"))))

  (testing "workably formatted dates make nice filenames"
    (is (= "vipfeed-52-2015-03-27.zip"
           (zip-filename* "52" "2015/03/27"))))

  (testing "poorly formatted dates use a workable filename"
    (is (= "vipfeed-12345.zip"
           (zip-filename* "12345" nil)))))
<commit_msg>We ignore "dates" that aren't useful<commit_after>(ns vip.data-processor.s3-test
  (:require [vip.data-processor.s3 :refer :all]
            [clojure.test :refer [deftest testing is]]))

(deftest zip-filename*-test
  (testing "nicely formatted dates make nice filenames"
    (is (= "vipfeed-51-2015-03-24.zip"
           (zip-filename* "51" "2015-03-24"))))

  (testing "workably formatted dates make nice filenames"
    (is (= "vipfeed-52-2015-03-27.zip"
           (zip-filename* "52" "2015/03/27"))))

  (testing "poorly formatted dates use a workable filename"
    (is (= "vipfeed-12345-.zip"
           (zip-filename* "12345" nil)))))

{'fraction_non_alphanumeric': 0.16401273885350318, 'fraction_numerical': 0.08280254777070063, 'mean_word_length': 3.0944625407166124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2253257', 'n_tokens_mistral': 528, 'n_tokens_neox': 446, 'n_words': 105}